Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 72%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union 

34 

35import sqlalchemy 

36 

37from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

38from ._versioning import VersionedExtension 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from ...core import ( 

42 DataCoordinateIterable, 

43 DimensionElement, 

44 DimensionRecord, 

45 DimensionUniverse, 

46 NamedKeyDict, 

47 NamedKeyMapping, 

48 TimespanDatabaseRepresentation, 

49 ) 

50 from ..queries import QueryBuilder 

51 from ._database import Database, StaticTablesContext 

52 

53 

54OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

55 

56 

57class DimensionRecordStorage(ABC): 

58 """An abstract base class that represents a way of storing the records 

59 associated with a single `DimensionElement`. 

60 

61 Concrete `DimensionRecordStorage` instances should generally be constructed 

62 via a call to `setupDimensionStorage`, which selects the appropriate 

63 subclass for each element according to its configuration. 

64 

65 All `DimensionRecordStorage` methods are pure abstract, even though in some 

66 cases a reasonable default implementation might be possible, in order to 

67 better guarantee all methods are correctly overridden. All of these 

68 potentially-defaultable implementations are extremely trivial, so asking 

69 subclasses to provide them is not a significant burden. 

70 """ 

71 

72 @property 

73 @abstractmethod 

74 def element(self) -> DimensionElement: 

75 """The element whose records this instance managers 

76 (`DimensionElement`). 

77 """ 

78 raise NotImplementedError() 

79 

80 @abstractmethod 

81 def clearCaches(self) -> None: 

82 """Clear any in-memory caches held by the storage instance. 

83 

84 This is called by `Registry` when transactions are rolled back, to 

85 avoid in-memory caches from ever containing records that are not 

86 present in persistent storage. 

87 """ 

88 raise NotImplementedError() 

89 

90 @abstractmethod 

91 def join( 

92 self, 

93 builder: QueryBuilder, 

94 *, 

95 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

96 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

97 ) -> sqlalchemy.sql.FromClause: 

98 """Add the dimension element's logical table to a query under 

99 construction. 

100 

101 This is a visitor pattern interface that is expected to be called only 

102 by `QueryBuilder.joinDimensionElement`. 

103 

104 Parameters 

105 ---------- 

106 builder : `QueryBuilder` 

107 Builder for the query that should contain this element. 

108 regions : `NamedKeyDict`, optional 

109 A mapping from `DimensionElement` to a SQLAlchemy column containing 

110 the region for that element, which should be updated to include a 

111 region column for this element if one exists. If `None`, 

112 ``self.element`` is not being included in the query via a spatial 

113 join. 

114 timespan : `NamedKeyDict`, optional 

115 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

116 columns containing the timespan for that element, which should be 

117 updated to include timespan columns for this element if they exist. 

118 If `None`, ``self.element`` is not being included in the query via 

119 a temporal join. 

120 

121 Returns 

122 ------- 

123 fromClause : `sqlalchemy.sql.FromClause` 

124 Table or clause for the element which is joined. 

125 

126 Notes 

127 ----- 

128 Elements are only included in queries via spatial and/or temporal joins 

129 when necessary to connect them to other elements in the query, so 

130 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

131 because an element has a region or timespan. 

132 """ 

133 raise NotImplementedError() 

134 

135 @abstractmethod 

136 def insert(self, *records: DimensionRecord, replace: bool = False) -> None: 

137 """Insert one or more records into storage. 

138 

139 Parameters 

140 ---------- 

141 records 

142 One or more instances of the `DimensionRecord` subclass for the 

143 element this storage is associated with. 

144 replace: `bool`, optional 

145 If `True` (`False` is default), replace existing records in the 

146 database if there is a conflict. 

147 

148 Raises 

149 ------ 

150 TypeError 

151 Raised if the element does not support record insertion. 

152 sqlalchemy.exc.IntegrityError 

153 Raised if one or more records violate database integrity 

154 constraints. 

155 

156 Notes 

157 ----- 

158 As `insert` is expected to be called only by a `Registry`, we rely 

159 on `Registry` to provide transactionality, both by using a SQLALchemy 

160 connection shared with the `Registry` and by relying on it to call 

161 `clearCaches` when rolling back transactions. 

162 """ 

163 raise NotImplementedError() 

164 

165 @abstractmethod 

166 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]: 

167 """Synchronize a record with the database, inserting it only if it does 

168 not exist and comparing values if it does. 

169 

170 Parameters 

171 ---------- 

172 record : `DimensionRecord`. 

173 An instance of the `DimensionRecord` subclass for the 

174 element this storage is associated with. 

175 update: `bool`, optional 

176 If `True` (`False` is default), update the existing record in the 

177 database if there is a conflict. 

178 

179 Returns 

180 ------- 

181 inserted_or_updated : `bool` or `dict` 

182 `True` if a new row was inserted, `False` if no changes were 

183 needed, or a `dict` mapping updated column names to their old 

184 values if an update was performed (only possible if 

185 ``update=True``). 

186 

187 Raises 

188 ------ 

189 DatabaseConflictError 

190 Raised if the record exists in the database (according to primary 

191 key lookup) but is inconsistent with the given one. 

192 TypeError 

193 Raised if the element does not support record synchronization. 

194 sqlalchemy.exc.IntegrityError 

195 Raised if one or more records violate database integrity 

196 constraints. 

197 """ 

198 raise NotImplementedError() 

199 

200 @abstractmethod 

201 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

202 """Retrieve records from storage. 

203 

204 Parameters 

205 ---------- 

206 dataIds : `DataCoordinateIterable` 

207 Data IDs that identify the records to be retrieved. 

208 

209 Returns 

210 ------- 

211 records : `Iterable` [ `DimensionRecord` ] 

212 Record retrieved from storage. Not all data IDs may have 

213 corresponding records (if there are no records that match a data 

214 ID), and even if they are, the order of inputs is not preserved. 

215 """ 

216 raise NotImplementedError() 

217 

218 @abstractmethod 

219 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

220 """Return tables used for schema digest. 

221 

222 Returns 

223 ------- 

224 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

225 Possibly empty set of tables for schema digest calculations. 

226 """ 

227 raise NotImplementedError() 

228 

229 

230class GovernorDimensionRecordStorage(DimensionRecordStorage): 

231 """Intermediate interface for `DimensionRecordStorage` objects that provide 

232 storage for `GovernorDimension` instances. 

233 """ 

234 

235 @classmethod 

236 @abstractmethod 

237 def initialize( 

238 cls, 

239 db: Database, 

240 dimension: GovernorDimension, 

241 *, 

242 context: Optional[StaticTablesContext] = None, 

243 config: Mapping[str, Any], 

244 ) -> GovernorDimensionRecordStorage: 

245 """Construct an instance of this class using a standardized interface. 

246 

247 Parameters 

248 ---------- 

249 db : `Database` 

250 Interface to the underlying database engine and namespace. 

251 dimension : `GovernorDimension` 

252 Dimension the new instance will manage records for. 

253 context : `StaticTablesContext`, optional 

254 If provided, an object to use to create any new tables. If not 

255 provided, ``db.ensureTableExists`` should be used instead. 

256 config : `Mapping` 

257 Extra configuration options specific to the implementation. 

258 

259 Returns 

260 ------- 

261 storage : `GovernorDimensionRecordStorage` 

262 A new `GovernorDimensionRecordStorage` subclass instance. 

263 """ 

264 raise NotImplementedError() 

265 

266 @property 

267 @abstractmethod 

268 def element(self) -> GovernorDimension: 

269 # Docstring inherited from DimensionRecordStorage. 

270 raise NotImplementedError() 

271 

272 @abstractmethod 

273 def refresh(self) -> None: 

274 """Ensure all other operations on this manager are aware of any 

275 changes made by other clients since it was initialized or last 

276 refreshed. 

277 """ 

278 raise NotImplementedError() 

279 

280 @property 

281 @abstractmethod 

282 def values(self) -> AbstractSet[str]: 

283 """All primary key values for this dimension (`set` [ `str` ]). 

284 

285 This may rely on an in-memory cache and hence not reflect changes to 

286 the set of values made by other `Butler` / `Registry` clients. Call 

287 `refresh` to ensure up-to-date results. 

288 """ 

289 raise NotImplementedError() 

290 

291 @property 

292 @abstractmethod 

293 def table(self) -> sqlalchemy.schema.Table: 

294 """The SQLAlchemy table that backs this dimension 

295 (`sqlalchemy.schema.Table`). 

296 """ 

297 raise NotImplementedError 

298 

299 @abstractmethod 

300 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

301 """Add a function or method to be called after new records for this 

302 dimension are inserted by `insert` or `sync`. 

303 

304 Parameters 

305 ---------- 

306 callback 

307 Callable that takes a single `DimensionRecord` argument. This will 

308 be called immediately after any successful insertion, in the same 

309 transaction. 

310 """ 

311 raise NotImplementedError() 

312 

313 

314class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

315 """Intermediate interface for `DimensionRecordStorage` objects that provide 

316 storage for `SkyPixDimension` instances. 

317 """ 

318 

319 @property 

320 @abstractmethod 

321 def element(self) -> SkyPixDimension: 

322 # Docstring inherited from DimensionRecordStorage. 

323 raise NotImplementedError() 

324 

325 

326class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

327 """Intermediate interface for `DimensionRecordStorage` objects that provide 

328 storage for `DatabaseDimensionElement` instances. 

329 """ 

330 

331 @classmethod 

332 @abstractmethod 

333 def initialize( 

334 cls, 

335 db: Database, 

336 element: DatabaseDimensionElement, 

337 *, 

338 context: Optional[StaticTablesContext] = None, 

339 config: Mapping[str, Any], 

340 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

341 ) -> DatabaseDimensionRecordStorage: 

342 """Construct an instance of this class using a standardized interface. 

343 

344 Parameters 

345 ---------- 

346 db : `Database` 

347 Interface to the underlying database engine and namespace. 

348 element : `DatabaseDimensionElement` 

349 Dimension element the new instance will manage records for. 

350 context : `StaticTablesContext`, optional 

351 If provided, an object to use to create any new tables. If not 

352 provided, ``db.ensureTableExists`` should be used instead. 

353 config : `Mapping` 

354 Extra configuration options specific to the implementation. 

355 governors : `NamedKeyMapping` 

356 Mapping containing all governor dimension storage implementations. 

357 

358 Returns 

359 ------- 

360 storage : `DatabaseDimensionRecordStorage` 

361 A new `DatabaseDimensionRecordStorage` subclass instance. 

362 """ 

363 raise NotImplementedError() 

364 

365 @property 

366 @abstractmethod 

367 def element(self) -> DatabaseDimensionElement: 

368 # Docstring inherited from DimensionRecordStorage. 

369 raise NotImplementedError() 

370 

371 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

372 """Inform this record storage object of the object that will manage 

373 the overlaps between this element and another element. 

374 

375 This will only be called if ``self.element.spatial is not None``, 

376 and will be called immediately after construction (before any other 

377 methods). In the future, implementations will be required to call a 

378 method on any connected overlap storage objects any time new records 

379 for the element are inserted. 

380 

381 Parameters 

382 ---------- 

383 overlaps : `DatabaseDimensionRecordStorage` 

384 Object managing overlaps between this element and another 

385 database-backed element. 

386 """ 

387 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

388 

389 

390class DatabaseDimensionOverlapStorage(ABC): 

391 """A base class for objects that manage overlaps between a pair of 

392 database-backed dimensions. 

393 """ 

394 

395 @classmethod 

396 @abstractmethod 

397 def initialize( 

398 cls, 

399 db: Database, 

400 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

401 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

402 context: Optional[StaticTablesContext] = None, 

403 ) -> DatabaseDimensionOverlapStorage: 

404 """Construct an instance of this class using a standardized interface. 

405 

406 Parameters 

407 ---------- 

408 db : `Database` 

409 Interface to the underlying database engine and namespace. 

410 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

411 Storage objects for the elements this object will related. 

412 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

413 Storage objects for the governor dimensions of the elements this 

414 object will related. 

415 context : `StaticTablesContext`, optional 

416 If provided, an object to use to create any new tables. If not 

417 provided, ``db.ensureTableExists`` should be used instead. 

418 

419 Returns 

420 ------- 

421 storage : `DatabaseDimensionOverlapStorage` 

422 A new `DatabaseDimensionOverlapStorage` subclass instance. 

423 """ 

424 raise NotImplementedError() 

425 

426 @property 

427 @abstractmethod 

428 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

429 """The pair of elements whose overlaps this object manages. 

430 

431 The order of elements is the same as their ordering within the 

432 `DimensionUniverse`. 

433 """ 

434 raise NotImplementedError() 

435 

436 @abstractmethod 

437 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

438 """Return tables used for schema digest. 

439 

440 Returns 

441 ------- 

442 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

443 Possibly empty set of tables for schema digest calculations. 

444 """ 

445 raise NotImplementedError() 

446 

447 

448class DimensionRecordStorageManager(VersionedExtension): 

449 """An interface for managing the dimension records in a `Registry`. 

450 

451 `DimensionRecordStorageManager` primarily serves as a container and factory 

452 for `DimensionRecordStorage` instances, which each provide access to the 

453 records for a different `DimensionElement`. 

454 

455 Parameters 

456 ---------- 

457 universe : `DimensionUniverse` 

458 Universe of all dimensions and dimension elements known to the 

459 `Registry`. 

460 

461 Notes 

462 ----- 

463 In a multi-layer `Registry`, many dimension elements will only have 

464 records in one layer (often the base layer). The union of the records 

465 across all layers forms the logical table for the full `Registry`. 

466 """ 

467 

468 def __init__(self, *, universe: DimensionUniverse): 

469 self.universe = universe 

470 

471 @classmethod 

472 @abstractmethod 

473 def initialize( 

474 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse 

475 ) -> DimensionRecordStorageManager: 

476 """Construct an instance of the manager. 

477 

478 Parameters 

479 ---------- 

480 db : `Database` 

481 Interface to the underlying database engine and namespace. 

482 context : `StaticTablesContext` 

483 Context object obtained from `Database.declareStaticTables`; used 

484 to declare any tables that should always be present in a layer 

485 implemented with this manager. 

486 universe : `DimensionUniverse` 

487 Universe graph containing dimensions known to this `Registry`. 

488 

489 Returns 

490 ------- 

491 manager : `DimensionRecordStorageManager` 

492 An instance of a concrete `DimensionRecordStorageManager` subclass. 

493 """ 

494 raise NotImplementedError() 

495 

496 @abstractmethod 

497 def refresh(self) -> None: 

498 """Ensure all other operations on this manager are aware of any 

499 changes made by other clients since it was initialized or last 

500 refreshed. 

501 """ 

502 raise NotImplementedError() 

503 

504 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

505 """Interface to `get` that raises `LookupError` instead of returning 

506 `None` on failure. 

507 """ 

508 r = self.get(element) 

509 if r is None: 

510 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

511 return r 

512 

513 @abstractmethod 

514 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

515 """Return an object that provides access to the records associated with 

516 the given element, if one exists in this layer. 

517 

518 Parameters 

519 ---------- 

520 element : `DimensionElement` 

521 Element for which records should be returned. 

522 

523 Returns 

524 ------- 

525 records : `DimensionRecordStorage` or `None` 

526 The object representing the records for the given element in this 

527 layer, or `None` if there are no records for that element in this 

528 layer. 

529 

530 Notes 

531 ----- 

532 Dimension elements registered by another client of the same layer since 

533 the last call to `initialize` or `refresh` may not be found. 

534 """ 

535 raise NotImplementedError() 

536 

537 @abstractmethod 

538 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

539 """Ensure that this layer can hold records for the given element, 

540 creating new tables as necessary. 

541 

542 Parameters 

543 ---------- 

544 element : `DimensionElement` 

545 Element for which a table should created (as necessary) and 

546 an associated `DimensionRecordStorage` returned. 

547 

548 Returns 

549 ------- 

550 records : `DimensionRecordStorage` 

551 The object representing the records for the given element in this 

552 layer. 

553 

554 Raises 

555 ------ 

556 TransactionInterruption 

557 Raised if this operation is invoked within a `Database.transaction` 

558 context. 

559 """ 

560 raise NotImplementedError() 

561 

562 @abstractmethod 

563 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

564 """Save a `DimensionGraph` definition to the database, allowing it to 

565 be retrieved later via the returned key. 

566 

567 Parameters 

568 ---------- 

569 graph : `DimensionGraph` 

570 Set of dimensions to save. 

571 

572 Returns 

573 ------- 

574 key : `int` 

575 Integer used as the unique key for this `DimensionGraph` in the 

576 database. 

577 

578 Raises 

579 ------ 

580 TransactionInterruption 

581 Raised if this operation is invoked within a `Database.transaction` 

582 context. 

583 """ 

584 raise NotImplementedError() 

585 

586 @abstractmethod 

587 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

588 """Retrieve a `DimensionGraph` that was previously saved in the 

589 database. 

590 

591 Parameters 

592 ---------- 

593 key : `int` 

594 Integer used as the unique key for this `DimensionGraph` in the 

595 database. 

596 

597 Returns 

598 ------- 

599 graph : `DimensionGraph` 

600 Retrieved graph. 

601 

602 Raises 

603 ------ 

604 KeyError 

605 Raised if the given key cannot be found in the database. 

606 """ 

607 raise NotImplementedError() 

608 

609 @abstractmethod 

610 def clearCaches(self) -> None: 

611 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

612 instances. 

613 

614 This is called by `Registry` when transactions are rolled back, to 

615 avoid in-memory caches from ever containing records that are not 

616 present in persistent storage. 

617 """ 

618 raise NotImplementedError() 

619 

620 universe: DimensionUniverse 

621 """Universe of all dimensions and dimension elements known to the 

622 `Registry` (`DimensionUniverse`). 

623 """