Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 69%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from typing import ( 

34 AbstractSet, Any, 

35 Callable, 

36 Dict, 

37 Iterable, Mapping, 

38 Optional, 

39 Tuple, 

40 TYPE_CHECKING, 

41 Union, 

42) 

43 

44import sqlalchemy 

45 

46from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

47from ._versioning import VersionedExtension 

48 

49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true

50 from ...core import ( 

51 DataCoordinateIterable, 

52 DimensionElement, 

53 DimensionRecord, 

54 DimensionUniverse, 

55 NamedKeyDict, 

56 NamedKeyMapping, 

57 TimespanDatabaseRepresentation, 

58 ) 

59 from ..queries import QueryBuilder 

60 from ._database import Database, StaticTablesContext 

61 

62 

63OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

64 

65 

66class DimensionRecordStorage(ABC): 

67 """An abstract base class that represents a way of storing the records 

68 associated with a single `DimensionElement`. 

69 

70 Concrete `DimensionRecordStorage` instances should generally be constructed 

71 via a call to `setupDimensionStorage`, which selects the appropriate 

72 subclass for each element according to its configuration. 

73 

74 All `DimensionRecordStorage` methods are pure abstract, even though in some 

75 cases a reasonable default implementation might be possible, in order to 

76 better guarantee all methods are correctly overridden. All of these 

77 potentially-defaultable implementations are extremely trivial, so asking 

78 subclasses to provide them is not a significant burden. 

79 """ 

80 

81 @property 

82 @abstractmethod 

83 def element(self) -> DimensionElement: 

84 """The element whose records this instance holds (`DimensionElement`). 

85 """ 

86 raise NotImplementedError() 

87 

88 @abstractmethod 

89 def clearCaches(self) -> None: 

90 """Clear any in-memory caches held by the storage instance. 

91 

92 This is called by `Registry` when transactions are rolled back, to 

93 avoid in-memory caches from ever containing records that are not 

94 present in persistent storage. 

95 """ 

96 raise NotImplementedError() 

97 

98 @abstractmethod 

99 def join( 

100 self, 

101 builder: QueryBuilder, *, 

102 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

103 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

104 ) -> sqlalchemy.sql.FromClause: 

105 """Add the dimension element's logical table to a query under 

106 construction. 

107 

108 This is a visitor pattern interface that is expected to be called only 

109 by `QueryBuilder.joinDimensionElement`. 

110 

111 Parameters 

112 ---------- 

113 builder : `QueryBuilder` 

114 Builder for the query that should contain this element. 

115 regions : `NamedKeyDict`, optional 

116 A mapping from `DimensionElement` to a SQLAlchemy column containing 

117 the region for that element, which should be updated to include a 

118 region column for this element if one exists. If `None`, 

119 ``self.element`` is not being included in the query via a spatial 

120 join. 

121 timespan : `NamedKeyDict`, optional 

122 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

123 columns containing the timespan for that element, which should be 

124 updated to include timespan columns for this element if they exist. 

125 If `None`, ``self.element`` is not being included in the query via 

126 a temporal join. 

127 

128 Returns 

129 ------- 

130 fromClause : `sqlalchemy.sql.FromClause` 

131 Table or clause for the element which is joined. 

132 

133 Notes 

134 ----- 

135 Elements are only included in queries via spatial and/or temporal joins 

136 when necessary to connect them to other elements in the query, so 

137 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

138 because an element has a region or timespan. 

139 """ 

140 raise NotImplementedError() 

141 

142 @abstractmethod 

143 def insert(self, *records: DimensionRecord, replace: bool = False) -> None: 

144 """Insert one or more records into storage. 

145 

146 Parameters 

147 ---------- 

148 records 

149 One or more instances of the `DimensionRecord` subclass for the 

150 element this storage is associated with. 

151 replace: `bool`, optional 

152 If `True` (`False` is default), replace existing records in the 

153 database if there is a conflict. 

154 

155 Raises 

156 ------ 

157 TypeError 

158 Raised if the element does not support record insertion. 

159 sqlalchemy.exc.IntegrityError 

160 Raised if one or more records violate database integrity 

161 constraints. 

162 

163 Notes 

164 ----- 

165 As `insert` is expected to be called only by a `Registry`, we rely 

166 on `Registry` to provide transactionality, both by using a SQLALchemy 

167 connection shared with the `Registry` and by relying on it to call 

168 `clearCaches` when rolling back transactions. 

169 """ 

170 raise NotImplementedError() 

171 

172 @abstractmethod 

173 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]: 

174 """Synchronize a record with the database, inserting it only if it does 

175 not exist and comparing values if it does. 

176 

177 Parameters 

178 ---------- 

179 record : `DimensionRecord`. 

180 An instance of the `DimensionRecord` subclass for the 

181 element this storage is associated with. 

182 update: `bool`, optional 

183 If `True` (`False` is default), update the existing record in the 

184 database if there is a conflict. 

185 

186 Returns 

187 ------- 

188 inserted_or_updated : `bool` or `dict` 

189 `True` if a new row was inserted, `False` if no changes were 

190 needed, or a `dict` mapping updated column names to their old 

191 values if an update was performed (only possible if 

192 ``update=True``). 

193 

194 Raises 

195 ------ 

196 DatabaseConflictError 

197 Raised if the record exists in the database (according to primary 

198 key lookup) but is inconsistent with the given one. 

199 TypeError 

200 Raised if the element does not support record synchronization. 

201 sqlalchemy.exc.IntegrityError 

202 Raised if one or more records violate database integrity 

203 constraints. 

204 """ 

205 raise NotImplementedError() 

206 

207 @abstractmethod 

208 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

209 """Retrieve records from storage. 

210 

211 Parameters 

212 ---------- 

213 dataIds : `DataCoordinateIterable` 

214 Data IDs that identify the records to be retrieved. 

215 

216 Returns 

217 ------- 

218 records : `Iterable` [ `DimensionRecord` ] 

219 Record retrieved from storage. Not all data IDs may have 

220 corresponding records (if there are no records that match a data 

221 ID), and even if they are, the order of inputs is not preserved. 

222 """ 

223 raise NotImplementedError() 

224 

225 @abstractmethod 

226 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

227 """Return tables used for schema digest. 

228 

229 Returns 

230 ------- 

231 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

232 Possibly empty set of tables for schema digest calculations. 

233 """ 

234 raise NotImplementedError() 

235 

236 

237class GovernorDimensionRecordStorage(DimensionRecordStorage): 

238 """Intermediate interface for `DimensionRecordStorage` objects that provide 

239 storage for `GovernorDimension` instances. 

240 """ 

241 

242 @classmethod 

243 @abstractmethod 

244 def initialize(cls, db: Database, dimension: GovernorDimension, *, 

245 context: Optional[StaticTablesContext] = None, 

246 config: Mapping[str, Any], 

247 ) -> GovernorDimensionRecordStorage: 

248 """Construct an instance of this class using a standardized interface. 

249 

250 Parameters 

251 ---------- 

252 db : `Database` 

253 Interface to the underlying database engine and namespace. 

254 dimension : `GovernorDimension` 

255 Dimension the new instance will manage records for. 

256 context : `StaticTablesContext`, optional 

257 If provided, an object to use to create any new tables. If not 

258 provided, ``db.ensureTableExists`` should be used instead. 

259 config : `Mapping` 

260 Extra configuration options specific to the implementation. 

261 

262 Returns 

263 ------- 

264 storage : `GovernorDimensionRecordStorage` 

265 A new `GovernorDimensionRecordStorage` subclass instance. 

266 """ 

267 raise NotImplementedError() 

268 

269 @property 

270 @abstractmethod 

271 def element(self) -> GovernorDimension: 

272 # Docstring inherited from DimensionRecordStorage. 

273 raise NotImplementedError() 

274 

275 @abstractmethod 

276 def refresh(self) -> None: 

277 """Ensure all other operations on this manager are aware of any 

278 changes made by other clients since it was initialized or last 

279 refreshed. 

280 """ 

281 raise NotImplementedError() 

282 

283 @property 

284 @abstractmethod 

285 def values(self) -> AbstractSet[str]: 

286 """All primary key values for this dimension (`set` [ `str` ]). 

287 

288 This may rely on an in-memory cache and hence not reflect changes to 

289 the set of values made by other `Butler` / `Registry` clients. Call 

290 `refresh` to ensure up-to-date results. 

291 """ 

292 raise NotImplementedError() 

293 

294 @property 

295 @abstractmethod 

296 def table(self) -> sqlalchemy.schema.Table: 

297 """The SQLAlchemy table that backs this dimension 

298 (`sqlalchemy.schema.Table`). 

299 """ 

300 raise NotImplementedError 

301 

302 @abstractmethod 

303 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

304 """Add a function or method to be called after new records for this 

305 dimension are inserted by `insert` or `sync`. 

306 

307 Parameters 

308 ---------- 

309 callback 

310 Callable that takes a single `DimensionRecord` argument. This will 

311 be called immediately after any successful insertion, in the same 

312 transaction. 

313 """ 

314 raise NotImplementedError() 

315 

316 

317class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

318 """Intermediate interface for `DimensionRecordStorage` objects that provide 

319 storage for `SkyPixDimension` instances. 

320 """ 

321 

322 @property 

323 @abstractmethod 

324 def element(self) -> SkyPixDimension: 

325 # Docstring inherited from DimensionRecordStorage. 

326 raise NotImplementedError() 

327 

328 

329class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

330 """Intermediate interface for `DimensionRecordStorage` objects that provide 

331 storage for `DatabaseDimensionElement` instances. 

332 """ 

333 

334 @classmethod 

335 @abstractmethod 

336 def initialize(cls, db: Database, element: DatabaseDimensionElement, *, 

337 context: Optional[StaticTablesContext] = None, 

338 config: Mapping[str, Any], 

339 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

340 ) -> DatabaseDimensionRecordStorage: 

341 """Construct an instance of this class using a standardized interface. 

342 

343 Parameters 

344 ---------- 

345 db : `Database` 

346 Interface to the underlying database engine and namespace. 

347 element : `DatabaseDimensionElement` 

348 Dimension element the new instance will manage records for. 

349 context : `StaticTablesContext`, optional 

350 If provided, an object to use to create any new tables. If not 

351 provided, ``db.ensureTableExists`` should be used instead. 

352 config : `Mapping` 

353 Extra configuration options specific to the implementation. 

354 governors : `NamedKeyMapping` 

355 Mapping containing all governor dimension storage implementations. 

356 

357 Returns 

358 ------- 

359 storage : `DatabaseDimensionRecordStorage` 

360 A new `DatabaseDimensionRecordStorage` subclass instance. 

361 """ 

362 raise NotImplementedError() 

363 

364 @property 

365 @abstractmethod 

366 def element(self) -> DatabaseDimensionElement: 

367 # Docstring inherited from DimensionRecordStorage. 

368 raise NotImplementedError() 

369 

370 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

371 """Inform this record storage object of the object that will manage 

372 the overlaps between this element and another element. 

373 

374 This will only be called if ``self.element.spatial is not None``, 

375 and will be called immediately after construction (before any other 

376 methods). In the future, implementations will be required to call a 

377 method on any connected overlap storage objects any time new records 

378 for the element are inserted. 

379 

380 Parameters 

381 ---------- 

382 overlaps : `DatabaseDimensionRecordStorage` 

383 Object managing overlaps between this element and another 

384 database-backed element. 

385 """ 

386 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

387 

388 

389class DatabaseDimensionOverlapStorage(ABC): 

390 """A base class for objects that manage overlaps between a pair of 

391 database-backed dimensions. 

392 """ 

393 

394 @classmethod 

395 @abstractmethod 

396 def initialize( 

397 cls, 

398 db: Database, 

399 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

400 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

401 context: Optional[StaticTablesContext] = None, 

402 ) -> DatabaseDimensionOverlapStorage: 

403 """Construct an instance of this class using a standardized interface. 

404 

405 Parameters 

406 ---------- 

407 db : `Database` 

408 Interface to the underlying database engine and namespace. 

409 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

410 Storage objects for the elements this object will related. 

411 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

412 Storage objects for the governor dimensions of the elements this 

413 object will related. 

414 context : `StaticTablesContext`, optional 

415 If provided, an object to use to create any new tables. If not 

416 provided, ``db.ensureTableExists`` should be used instead. 

417 

418 Returns 

419 ------- 

420 storage : `DatabaseDimensionOverlapStorage` 

421 A new `DatabaseDimensionOverlapStorage` subclass instance. 

422 """ 

423 raise NotImplementedError() 

424 

425 @property 

426 @abstractmethod 

427 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

428 """The pair of elements whose overlaps this object manages. 

429 

430 The order of elements is the same as their ordering within the 

431 `DimensionUniverse`. 

432 """ 

433 raise NotImplementedError() 

434 

435 @abstractmethod 

436 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

437 """Return tables used for schema digest. 

438 

439 Returns 

440 ------- 

441 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

442 Possibly empty set of tables for schema digest calculations. 

443 """ 

444 raise NotImplementedError() 

445 

446 

447class DimensionRecordStorageManager(VersionedExtension): 

448 """An interface for managing the dimension records in a `Registry`. 

449 

450 `DimensionRecordStorageManager` primarily serves as a container and factory 

451 for `DimensionRecordStorage` instances, which each provide access to the 

452 records for a different `DimensionElement`. 

453 

454 Parameters 

455 ---------- 

456 universe : `DimensionUniverse` 

457 Universe of all dimensions and dimension elements known to the 

458 `Registry`. 

459 

460 Notes 

461 ----- 

462 In a multi-layer `Registry`, many dimension elements will only have 

463 records in one layer (often the base layer). The union of the records 

464 across all layers forms the logical table for the full `Registry`. 

465 """ 

466 def __init__(self, *, universe: DimensionUniverse): 

467 self.universe = universe 

468 

469 @classmethod 

470 @abstractmethod 

471 def initialize(cls, db: Database, context: StaticTablesContext, *, 

472 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

473 """Construct an instance of the manager. 

474 

475 Parameters 

476 ---------- 

477 db : `Database` 

478 Interface to the underlying database engine and namespace. 

479 context : `StaticTablesContext` 

480 Context object obtained from `Database.declareStaticTables`; used 

481 to declare any tables that should always be present in a layer 

482 implemented with this manager. 

483 universe : `DimensionUniverse` 

484 Universe graph containing dimensions known to this `Registry`. 

485 

486 Returns 

487 ------- 

488 manager : `DimensionRecordStorageManager` 

489 An instance of a concrete `DimensionRecordStorageManager` subclass. 

490 """ 

491 raise NotImplementedError() 

492 

493 @abstractmethod 

494 def refresh(self) -> None: 

495 """Ensure all other operations on this manager are aware of any 

496 changes made by other clients since it was initialized or last 

497 refreshed. 

498 """ 

499 raise NotImplementedError() 

500 

501 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

502 """Interface to `get` that raises `LookupError` instead of returning 

503 `None` on failure. 

504 """ 

505 r = self.get(element) 

506 if r is None: 

507 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

508 return r 

509 

510 @abstractmethod 

511 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

512 """Return an object that provides access to the records associated with 

513 the given element, if one exists in this layer. 

514 

515 Parameters 

516 ---------- 

517 element : `DimensionElement` 

518 Element for which records should be returned. 

519 

520 Returns 

521 ------- 

522 records : `DimensionRecordStorage` or `None` 

523 The object representing the records for the given element in this 

524 layer, or `None` if there are no records for that element in this 

525 layer. 

526 

527 Notes 

528 ----- 

529 Dimension elements registered by another client of the same layer since 

530 the last call to `initialize` or `refresh` may not be found. 

531 """ 

532 raise NotImplementedError() 

533 

534 @abstractmethod 

535 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

536 """Ensure that this layer can hold records for the given element, 

537 creating new tables as necessary. 

538 

539 Parameters 

540 ---------- 

541 element : `DimensionElement` 

542 Element for which a table should created (as necessary) and 

543 an associated `DimensionRecordStorage` returned. 

544 

545 Returns 

546 ------- 

547 records : `DimensionRecordStorage` 

548 The object representing the records for the given element in this 

549 layer. 

550 

551 Raises 

552 ------ 

553 TransactionInterruption 

554 Raised if this operation is invoked within a `Database.transaction` 

555 context. 

556 """ 

557 raise NotImplementedError() 

558 

559 @abstractmethod 

560 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

561 """Save a `DimensionGraph` definition to the database, allowing it to 

562 be retrieved later via the returned key. 

563 

564 Parameters 

565 ---------- 

566 graph : `DimensionGraph` 

567 Set of dimensions to save. 

568 

569 Returns 

570 ------- 

571 key : `int` 

572 Integer used as the unique key for this `DimensionGraph` in the 

573 database. 

574 

575 Raises 

576 ------ 

577 TransactionInterruption 

578 Raised if this operation is invoked within a `Database.transaction` 

579 context. 

580 """ 

581 raise NotImplementedError() 

582 

583 @abstractmethod 

584 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

585 """Retrieve a `DimensionGraph` that was previously saved in the 

586 database. 

587 

588 Parameters 

589 ---------- 

590 key : `int` 

591 Integer used as the unique key for this `DimensionGraph` in the 

592 database. 

593 

594 Returns 

595 ------- 

596 graph : `DimensionGraph` 

597 Retrieved graph. 

598 

599 Raises 

600 ------ 

601 KeyError 

602 Raised if the given key cannot be found in the database. 

603 """ 

604 raise NotImplementedError() 

605 

606 @abstractmethod 

607 def clearCaches(self) -> None: 

608 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

609 instances. 

610 

611 This is called by `Registry` when transactions are rolled back, to 

612 avoid in-memory caches from ever containing records that are not 

613 present in persistent storage. 

614 """ 

615 raise NotImplementedError() 

616 

617 universe: DimensionUniverse 

618 """Universe of all dimensions and dimension elements known to the 

619 `Registry` (`DimensionUniverse`). 

620 """