Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["DimensionRecordStorage", "DimensionRecordStorageManager"] 

24 

25from abc import ABC, abstractmethod 

26from typing import Iterable, Optional, Type, TYPE_CHECKING 

27 

28import sqlalchemy 

29 

30from ...core import SkyPixDimension 

31from ._versioning import VersionedExtension 

32 

33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 from ...core import ( 

35 DataCoordinateIterable, 

36 DimensionElement, 

37 DimensionRecord, 

38 DimensionUniverse, 

39 NamedKeyDict, 

40 Timespan, 

41 ) 

42 from ..queries import QueryBuilder 

43 from ._database import Database, StaticTablesContext 

44 

45 

46class DimensionRecordStorage(ABC): 

47 """An abstract base class that represents a way of storing the records 

48 associated with a single `DimensionElement`. 

49 

50 Concrete `DimensionRecordStorage` instances should generally be constructed 

51 via a call to `setupDimensionStorage`, which selects the appropriate 

52 subclass for each element according to its configuration. 

53 

54 All `DimensionRecordStorage` methods are pure abstract, even though in some 

55 cases a reasonable default implementation might be possible, in order to 

56 better guarantee all methods are correctly overridden. All of these 

57 potentially-defaultable implementations are extremely trivial, so asking 

58 subclasses to provide them is not a significant burden. 

59 """ 

60 

61 @classmethod 

62 @abstractmethod 

63 def initialize(cls, db: Database, element: DimensionElement, *, 

64 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage: 

65 """Construct an instance of this class using a standardized interface. 

66 

67 Parameters 

68 ---------- 

69 db : `Database` 

70 Interface to the underlying database engine and namespace. 

71 element : `DimensionElement` 

72 Dimension element the new instance will manage records for. 

73 context : `StaticTablesContext`, optional 

74 If provided, an object to use to create any new tables. If not 

75 provided, ``db.ensureTableExists`` should be used instead. 

76 

77 Returns 

78 ------- 

79 storage : `DimensionRecordStorage` 

80 A new `DimensionRecordStorage` subclass instance. 

81 """ 

82 raise NotImplementedError() 

83 

84 @staticmethod 

85 def getDefaultImplementation(element: DimensionElement, ignoreCached: bool = False 

86 ) -> Type[DimensionRecordStorage]: 

87 """Return the default `DimensionRecordStorage` implementation for the 

88 given `DimensionElement`. 

89 

90 Parameters 

91 ---------- 

92 element : `DimensionElement` 

93 The element whose properties should be examined to determine the 

94 appropriate default implementation class. 

95 ignoreCached : `bool`, optional 

96 If `True`, ignore `DimensionElement.cached` and always return the 

97 storage implementation that would be used without caching. 

98 

99 Returns 

100 ------- 

101 cls : `type` 

102 A concrete subclass of `DimensionRecordStorage`. 

103 

104 Notes 

105 ----- 

106 At present, these defaults are always used, but we may add support for 

107 explicitly setting the class to use in configuration in the future. 

108 """ 

109 if not ignoreCached and element.cached: 

110 from ..dimensions.caching import CachingDimensionRecordStorage 

111 return CachingDimensionRecordStorage 

112 elif element.hasTable(): 

113 if element.viewOf is not None: 

114 if element.spatial is not None: 

115 raise NotImplementedError("Spatial view dimension storage is not supported.") 

116 from ..dimensions.query import QueryDimensionRecordStorage 

117 return QueryDimensionRecordStorage 

118 elif element.spatial is not None: 

119 from ..dimensions.spatial import SpatialDimensionRecordStorage 

120 return SpatialDimensionRecordStorage 

121 else: 

122 from ..dimensions.table import TableDimensionRecordStorage 

123 return TableDimensionRecordStorage 

124 elif isinstance(element, SkyPixDimension): 

125 from ..dimensions.skypix import SkyPixDimensionRecordStorage 

126 return SkyPixDimensionRecordStorage 

127 raise NotImplementedError(f"No default DimensionRecordStorage class for {element}.") 

128 

129 @property 

130 @abstractmethod 

131 def element(self) -> DimensionElement: 

132 """The element whose records this instance holds (`DimensionElement`). 

133 """ 

134 raise NotImplementedError() 

135 

136 @abstractmethod 

137 def clearCaches(self) -> None: 

138 """Clear any in-memory caches held by the storage instance. 

139 

140 This is called by `Registry` when transactions are rolled back, to 

141 avoid in-memory caches from ever containing records that are not 

142 present in persistent storage. 

143 """ 

144 raise NotImplementedError() 

145 

146 @abstractmethod 

147 def join( 

148 self, 

149 builder: QueryBuilder, *, 

150 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

151 timespans: Optional[NamedKeyDict[DimensionElement, Timespan[sqlalchemy.sql.ColumnElement]]] = None, 

152 ) -> sqlalchemy.sql.FromClause: 

153 """Add the dimension element's logical table to a query under 

154 construction. 

155 

156 This is a visitor pattern interface that is expected to be called only 

157 by `QueryBuilder.joinDimensionElement`. 

158 

159 Parameters 

160 ---------- 

161 builder : `QueryBuilder` 

162 Builder for the query that should contain this element. 

163 regions : `NamedKeyDict`, optional 

164 A mapping from `DimensionElement` to a SQLAlchemy column containing 

165 the region for that element, which should be updated to include a 

166 region column for this element if one exists. If `None`, 

167 ``self.element`` is not being included in the query via a spatial 

168 join. 

169 timespan : `NamedKeyDict`, optional 

170 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

171 columns containing the timespan for that element, which should be 

172 updated to include timespan columns for this element if they exist. 

173 If `None`, ``self.element`` is not being included in the query via 

174 a temporal join. 

175 

176 Returns 

177 ------- 

178 fromClause : `sqlalchemy.sql.FromClause` 

179 Table or clause for the element which is joined. 

180 

181 Notes 

182 ----- 

183 Elements are only included in queries via spatial and/or temporal joins 

184 when necessary to connect them to other elements in the query, so 

185 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

186 because an element has a region or timespan. 

187 """ 

188 raise NotImplementedError() 

189 

190 @abstractmethod 

191 def insert(self, *records: DimensionRecord) -> None: 

192 """Insert one or more records into storage. 

193 

194 Parameters 

195 ---------- 

196 records 

197 One or more instances of the `DimensionRecord` subclass for the 

198 element this storage is associated with. 

199 

200 Raises 

201 ------ 

202 TypeError 

203 Raised if the element does not support record insertion. 

204 sqlalchemy.exc.IntegrityError 

205 Raised if one or more records violate database integrity 

206 constraints. 

207 

208 Notes 

209 ----- 

210 As `insert` is expected to be called only by a `Registry`, we rely 

211 on `Registry` to provide transactionality, both by using a SQLALchemy 

212 connection shared with the `Registry` and by relying on it to call 

213 `clearCaches` when rolling back transactions. 

214 """ 

215 raise NotImplementedError() 

216 

217 @abstractmethod 

218 def sync(self, record: DimensionRecord) -> bool: 

219 """Synchronize a record with the database, inserting it only if it does 

220 not exist and comparing values if it does. 

221 

222 Parameters 

223 ---------- 

224 record : `DimensionRecord`. 

225 An instance of the `DimensionRecord` subclass for the 

226 element this storage is associated with. 

227 

228 Returns 

229 ------- 

230 inserted : `bool` 

231 `True` if a new row was inserted, `False` otherwise. 

232 

233 Raises 

234 ------ 

235 DatabaseConflictError 

236 Raised if the record exists in the database (according to primary 

237 key lookup) but is inconsistent with the given one. 

238 TypeError 

239 Raised if the element does not support record synchronization. 

240 sqlalchemy.exc.IntegrityError 

241 Raised if one or more records violate database integrity 

242 constraints. 

243 

244 Notes 

245 ----- 

246 This method cannot be called within transactions, as it needs to be 

247 able to perform its own transaction to be concurrent. 

248 """ 

249 raise NotImplementedError() 

250 

251 @abstractmethod 

252 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

253 """Retrieve records from storage. 

254 

255 Parameters 

256 ---------- 

257 dataIds : `DataCoordinateIterable` 

258 Data IDs that identify the records to be retrieved. 

259 

260 Returns 

261 ------- 

262 records : `Iterable` [ `DimensionRecord` ] 

263 Record retrieved from storage. Not all data IDs may have 

264 corresponding records (if there are no records that match a data 

265 ID), and even if they are, the order of inputs is not preserved. 

266 """ 

267 raise NotImplementedError() 

268 

269 @abstractmethod 

270 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

271 """Return tables used for schema digest. 

272 

273 Returns 

274 ------- 

275 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

276 Possibly empty set of tables for schema digest calculations. 

277 """ 

278 raise NotImplementedError() 

279 

280 

281class DimensionRecordStorageManager(VersionedExtension): 

282 """An interface for managing the dimension records in a `Registry`. 

283 

284 `DimensionRecordStorageManager` primarily serves as a container and factory 

285 for `DimensionRecordStorage` instances, which each provide access to the 

286 records for a different `DimensionElement`. 

287 

288 Parameters 

289 ---------- 

290 universe : `DimensionUniverse` 

291 Universe of all dimensions and dimension elements known to the 

292 `Registry`. 

293 

294 Notes 

295 ----- 

296 In a multi-layer `Registry`, many dimension elements will only have 

297 records in one layer (often the base layer). The union of the records 

298 across all layers forms the logical table for the full `Registry`. 

299 """ 

300 def __init__(self, *, universe: DimensionUniverse): 

301 self.universe = universe 

302 

303 @classmethod 

304 @abstractmethod 

305 def initialize(cls, db: Database, context: StaticTablesContext, *, 

306 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

307 """Construct an instance of the manager. 

308 

309 Parameters 

310 ---------- 

311 db : `Database` 

312 Interface to the underlying database engine and namespace. 

313 context : `StaticTablesContext` 

314 Context object obtained from `Database.declareStaticTables`; used 

315 to declare any tables that should always be present in a layer 

316 implemented with this manager. 

317 universe : `DimensionUniverse` 

318 Universe graph containing dimensions known to this `Registry`. 

319 

320 Returns 

321 ------- 

322 manager : `DimensionRecordStorageManager` 

323 An instance of a concrete `DimensionRecordStorageManager` subclass. 

324 """ 

325 raise NotImplementedError() 

326 

327 @abstractmethod 

328 def refresh(self) -> None: 

329 """Ensure all other operations on this manager are aware of any 

330 dataset types that may have been registered by other clients since 

331 it was initialized or last refreshed. 

332 """ 

333 raise NotImplementedError() 

334 

335 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

336 """Interface to `get` that raises `LookupError` instead of returning 

337 `None` on failure. 

338 """ 

339 r = self.get(element) 

340 if r is None: 

341 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

342 return r 

343 

344 @abstractmethod 

345 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

346 """Return an object that provides access to the records associated with 

347 the given element, if one exists in this layer. 

348 

349 Parameters 

350 ---------- 

351 element : `DimensionElement` 

352 Element for which records should be returned. 

353 

354 Returns 

355 ------- 

356 records : `DimensionRecordStorage` or `None` 

357 The object representing the records for the given element in this 

358 layer, or `None` if there are no records for that element in this 

359 layer. 

360 

361 Notes 

362 ----- 

363 Dimension elements registered by another client of the same layer since 

364 the last call to `initialize` or `refresh` may not be found. 

365 """ 

366 raise NotImplementedError() 

367 

368 @abstractmethod 

369 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

370 """Ensure that this layer can hold records for the given element, 

371 creating new tables as necessary. 

372 

373 Parameters 

374 ---------- 

375 element : `DimensionElement` 

376 Element for which a table should created (as necessary) and 

377 an associated `DimensionRecordStorage` returned. 

378 

379 Returns 

380 ------- 

381 records : `DimensionRecordStorage` 

382 The object representing the records for the given element in this 

383 layer. 

384 

385 Raises 

386 ------ 

387 TransactionInterruption 

388 Raised if this operation is invoked within a `Database.transaction` 

389 context. 

390 """ 

391 raise NotImplementedError() 

392 

393 @abstractmethod 

394 def clearCaches(self) -> None: 

395 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

396 instances. 

397 

398 This is called by `Registry` when transactions are rolled back, to 

399 avoid in-memory caches from ever containing records that are not 

400 present in persistent storage. 

401 """ 

402 raise NotImplementedError() 

403 

404 universe: DimensionUniverse 

405 """Universe of all dimensions and dimension elements known to the 

406 `Registry` (`DimensionUniverse`). 

407 """