Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["DimensionRecordStorage", "DimensionRecordStorageManager"] 

24 

25from abc import ABC, abstractmethod 

26from typing import Optional, Type, TYPE_CHECKING 

27 

28import sqlalchemy 

29 

30from ...core import SkyPixDimension 

31 

32if TYPE_CHECKING: 32 ↛ 33line 32 didn't jump to line 33, because the condition on line 32 was never true

33 from ...core import ( 

34 DataId, 

35 DimensionElement, 

36 DimensionRecord, 

37 DimensionUniverse, 

38 Timespan, 

39 ) 

40 from ...core.utils import NamedKeyDict 

41 from ..queries import QueryBuilder 

42 from ._database import Database, StaticTablesContext 

43 

44 

45class DimensionRecordStorage(ABC): 

46 """An abstract base class that represents a way of storing the records 

47 associated with a single `DimensionElement`. 

48 

49 Concrete `DimensionRecordStorage` instances should generally be constructed 

50 via a call to `setupDimensionStorage`, which selects the appropriate 

51 subclass for each element according to its configuration. 

52 

53 All `DimensionRecordStorage` methods are pure abstract, even though in some 

54 cases a reasonable default implementation might be possible, in order to 

55 better guarantee all methods are correctly overridden. All of these 

56 potentially-defaultable implementations are extremely trivial, so asking 

57 subclasses to provide them is not a significant burden. 

58 """ 

59 

60 @classmethod 

61 @abstractmethod 

62 def initialize(cls, db: Database, element: DimensionElement, *, 

63 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage: 

64 """Construct an instance of this class using a standardized interface. 

65 

66 Parameters 

67 ---------- 

68 db : `Database` 

69 Interface to the underlying database engine and namespace. 

70 element : `DimensionElement` 

71 Dimension element the new instance will manage records for. 

72 context : `StaticTablesContext`, optional 

73 If provided, an object to use to create any new tables. If not 

74 provided, ``db.ensureTableExists`` should be used instead. 

75 

76 Returns 

77 ------- 

78 storage : `DimensionRecordStorage` 

79 A new `DimensionRecordStorage` subclass instance. 

80 """ 

81 raise NotImplementedError() 

82 

83 @staticmethod 

84 def getDefaultImplementation(element: DimensionElement, ignoreCached: bool = False 

85 ) -> Type[DimensionRecordStorage]: 

86 """Return the default `DimensionRecordStorage` implementation for the 

87 given `DimensionElement`. 

88 

89 Parameters 

90 ---------- 

91 element : `DimensionElement` 

92 The element whose properties should be examined to determine the 

93 appropriate default implementation class. 

94 ignoreCached : `bool`, optional 

95 If `True`, ignore `DimensionElement.cached` and always return the 

96 storage implementation that would be used without caching. 

97 

98 Returns 

99 ------- 

100 cls : `type` 

101 A concrete subclass of `DimensionRecordStorage`. 

102 

103 Notes 

104 ----- 

105 At present, these defaults are always used, but we may add support for 

106 explicitly setting the class to use in configuration in the future. 

107 """ 

108 if not ignoreCached and element.cached: 

109 from ..dimensions.caching import CachingDimensionRecordStorage 

110 return CachingDimensionRecordStorage 

111 elif element.hasTable(): 

112 if element.viewOf is not None: 

113 if element.spatial is not None: 

114 raise NotImplementedError("Spatial view dimension storage is not supported.") 

115 from ..dimensions.query import QueryDimensionRecordStorage 

116 return QueryDimensionRecordStorage 

117 elif element.spatial is not None: 

118 from ..dimensions.spatial import SpatialDimensionRecordStorage 

119 return SpatialDimensionRecordStorage 

120 else: 

121 from ..dimensions.table import TableDimensionRecordStorage 

122 return TableDimensionRecordStorage 

123 elif isinstance(element, SkyPixDimension): 

124 from ..dimensions.skypix import SkyPixDimensionRecordStorage 

125 return SkyPixDimensionRecordStorage 

126 raise NotImplementedError(f"No default DimensionRecordStorage class for {element}.") 

127 

128 @property 

129 @abstractmethod 

130 def element(self) -> DimensionElement: 

131 """The element whose records this instance holds (`DimensionElement`). 

132 """ 

133 raise NotImplementedError() 

134 

135 @abstractmethod 

136 def clearCaches(self) -> None: 

137 """Clear any in-memory caches held by the storage instance. 

138 

139 This is called by `Registry` when transactions are rolled back, to 

140 avoid in-memory caches from ever containing records that are not 

141 present in persistent storage. 

142 """ 

143 raise NotImplementedError() 

144 

145 @abstractmethod 

146 def join( 

147 self, 

148 builder: QueryBuilder, *, 

149 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

150 timespans: Optional[NamedKeyDict[DimensionElement, Timespan[sqlalchemy.sql.ColumnElement]]] = None, 

151 ) -> sqlalchemy.sql.FromClause: 

152 """Add the dimension element's logical table to a query under 

153 construction. 

154 

155 This is a visitor pattern interface that is expected to be called only 

156 by `QueryBuilder.joinDimensionElement`. 

157 

158 Parameters 

159 ---------- 

160 builder : `QueryBuilder` 

161 Builder for the query that should contain this element. 

162 regions : `NamedKeyDict`, optional 

163 A mapping from `DimensionElement` to a SQLAlchemy column containing 

164 the region for that element, which should be updated to include a 

165 region column for this element if one exists. If `None`, 

166 ``self.element`` is not being included in the query via a spatial 

167 join. 

168 timespan : `NamedKeyDict`, optional 

169 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

170 columns containing the timespan for that element, which should be 

171 updated to include timespan columns for this element if they exist. 

172 If `None`, ``self.element`` is not being included in the query via 

173 a temporal join. 

174 

175 Returns 

176 ------- 

177 fromClause : `sqlalchemy.sql.FromClause` 

178 Table or clause for the element which is joined. 

179 

180 Notes 

181 ----- 

182 Elements are only included in queries via spatial and/or temporal joins 

183 when necessary to connect them to other elements in the query, so 

184 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

185 because an element has a region or timespan. 

186 """ 

187 raise NotImplementedError() 

188 

189 @abstractmethod 

190 def insert(self, *records: DimensionRecord) -> None: 

191 """Insert one or more records into storage. 

192 

193 Parameters 

194 ---------- 

195 records 

196 One or more instances of the `DimensionRecord` subclass for the 

197 element this storage is associated with. 

198 

199 Raises 

200 ------ 

201 TypeError 

202 Raised if the element does not support record insertion. 

203 sqlalchemy.exc.IntegrityError 

204 Raised if one or more records violate database integrity 

205 constraints. 

206 

207 Notes 

208 ----- 

209 As `insert` is expected to be called only by a `Registry`, we rely 

210 on `Registry` to provide transactionality, both by using a SQLALchemy 

211 connection shared with the `Registry` and by relying on it to call 

212 `clearCaches` when rolling back transactions. 

213 """ 

214 raise NotImplementedError() 

215 

216 @abstractmethod 

217 def sync(self, record: DimensionRecord) -> bool: 

218 """Synchronize a record with the database, inserting it only if it does 

219 not exist and comparing values if it does. 

220 

221 Parameters 

222 ---------- 

223 record : `DimensionRecord`. 

224 An instance of the `DimensionRecord` subclass for the 

225 element this storage is associated with. 

226 

227 Returns 

228 ------- 

229 inserted : `bool` 

230 `True` if a new row was inserted, `False` otherwise. 

231 

232 Raises 

233 ------ 

234 DatabaseConflictError 

235 Raised if the record exists in the database (according to primary 

236 key lookup) but is inconsistent with the given one. 

237 TypeError 

238 Raised if the element does not support record synchronization. 

239 sqlalchemy.exc.IntegrityError 

240 Raised if one or more records violate database integrity 

241 constraints. 

242 

243 Notes 

244 ----- 

245 This method cannot be called within transactions, as it needs to be 

246 able to perform its own transaction to be concurrent. 

247 """ 

248 raise NotImplementedError() 

249 

250 @abstractmethod 

251 def fetch(self, dataId: DataId) -> Optional[DimensionRecord]: 

252 """Retrieve a record from storage. 

253 

254 Parameters 

255 ---------- 

256 dataId : `DataId` 

257 A data ID that identifies the record to be retrieved. This may 

258 be an informal data ID dict or a validated `DataCoordinate`. 

259 

260 Returns 

261 ------- 

262 record : `DimensionRecord` or `None` 

263 A record retrieved from storage, or `None` if there is no such 

264 record. 

265 """ 

266 raise NotImplementedError() 

267 

268 

269class DimensionRecordStorageManager(ABC): 

270 """An interface for managing the dimension records in a `Registry`. 

271 

272 `DimensionRecordStorageManager` primarily serves as a container and factory 

273 for `DimensionRecordStorage` instances, which each provide access to the 

274 records for a different `DimensionElement`. 

275 

276 Parameters 

277 ---------- 

278 universe : `DimensionUniverse` 

279 Universe of all dimensions and dimension elements known to the 

280 `Registry`. 

281 

282 Notes 

283 ----- 

284 In a multi-layer `Registry`, many dimension elements will only have 

285 records in one layer (often the base layer). The union of the records 

286 across all layers forms the logical table for the full `Registry`. 

287 """ 

288 def __init__(self, *, universe: DimensionUniverse): 

289 self.universe = universe 

290 

291 @classmethod 

292 @abstractmethod 

293 def initialize(cls, db: Database, context: StaticTablesContext, *, 

294 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

295 """Construct an instance of the manager. 

296 

297 Parameters 

298 ---------- 

299 db : `Database` 

300 Interface to the underlying database engine and namespace. 

301 context : `StaticTablesContext` 

302 Context object obtained from `Database.declareStaticTables`; used 

303 to declare any tables that should always be present in a layer 

304 implemented with this manager. 

305 universe : `DimensionUniverse` 

306 Universe graph containing dimensions known to this `Registry`. 

307 

308 Returns 

309 ------- 

310 manager : `DimensionRecordStorageManager` 

311 An instance of a concrete `DimensionRecordStorageManager` subclass. 

312 """ 

313 raise NotImplementedError() 

314 

315 @abstractmethod 

316 def refresh(self) -> None: 

317 """Ensure all other operations on this manager are aware of any 

318 dataset types that may have been registered by other clients since 

319 it was initialized or last refreshed. 

320 """ 

321 raise NotImplementedError() 

322 

323 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

324 """Interface to `get` that raises `LookupError` instead of returning 

325 `None` on failure. 

326 """ 

327 r = self.get(element) 

328 if r is None: 

329 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

330 return r 

331 

332 @abstractmethod 

333 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

334 """Return an object that provides access to the records associated with 

335 the given element, if one exists in this layer. 

336 

337 Parameters 

338 ---------- 

339 element : `DimensionElement` 

340 Element for which records should be returned. 

341 

342 Returns 

343 ------- 

344 records : `DimensionRecordStorage` or `None` 

345 The object representing the records for the given element in this 

346 layer, or `None` if there are no records for that element in this 

347 layer. 

348 

349 Notes 

350 ----- 

351 Dimension elements registered by another client of the same layer since 

352 the last call to `initialize` or `refresh` may not be found. 

353 """ 

354 raise NotImplementedError() 

355 

356 @abstractmethod 

357 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

358 """Ensure that this layer can hold records for the given element, 

359 creating new tables as necessary. 

360 

361 Parameters 

362 ---------- 

363 element : `DimensionElement` 

364 Element for which a table should created (as necessary) and 

365 an associated `DimensionRecordStorage` returned. 

366 

367 Returns 

368 ------- 

369 records : `DimensionRecordStorage` 

370 The object representing the records for the given element in this 

371 layer. 

372 

373 Raises 

374 ------ 

375 TransactionInterruption 

376 Raised if this operation is invoked within a `Database.transaction` 

377 context. 

378 """ 

379 raise NotImplementedError() 

380 

381 @abstractmethod 

382 def clearCaches(self) -> None: 

383 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

384 instances. 

385 

386 This is called by `Registry` when transactions are rolled back, to 

387 avoid in-memory caches from ever containing records that are not 

388 present in persistent storage. 

389 """ 

390 raise NotImplementedError() 

391 

392 universe: DimensionUniverse 

393 """Universe of all dimensions and dimension elements known to the 

394 `Registry` (`DimensionUniverse`). 

395 """