Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["DimensionRecordStorage", "DimensionRecordStorageManager"] 

24 

25from abc import ABC, abstractmethod 

26from typing import Optional, Type, TYPE_CHECKING 

27 

28import sqlalchemy 

29 

30from ...core import SkyPixDimension 

31 

32if TYPE_CHECKING: 32 ↛ 33line 32 didn't jump to line 33, because the condition on line 32 was never true

33 from ...core import ( 

34 DataId, 

35 DimensionElement, 

36 DimensionRecord, 

37 DimensionUniverse, 

38 Timespan, 

39 ) 

40 from ...core.utils import NamedKeyDict 

41 from ..queries import QueryBuilder 

42 from ._database import Database, StaticTablesContext 

43 

44 

45class DimensionRecordStorage(ABC): 

46 """An abstract base class that represents a way of storing the records 

47 associated with a single `DimensionElement`. 

48 

49 Concrete `DimensionRecordStorage` instances should generally be constructed 

50 via a call to `setupDimensionStorage`, which selects the appropriate 

51 subclass for each element according to its configuration. 

52 

53 All `DimensionRecordStorage` methods are pure abstract, even though in some 

54 cases a reasonable default implementation might be possible, in order to 

55 better guarantee all methods are correctly overridden. All of these 

56 potentially-defaultable implementations are extremely trivial, so asking 

57 subclasses to provide them is not a significant burden. 

58 """ 

59 

60 @classmethod 

61 @abstractmethod 

62 def initialize(cls, db: Database, element: DimensionElement, *, 

63 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage: 

64 """Construct an instance of this class using a standardized interface. 

65 

66 Parameters 

67 ---------- 

68 db : `Database` 

69 Interface to the underlying database engine and namespace. 

70 element : `DimensionElement` 

71 Dimension element the new instance will manage records for. 

72 context : `StaticTablesContext`, optional 

73 If provided, an object to use to create any new tables. If not 

74 provided, ``db.ensureTableExists`` should be used instead. 

75 

76 Returns 

77 ------- 

78 storage : `DimensionRecordStorage` 

79 A new `DimensionRecordStorage` subclass instance. 

80 """ 

81 raise NotImplementedError() 

82 

83 @staticmethod 

84 def getDefaultImplementation(element: DimensionElement, ignoreCached: bool = False 

85 ) -> Type[DimensionRecordStorage]: 

86 """Return the default `DimensionRecordStorage` implementation for the 

87 given `DimensionElement`. 

88 

89 Parameters 

90 ---------- 

91 element : `DimensionElement` 

92 The element whose properties should be examined to determine the 

93 appropriate default implementation class. 

94 ignoreCached : `bool`, optional 

95 If `True`, ignore `DimensionElement.cached` and always return the 

96 storage implementation that would be used without caching. 

97 

98 Returns 

99 ------- 

100 cls : `type` 

101 A concrete subclass of `DimensionRecordStorage`. 

102 

103 Notes 

104 ----- 

105 At present, these defaults are always used, but we may add support for 

106 explicitly setting the class to use in configuration in the future. 

107 """ 

108 if not ignoreCached and element.cached: 

109 from ..dimensions.caching import CachingDimensionRecordStorage 

110 return CachingDimensionRecordStorage 

111 elif element.hasTable(): 

112 if element.viewOf is not None: 

113 if element.spatial: 

114 raise NotImplementedError("Spatial view dimension storage is not supported.") 

115 from ..dimensions.query import QueryDimensionRecordStorage 

116 return QueryDimensionRecordStorage 

117 elif element.spatial: 

118 from ..dimensions.spatial import SpatialDimensionRecordStorage 

119 return SpatialDimensionRecordStorage 

120 else: 

121 from ..dimensions.table import TableDimensionRecordStorage 

122 return TableDimensionRecordStorage 

123 elif isinstance(element, SkyPixDimension): 

124 from ..dimensions.skypix import SkyPixDimensionRecordStorage 

125 return SkyPixDimensionRecordStorage 

126 raise NotImplementedError(f"No default DimensionRecordStorage class for {element}.") 

127 

128 @property 

129 @abstractmethod 

130 def element(self) -> DimensionElement: 

131 """The element whose records this instance holds (`DimensionElement`). 

132 """ 

133 raise NotImplementedError() 

134 

135 @abstractmethod 

136 def clearCaches(self): 

137 """Clear any in-memory caches held by the storage instance. 

138 

139 This is called by `Registry` when transactions are rolled back, to 

140 avoid in-memory caches from ever containing records that are not 

141 present in persistent storage. 

142 """ 

143 raise NotImplementedError() 

144 

145 @abstractmethod 

146 def join( 

147 self, 

148 builder: QueryBuilder, *, 

149 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

150 timespans: Optional[NamedKeyDict[DimensionElement, Timespan[sqlalchemy.sql.ColumnElement]]] = None, 

151 ): 

152 """Add the dimension element's logical table to a query under 

153 construction. 

154 

155 This is a visitor pattern interface that is expected to be called only 

156 by `QueryBuilder.joinDimensionElement`. 

157 

158 Parameters 

159 ---------- 

160 builder : `QueryBuilder` 

161 Builder for the query that should contain this element. 

162 regions : `NamedKeyDict`, optional 

163 A mapping from `DimensionElement` to a SQLAlchemy column containing 

164 the region for that element, which should be updated to include a 

165 region column for this element if one exists. If `None`, 

166 ``self.element`` is not being included in the query via a spatial 

167 join. 

168 timespan : `NamedKeyDict`, optional 

169 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

170 columns containing the timespan for that element, which should be 

171 updated to include timespan columns for this element if they exist. 

172 If `None`, ``self.element`` is not being included in the query via 

173 a temporal join. 

174 

175 Notes 

176 ----- 

177 Elements are only included in queries via spatial and/or temporal joins 

178 when necessary to connect them to other elements in the query, so 

179 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

180 because an element has a region or timespan. 

181 """ 

182 raise NotImplementedError() 

183 

184 @abstractmethod 

185 def insert(self, *records: DimensionRecord): 

186 """Insert one or more records into storage. 

187 

188 Parameters 

189 ---------- 

190 records 

191 One or more instances of the `DimensionRecord` subclass for the 

192 element this storage is associated with. 

193 

194 Raises 

195 ------ 

196 TypeError 

197 Raised if the element does not support record insertion. 

198 sqlalchemy.exc.IntegrityError 

199 Raised if one or more records violate database integrity 

200 constraints. 

201 

202 Notes 

203 ----- 

204 As `insert` is expected to be called only by a `Registry`, we rely 

205 on `Registry` to provide transactionality, both by using a SQLALchemy 

206 connection shared with the `Registry` and by relying on it to call 

207 `clearCaches` when rolling back transactions. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def fetch(self, dataId: DataId) -> Optional[DimensionRecord]: 

213 """Retrieve a record from storage. 

214 

215 Parameters 

216 ---------- 

217 dataId : `DataId` 

218 A data ID that identifies the record to be retrieved. This may 

219 be an informal data ID dict or a validated `DataCoordinate`. 

220 

221 Returns 

222 ------- 

223 record : `DimensionRecord` or `None` 

224 A record retrieved from storage, or `None` if there is no such 

225 record. 

226 """ 

227 raise NotImplementedError() 

228 

229 

230class DimensionRecordStorageManager(ABC): 

231 """An interface for managing the dimension records in a `Registry`. 

232 

233 `DimensionRecordStorageManager` primarily serves as a container and factory 

234 for `DimensionRecordStorage` instances, which each provide access to the 

235 records for a different `DimensionElement`. 

236 

237 Parameters 

238 ---------- 

239 universe : `DimensionUniverse` 

240 Universe of all dimensions and dimension elements known to the 

241 `Registry`. 

242 

243 Notes 

244 ----- 

245 In a multi-layer `Registry`, many dimension elements will only have 

246 records in one layer (often the base layer). The union of the records 

247 across all layers forms the logical table for the full `Registry`. 

248 """ 

249 def __init__(self, *, universe: DimensionUniverse): 

250 self.universe = universe 

251 

252 @classmethod 

253 @abstractmethod 

254 def initialize(cls, db: Database, context: StaticTablesContext, *, 

255 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

256 """Construct an instance of the manager. 

257 

258 Parameters 

259 ---------- 

260 db : `Database` 

261 Interface to the underlying database engine and namespace. 

262 context : `StaticTablesContext` 

263 Context object obtained from `Database.declareStaticTables`; used 

264 to declare any tables that should always be present in a layer 

265 implemented with this manager. 

266 universe : `DimensionUniverse` 

267 Universe graph containing dimensions known to this `Registry`. 

268 

269 Returns 

270 ------- 

271 manager : `DimensionRecordStorageManager` 

272 An instance of a concrete `DimensionRecordStorageManager` subclass. 

273 """ 

274 raise NotImplementedError() 

275 

276 @abstractmethod 

277 def refresh(self): 

278 """Ensure all other operations on this manager are aware of any 

279 dataset types that may have been registered by other clients since 

280 it was initialized or last refreshed. 

281 """ 

282 raise NotImplementedError() 

283 

284 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

285 """Interface to `get` that raises `LookupError` instead of returning 

286 `None` on failure. 

287 """ 

288 r = self.get(element) 

289 if r is None: 

290 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

291 return r 

292 

293 @abstractmethod 

294 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

295 """Return an object that provides access to the records associated with 

296 the given element, if one exists in this layer. 

297 

298 Parameters 

299 ---------- 

300 element : `DimensionElement` 

301 Element for which records should be returned. 

302 

303 Returns 

304 ------- 

305 records : `DimensionRecordStorage` or `None` 

306 The object representing the records for the given element in this 

307 layer, or `None` if there are no records for that element in this 

308 layer. 

309 

310 Note 

311 ---- 

312 Dimension elements registered by another client of the same layer since 

313 the last call to `initialize` or `refresh` may not be found. 

314 """ 

315 raise NotImplementedError() 

316 

317 @abstractmethod 

318 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

319 """Ensure that this layer can hold records for the given element, 

320 creating new tables as necessary. 

321 

322 Parameters 

323 ---------- 

324 element : `DimensionElement` 

325 Element for which a table should created (as necessary) and 

326 an associated `DimensionRecordStorage` returned. 

327 

328 Returns 

329 ------- 

330 records : `DimensionRecordStorage` 

331 The object representing the records for the given element in this 

332 layer. 

333 

334 Raises 

335 ------ 

336 TransactionInterruption 

337 Raised if this operation is invoked within a `Database.transaction` 

338 context. 

339 """ 

340 raise NotImplementedError() 

341 

342 @abstractmethod 

343 def clearCaches(self): 

344 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

345 instances. 

346 

347 This is called by `Registry` when transactions are rolled back, to 

348 avoid in-memory caches from ever containing records that are not 

349 present in persistent storage. 

350 """ 

351 raise NotImplementedError() 

352 

353 universe: DimensionUniverse 

354 """Universe of all dimensions and dimension elements known to the 

355 `Registry` (`DimensionUniverse`). 

356 """