Coverage for python/lsst/daf/butler/registry/versions.py: 22%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingVersionError", 

28 "MissingManagerError", 

29 "ManagerMismatchError", 

30 "DigestMismatchError", 

31] 

32 

33import logging 

34from typing import ( 

35 Any, 

36 Mapping, 

37 MutableMapping, 

38 Optional, 

39 TYPE_CHECKING, 

40) 

41 

42from .interfaces import VersionTuple, VersionedExtension 

43 

44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from .interfaces import ( 

46 ButlerAttributeManager, 

47 ) 

48 

49 

50_LOG = logging.getLogger(__name__) 

51 

52 

53class MissingVersionError(RuntimeError): 

54 """Exception raised when existing database is missing attributes with 

55 version numbers. 

56 """ 

57 pass 

58 

59 

60class IncompatibleVersionError(RuntimeError): 

61 """Exception raised when configured version number is not compatible with 

62 database version. 

63 """ 

64 pass 

65 

66 

67class MissingManagerError(RuntimeError): 

68 """Exception raised when manager name is missing from registry. 

69 """ 

70 pass 

71 

72 

73class ManagerMismatchError(RuntimeError): 

74 """Exception raised when configured manager name does not match name 

75 stored in the database. 

76 """ 

77 pass 

78 

79 

80class DigestMismatchError(RuntimeError): 

81 """Exception raised when schema digest is not equal to stored digest. 

82 """ 

83 pass 

84 

85 

86class VersionInfo: 

87 """Representation of version information as defined by configuration. 

88 

89 Parameters 

90 ---------- 

91 version : `VersionTuple` 

92 Version number in parsed format. 

93 digest : `str`, optional 

94 Optional digest of the corresponding part of the schema definition. 

95 

96 Notes 

97 ----- 

98 Schema digest is supposed to help with detecting unintentional schema 

99 changes in the code without upgrading schema version. Digest is 

100 constructed whom the set of table definitions and is compared to a digest 

101 defined in configuration, if two digests differ it means schema was 

102 changed. Intentional schema updates will need to update both configured 

103 schema version and schema digest. 

104 """ 

105 def __init__(self, version: VersionTuple, digest: Optional[str] = None): 

106 self.version = version 

107 self.digest = digest 

108 

109 

110class ButlerVersionsManager: 

111 """Utility class to manage and verify schema version compatibility. 

112 

113 Parameters 

114 ---------- 

115 attributes : `ButlerAttributeManager` 

116 Attribute manager instance. 

117 managers : `dict` [`str`, `VersionedExtension`] 

118 Mapping of extension type as defined in configuration (e.g. 

119 "collections") to corresponding instance of manager. 

120 """ 

121 def __init__(self, attributes: ButlerAttributeManager, 

122 managers: Mapping[str, Any]): 

123 self._attributes = attributes 

124 self._managers: MutableMapping[str, VersionedExtension] = {} 

125 # we only care about managers implementing VersionedExtension interface 

126 for name, manager in managers.items(): 

127 if isinstance(manager, VersionedExtension): 

128 self._managers[name] = manager 

129 else: 

130 # All regular managers need to support versioning mechanism. 

131 _LOG.warning("extension %r does not implement VersionedExtension", name) 

132 self._emptyFlag: Optional[bool] = None 

133 

134 @classmethod 

135 def _managerConfigKey(cls, name: str) -> str: 

136 """Return key used to store manager config. 

137 

138 Parameters 

139 ---------- 

140 name : `str` 

141 Name of the namager type, e.g. "dimensions" 

142 

143 Returns 

144 ------- 

145 key : `str` 

146 Name of the key in attributes table. 

147 """ 

148 return f"config:registry.managers.{name}" 

149 

150 @classmethod 

151 def _managerVersionKey(cls, extension: VersionedExtension) -> str: 

152 """Return key used to store manager version. 

153 

154 Parameters 

155 ---------- 

156 extension : `VersionedExtension` 

157 Instance of the extension. 

158 

159 Returns 

160 ------- 

161 key : `str` 

162 Name of the key in attributes table. 

163 """ 

164 return "version:" + extension.extensionName() 

165 

166 @classmethod 

167 def _managerDigestKey(cls, extension: VersionedExtension) -> str: 

168 """Return key used to store manager schema digest. 

169 

170 Parameters 

171 ---------- 

172 extension : `VersionedExtension` 

173 Instance of the extension. 

174 

175 Returns 

176 ------- 

177 key : `str` 

178 Name of the key in attributes table. 

179 """ 

180 return "schema_digest:" + extension.extensionName() 

181 

182 @staticmethod 

183 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

184 """Compare two versions for compatibility. 

185 

186 Parameters 

187 ---------- 

188 old_version : `VersionTuple` 

189 Old schema version, typically one stored in a database. 

190 new_version : `VersionTuple` 

191 New schema version, typically version defined in configuration. 

192 update : `bool` 

193 If True then read-write access is expected. 

194 """ 

195 if old_version.major != new_version.major: 

196 # different major versions are not compatible at all 

197 return False 

198 if old_version.minor != new_version.minor: 

199 # different minor versions are backward compatible for read 

200 # access only 

201 return new_version.minor > old_version.minor and not update 

202 # patch difference does not matter 

203 return True 

204 

205 def storeManagersConfig(self) -> None: 

206 """Store configured extension names in attributes table. 

207 

208 For each extension we store a record with the key 

209 "config:registry.managers.{name}" and fully qualified class name as a 

210 value. 

211 """ 

212 for name, extension in self._managers.items(): 

213 key = self._managerConfigKey(name) 

214 value = extension.extensionName() 

215 self._attributes.set(key, value) 

216 _LOG.debug("saved manager config %s=%s", key, value) 

217 self._emptyFlag = False 

218 

219 def storeManagersVersions(self) -> None: 

220 """Store current manager versions in registry arttributes. 

221 

222 For each extension we store two records: 

223 

224 - record with the key "version:{fullExtensionName}" and version 

225 number in its string format as a value, 

226 - record with the key "schema_digest:{fullExtensionName}" and 

227 schema digest as a value. 

228 """ 

229 for extension in self._managers.values(): 

230 

231 version = extension.currentVersion() 

232 if version: 

233 key = self._managerVersionKey(extension) 

234 value = str(version) 

235 self._attributes.set(key, value) 

236 _LOG.debug("saved manager version %s=%s", key, value) 

237 

238 digest = extension.schemaDigest() 

239 if digest is not None: 

240 key = self._managerDigestKey(extension) 

241 self._attributes.set(key, digest) 

242 _LOG.debug("saved manager schema digest %s=%s", key, digest) 

243 

244 self._emptyFlag = False 

245 

246 @property 

247 def _attributesEmpty(self) -> bool: 

248 """True if attributes table is empty. 

249 """ 

250 # There are existing repositories where attributes table was not 

251 # filled, we don't want to force schema migration in this case yet 

252 # (and we don't have tools) so we allow this as valid use case and 

253 # skip all checks but print a warning. 

254 if self._emptyFlag is None: 

255 self._emptyFlag = self._attributes.empty() 

256 if self._emptyFlag: 

257 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

258 return self._emptyFlag 

259 

260 def checkManagersConfig(self) -> None: 

261 """Compare configured manager names with stored in database. 

262 

263 Raises 

264 ------ 

265 ManagerMismatchError 

266 Raised if manager names are different. 

267 MissingManagerError 

268 Raised if database has no stored manager name. 

269 """ 

270 if self._attributesEmpty: 

271 return 

272 

273 missing = [] 

274 mismatch = [] 

275 for name, extension in self._managers.items(): 

276 key = self._managerConfigKey(name) 

277 storedMgr = self._attributes.get(key) 

278 _LOG.debug("found manager config %s=%s", key, storedMgr) 

279 if storedMgr is None: 

280 missing.append(name) 

281 continue 

282 if extension.extensionName() != storedMgr: 

283 mismatch.append( 

284 f"{name}: configured {extension.extensionName()}, stored: {storedMgr}" 

285 ) 

286 if missing: 

287 raise MissingManagerError( 

288 "Cannot find stored configuration for managers: " 

289 + ", ".join(missing) 

290 ) 

291 if mismatch: 

292 raise ManagerMismatchError( 

293 "Configured managers do not match registry-stored names:\n" 

294 + "\n".join(missing) 

295 ) 

296 

297 def checkManagersVersions(self, writeable: bool) -> None: 

298 """Compare configured versions with the versions stored in database. 

299 

300 Parameters 

301 ---------- 

302 writeable : `bool` 

303 If ``True`` then read-write access needs to be checked. 

304 

305 Raises 

306 ------ 

307 IncompatibleVersionError 

308 Raised if versions are not compatible. 

309 MissingVersionError 

310 Raised if database has no stored version for one or more groups. 

311 """ 

312 if self._attributesEmpty: 

313 return 

314 

315 for extension in self._managers.values(): 

316 version = extension.currentVersion() 

317 if version: 

318 key = self._managerVersionKey(extension) 

319 storedVersionStr = self._attributes.get(key) 

320 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version) 

321 if storedVersionStr is None: 

322 raise MissingVersionError(f"Failed to read version number {key}") 

323 storedVersion = VersionTuple.fromString(storedVersionStr) 

324 if not self.checkCompatibility(storedVersion, version, writeable): 

325 raise IncompatibleVersionError( 

326 f"Configured version {version} is not compatible with stored version " 

327 f"{storedVersion} for extension {extension.extensionName()}" 

328 ) 

329 

330 def checkManagersDigests(self) -> None: 

331 """Compare current schema digests with digests stored in database. 

332 

333 Raises 

334 ------ 

335 DigestMismatchError 

336 Raised if digests are not equal. 

337 """ 

338 if self._attributesEmpty: 

339 return 

340 

341 for extension in self._managers.values(): 

342 digest = extension.schemaDigest() 

343 if digest is not None: 

344 key = self._managerDigestKey(extension) 

345 storedDigest = self._attributes.get(key) 

346 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest) 

347 if storedDigest != digest: 

348 raise DigestMismatchError( 

349 f"Current schema digest '{digest}' is not the same as stored digest " 

350 f"'{storedDigest}' for extension {extension.extensionName()}" 

351 )