Coverage for python/lsst/daf/butler/registry/versions.py: 28%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingVersionError", 

28 "MissingManagerError", 

29 "ManagerMismatchError", 

30 "DigestMismatchError", 

31] 

32 

33import logging 

34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional 

35 

36from .interfaces import VersionedExtension, VersionTuple 

37 

38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 from .interfaces import ButlerAttributeManager 

40 

41 

42_LOG = logging.getLogger(__name__) 

43 

44 

45class MissingVersionError(RuntimeError): 

46 """Exception raised when existing database is missing attributes with 

47 version numbers. 

48 """ 

49 

50 pass 

51 

52 

53class IncompatibleVersionError(RuntimeError): 

54 """Exception raised when configured version number is not compatible with 

55 database version. 

56 """ 

57 

58 pass 

59 

60 

61class MissingManagerError(RuntimeError): 

62 """Exception raised when manager name is missing from registry.""" 

63 

64 pass 

65 

66 

67class ManagerMismatchError(RuntimeError): 

68 """Exception raised when configured manager name does not match name 

69 stored in the database. 

70 """ 

71 

72 pass 

73 

74 

75class DigestMismatchError(RuntimeError): 

76 """Exception raised when schema digest is not equal to stored digest.""" 

77 

78 pass 

79 

80 

81class VersionInfo: 

82 """Representation of version information as defined by configuration. 

83 

84 Parameters 

85 ---------- 

86 version : `VersionTuple` 

87 Version number in parsed format. 

88 digest : `str`, optional 

89 Optional digest of the corresponding part of the schema definition. 

90 

91 Notes 

92 ----- 

93 Schema digest is supposed to help with detecting unintentional schema 

94 changes in the code without upgrading schema version. Digest is 

95 constructed whom the set of table definitions and is compared to a digest 

96 defined in configuration, if two digests differ it means schema was 

97 changed. Intentional schema updates will need to update both configured 

98 schema version and schema digest. 

99 """ 

100 

101 def __init__(self, version: VersionTuple, digest: Optional[str] = None): 

102 self.version = version 

103 self.digest = digest 

104 

105 

106class ButlerVersionsManager: 

107 """Utility class to manage and verify schema version compatibility. 

108 

109 Parameters 

110 ---------- 

111 attributes : `ButlerAttributeManager` 

112 Attribute manager instance. 

113 managers : `dict` [`str`, `VersionedExtension`] 

114 Mapping of extension type as defined in configuration (e.g. 

115 "collections") to corresponding instance of manager. 

116 """ 

117 

118 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]): 

119 self._attributes = attributes 

120 self._managers: MutableMapping[str, VersionedExtension] = {} 

121 # we only care about managers implementing VersionedExtension interface 

122 for name, manager in managers.items(): 

123 if isinstance(manager, VersionedExtension): 

124 self._managers[name] = manager 

125 else: 

126 # All regular managers need to support versioning mechanism. 

127 _LOG.warning("extension %r does not implement VersionedExtension", name) 

128 self._emptyFlag: Optional[bool] = None 

129 

130 @classmethod 

131 def _managerConfigKey(cls, name: str) -> str: 

132 """Return key used to store manager config. 

133 

134 Parameters 

135 ---------- 

136 name : `str` 

137 Name of the namager type, e.g. "dimensions" 

138 

139 Returns 

140 ------- 

141 key : `str` 

142 Name of the key in attributes table. 

143 """ 

144 return f"config:registry.managers.{name}" 

145 

146 @classmethod 

147 def _managerVersionKey(cls, extension: VersionedExtension) -> str: 

148 """Return key used to store manager version. 

149 

150 Parameters 

151 ---------- 

152 extension : `VersionedExtension` 

153 Instance of the extension. 

154 

155 Returns 

156 ------- 

157 key : `str` 

158 Name of the key in attributes table. 

159 """ 

160 return "version:" + extension.extensionName() 

161 

162 @classmethod 

163 def _managerDigestKey(cls, extension: VersionedExtension) -> str: 

164 """Return key used to store manager schema digest. 

165 

166 Parameters 

167 ---------- 

168 extension : `VersionedExtension` 

169 Instance of the extension. 

170 

171 Returns 

172 ------- 

173 key : `str` 

174 Name of the key in attributes table. 

175 """ 

176 return "schema_digest:" + extension.extensionName() 

177 

178 @staticmethod 

179 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

180 """Compare two versions for compatibility. 

181 

182 Parameters 

183 ---------- 

184 old_version : `VersionTuple` 

185 Old schema version, typically one stored in a database. 

186 new_version : `VersionTuple` 

187 New schema version, typically version defined in configuration. 

188 update : `bool` 

189 If True then read-write access is expected. 

190 """ 

191 if old_version.major != new_version.major: 

192 # different major versions are not compatible at all 

193 return False 

194 if old_version.minor != new_version.minor: 

195 # different minor versions are backward compatible for read 

196 # access only 

197 return new_version.minor > old_version.minor and not update 

198 # patch difference does not matter 

199 return True 

200 

201 def storeManagersConfig(self) -> None: 

202 """Store configured extension names in attributes table. 

203 

204 For each extension we store a record with the key 

205 "config:registry.managers.{name}" and fully qualified class name as a 

206 value. 

207 """ 

208 for name, extension in self._managers.items(): 

209 key = self._managerConfigKey(name) 

210 value = extension.extensionName() 

211 self._attributes.set(key, value) 

212 _LOG.debug("saved manager config %s=%s", key, value) 

213 self._emptyFlag = False 

214 

215 def storeManagersVersions(self) -> None: 

216 """Store current manager versions in registry arttributes. 

217 

218 For each extension we store two records: 

219 

220 - record with the key "version:{fullExtensionName}" and version 

221 number in its string format as a value, 

222 - record with the key "schema_digest:{fullExtensionName}" and 

223 schema digest as a value. 

224 """ 

225 for extension in self._managers.values(): 

226 

227 version = extension.currentVersion() 

228 if version: 

229 key = self._managerVersionKey(extension) 

230 value = str(version) 

231 self._attributes.set(key, value) 

232 _LOG.debug("saved manager version %s=%s", key, value) 

233 

234 digest = extension.schemaDigest() 

235 if digest is not None: 

236 key = self._managerDigestKey(extension) 

237 self._attributes.set(key, digest) 

238 _LOG.debug("saved manager schema digest %s=%s", key, digest) 

239 

240 self._emptyFlag = False 

241 

242 @property 

243 def _attributesEmpty(self) -> bool: 

244 """True if attributes table is empty.""" 

245 # There are existing repositories where attributes table was not 

246 # filled, we don't want to force schema migration in this case yet 

247 # (and we don't have tools) so we allow this as valid use case and 

248 # skip all checks but print a warning. 

249 if self._emptyFlag is None: 

250 self._emptyFlag = self._attributes.empty() 

251 if self._emptyFlag: 

252 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

253 return self._emptyFlag 

254 

255 def checkManagersConfig(self) -> None: 

256 """Compare configured manager names with stored in database. 

257 

258 Raises 

259 ------ 

260 ManagerMismatchError 

261 Raised if manager names are different. 

262 MissingManagerError 

263 Raised if database has no stored manager name. 

264 """ 

265 if self._attributesEmpty: 

266 return 

267 

268 missing = [] 

269 mismatch = [] 

270 for name, extension in self._managers.items(): 

271 key = self._managerConfigKey(name) 

272 storedMgr = self._attributes.get(key) 

273 _LOG.debug("found manager config %s=%s", key, storedMgr) 

274 if storedMgr is None: 

275 missing.append(name) 

276 continue 

277 if extension.extensionName() != storedMgr: 

278 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}") 

279 if missing: 

280 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

281 if mismatch: 

282 raise ManagerMismatchError( 

283 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

284 ) 

285 

286 def checkManagersVersions(self, writeable: bool) -> None: 

287 """Compare configured versions with the versions stored in database. 

288 

289 Parameters 

290 ---------- 

291 writeable : `bool` 

292 If ``True`` then read-write access needs to be checked. 

293 

294 Raises 

295 ------ 

296 IncompatibleVersionError 

297 Raised if versions are not compatible. 

298 MissingVersionError 

299 Raised if database has no stored version for one or more groups. 

300 """ 

301 if self._attributesEmpty: 

302 return 

303 

304 for extension in self._managers.values(): 

305 version = extension.currentVersion() 

306 if version: 

307 key = self._managerVersionKey(extension) 

308 storedVersionStr = self._attributes.get(key) 

309 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version) 

310 if storedVersionStr is None: 

311 raise MissingVersionError(f"Failed to read version number {key}") 

312 storedVersion = VersionTuple.fromString(storedVersionStr) 

313 if not self.checkCompatibility(storedVersion, version, writeable): 

314 raise IncompatibleVersionError( 

315 f"Configured version {version} is not compatible with stored version " 

316 f"{storedVersion} for extension {extension.extensionName()}" 

317 ) 

318 

319 def checkManagersDigests(self) -> None: 

320 """Compare current schema digests with digests stored in database. 

321 

322 Raises 

323 ------ 

324 DigestMismatchError 

325 Raised if digests are not equal. 

326 """ 

327 if self._attributesEmpty: 

328 return 

329 

330 for extension in self._managers.values(): 

331 digest = extension.schemaDigest() 

332 if digest is not None: 

333 key = self._managerDigestKey(extension) 

334 storedDigest = self._attributes.get(key) 

335 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest) 

336 if storedDigest != digest: 

337 raise DigestMismatchError( 

338 f"Current schema digest '{digest}' is not the same as stored digest " 

339 f"'{storedDigest}' for extension {extension.extensionName()}" 

340 )