Coverage for python/lsst/daf/butler/registry/versions.py: 22%

121 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-05 10:36 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingVersionError", 

28 "MissingManagerError", 

29 "ManagerMismatchError", 

30 "DigestMismatchError", 

31] 

32 

33import logging 

34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional 

35 

36from deprecated.sphinx import deprecated 

37 

38from .interfaces import VersionedExtension, VersionTuple 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from .interfaces import ButlerAttributeManager 

42 

43 

44_LOG = logging.getLogger(__name__) 

45 

46 

47class MissingVersionError(RuntimeError): 

48 """Exception raised when existing database is missing attributes with 

49 version numbers. 

50 """ 

51 

52 pass 

53 

54 

55class IncompatibleVersionError(RuntimeError): 

56 """Exception raised when configured version number is not compatible with 

57 database version. 

58 """ 

59 

60 pass 

61 

62 

63class MissingManagerError(RuntimeError): 

64 """Exception raised when manager name is missing from registry.""" 

65 

66 pass 

67 

68 

69class ManagerMismatchError(RuntimeError): 

70 """Exception raised when configured manager name does not match name 

71 stored in the database. 

72 """ 

73 

74 pass 

75 

76 

77class DigestMismatchError(RuntimeError): 

78 """Exception raised when schema digest is not equal to stored digest.""" 

79 

80 pass 

81 

82 

83class VersionInfo: 

84 """Representation of version information as defined by configuration. 

85 

86 Parameters 

87 ---------- 

88 version : `VersionTuple` 

89 Version number in parsed format. 

90 digest : `str`, optional 

91 Optional digest of the corresponding part of the schema definition. 

92 

93 Notes 

94 ----- 

95 Schema digest is supposed to help with detecting unintentional schema 

96 changes in the code without upgrading schema version. Digest is 

97 constructed whom the set of table definitions and is compared to a digest 

98 defined in configuration, if two digests differ it means schema was 

99 changed. Intentional schema updates will need to update both configured 

100 schema version and schema digest. 

101 """ 

102 

103 def __init__(self, version: VersionTuple, digest: Optional[str] = None): 

104 self.version = version 

105 self.digest = digest 

106 

107 

108class ButlerVersionsManager: 

109 """Utility class to manage and verify schema version compatibility. 

110 

111 Parameters 

112 ---------- 

113 attributes : `ButlerAttributeManager` 

114 Attribute manager instance. 

115 managers : `dict` [`str`, `VersionedExtension`] 

116 Mapping of extension type as defined in configuration (e.g. 

117 "collections") to corresponding instance of manager. 

118 """ 

119 

120 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]): 

121 self._attributes = attributes 

122 self._managers: MutableMapping[str, VersionedExtension] = {} 

123 # we only care about managers implementing VersionedExtension interface 

124 for name, manager in managers.items(): 

125 if isinstance(manager, VersionedExtension): 

126 self._managers[name] = manager 

127 elif manager is not None: 

128 # All regular managers need to support versioning mechanism. 

129 _LOG.warning("extension %r does not implement VersionedExtension", name) 

130 self._emptyFlag: Optional[bool] = None 

131 

132 @classmethod 

133 def _managerConfigKey(cls, name: str) -> str: 

134 """Return key used to store manager config. 

135 

136 Parameters 

137 ---------- 

138 name : `str` 

139 Name of the namager type, e.g. "dimensions" 

140 

141 Returns 

142 ------- 

143 key : `str` 

144 Name of the key in attributes table. 

145 """ 

146 return f"config:registry.managers.{name}" 

147 

148 @classmethod 

149 def _managerVersionKey(cls, extension: VersionedExtension) -> str: 

150 """Return key used to store manager version. 

151 

152 Parameters 

153 ---------- 

154 extension : `VersionedExtension` 

155 Instance of the extension. 

156 

157 Returns 

158 ------- 

159 key : `str` 

160 Name of the key in attributes table. 

161 """ 

162 return "version:" + extension.extensionName() 

163 

164 @classmethod 

165 def _managerDigestKey(cls, extension: VersionedExtension) -> str: 

166 """Return key used to store manager schema digest. 

167 

168 Parameters 

169 ---------- 

170 extension : `VersionedExtension` 

171 Instance of the extension. 

172 

173 Returns 

174 ------- 

175 key : `str` 

176 Name of the key in attributes table. 

177 """ 

178 return "schema_digest:" + extension.extensionName() 

179 

180 @staticmethod 

181 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

182 """Compare two versions for compatibility. 

183 

184 Parameters 

185 ---------- 

186 old_version : `VersionTuple` 

187 Old schema version, typically one stored in a database. 

188 new_version : `VersionTuple` 

189 New schema version, typically version defined in configuration. 

190 update : `bool` 

191 If True then read-write access is expected. 

192 """ 

193 if old_version.major != new_version.major: 

194 # different major versions are not compatible at all 

195 return False 

196 if old_version.minor != new_version.minor: 

197 # different minor versions are backward compatible for read 

198 # access only 

199 return new_version.minor > old_version.minor and not update 

200 # patch difference does not matter 

201 return True 

202 

203 def storeManagersConfig(self) -> None: 

204 """Store configured extension names in attributes table. 

205 

206 For each extension we store a record with the key 

207 "config:registry.managers.{name}" and fully qualified class name as a 

208 value. 

209 """ 

210 for name, extension in self._managers.items(): 

211 key = self._managerConfigKey(name) 

212 value = extension.extensionName() 

213 self._attributes.set(key, value) 

214 _LOG.debug("saved manager config %s=%s", key, value) 

215 self._emptyFlag = False 

216 

217 def storeManagersVersions(self) -> None: 

218 """Store current manager versions in registry arttributes. 

219 

220 For each extension we store two records: 

221 

222 - record with the key "version:{fullExtensionName}" and version 

223 number in its string format as a value, 

224 - record with the key "schema_digest:{fullExtensionName}" and 

225 schema digest as a value. 

226 """ 

227 for extension in self._managers.values(): 

228 

229 version = extension.currentVersion() 

230 if version: 

231 key = self._managerVersionKey(extension) 

232 value = str(version) 

233 self._attributes.set(key, value) 

234 _LOG.debug("saved manager version %s=%s", key, value) 

235 

236 digest = extension.schemaDigest() 

237 if digest is not None: 

238 key = self._managerDigestKey(extension) 

239 self._attributes.set(key, digest) 

240 _LOG.debug("saved manager schema digest %s=%s", key, digest) 

241 

242 self._emptyFlag = False 

243 

244 @property 

245 def _attributesEmpty(self) -> bool: 

246 """True if attributes table is empty.""" 

247 # There are existing repositories where attributes table was not 

248 # filled, we don't want to force schema migration in this case yet 

249 # (and we don't have tools) so we allow this as valid use case and 

250 # skip all checks but print a warning. 

251 if self._emptyFlag is None: 

252 self._emptyFlag = self._attributes.empty() 

253 if self._emptyFlag: 

254 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

255 return self._emptyFlag 

256 

257 def checkManagersConfig(self) -> None: 

258 """Compare configured manager names with stored in database. 

259 

260 Raises 

261 ------ 

262 ManagerMismatchError 

263 Raised if manager names are different. 

264 MissingManagerError 

265 Raised if database has no stored manager name. 

266 """ 

267 if self._attributesEmpty: 

268 return 

269 

270 missing = [] 

271 mismatch = [] 

272 for name, extension in self._managers.items(): 

273 key = self._managerConfigKey(name) 

274 storedMgr = self._attributes.get(key) 

275 _LOG.debug("found manager config %s=%s", key, storedMgr) 

276 if storedMgr is None: 

277 missing.append(name) 

278 continue 

279 if extension.extensionName() != storedMgr: 

280 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}") 

281 if missing: 

282 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

283 if mismatch: 

284 raise ManagerMismatchError( 

285 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

286 ) 

287 

288 def checkManagersVersions(self, writeable: bool) -> None: 

289 """Compare configured versions with the versions stored in database. 

290 

291 Parameters 

292 ---------- 

293 writeable : `bool` 

294 If ``True`` then read-write access needs to be checked. 

295 

296 Raises 

297 ------ 

298 IncompatibleVersionError 

299 Raised if versions are not compatible. 

300 MissingVersionError 

301 Raised if database has no stored version for one or more groups. 

302 """ 

303 if self._attributesEmpty: 

304 return 

305 

306 for extension in self._managers.values(): 

307 version = extension.currentVersion() 

308 if version: 

309 key = self._managerVersionKey(extension) 

310 storedVersionStr = self._attributes.get(key) 

311 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version) 

312 if storedVersionStr is None: 

313 raise MissingVersionError(f"Failed to read version number {key}") 

314 storedVersion = VersionTuple.fromString(storedVersionStr) 

315 if not self.checkCompatibility(storedVersion, version, writeable): 

316 raise IncompatibleVersionError( 

317 f"Configured version {version} is not compatible with stored version " 

318 f"{storedVersion} for extension {extension.extensionName()}" 

319 ) 

320 

321 @deprecated(reason="Schema checksums are ignored", category=FutureWarning, version="v24.0") 

322 def checkManagersDigests(self) -> None: 

323 """Compare current schema digests with digests stored in database. 

324 

325 Raises 

326 ------ 

327 DigestMismatchError 

328 Raised if digests are not equal. 

329 

330 Notes 

331 ----- 

332 This method is not used currently and will probably disappear in the 

333 future as we remove schema checksums. 

334 """ 

335 if self._attributesEmpty: 

336 return 

337 

338 for extension in self._managers.values(): 

339 digest = extension.schemaDigest() 

340 if digest is not None: 

341 key = self._managerDigestKey(extension) 

342 storedDigest = self._attributes.get(key) 

343 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest) 

344 if storedDigest != digest: 

345 raise DigestMismatchError( 

346 f"Current schema digest '{digest}' is not the same as stored digest " 

347 f"'{storedDigest}' for extension {extension.extensionName()}" 

348 )