Coverage for python/lsst/daf/butler/registry/versions.py: 23%

108 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-14 02:05 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingVersionError", 

28 "MissingManagerError", 

29 "ManagerMismatchError", 

30 "DigestMismatchError", 

31] 

32 

33import logging 

34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional 

35 

36from .interfaces import VersionedExtension, VersionTuple 

37 

38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 from .interfaces import ButlerAttributeManager 

40 

41 

42_LOG = logging.getLogger(__name__) 

43 

44 

45class MissingVersionError(RuntimeError): 

46 """Exception raised when existing database is missing attributes with 

47 version numbers. 

48 """ 

49 

50 pass 

51 

52 

53class IncompatibleVersionError(RuntimeError): 

54 """Exception raised when configured version number is not compatible with 

55 database version. 

56 """ 

57 

58 pass 

59 

60 

61class MissingManagerError(RuntimeError): 

62 """Exception raised when manager name is missing from registry.""" 

63 

64 pass 

65 

66 

67class ManagerMismatchError(RuntimeError): 

68 """Exception raised when configured manager name does not match name 

69 stored in the database. 

70 """ 

71 

72 pass 

73 

74 

75class DigestMismatchError(RuntimeError): 

76 """Exception raised when schema digest is not equal to stored digest.""" 

77 

78 pass 

79 

80 

81class VersionInfo: 

82 """Representation of version information as defined by configuration. 

83 

84 Parameters 

85 ---------- 

86 version : `VersionTuple` 

87 Version number in parsed format. 

88 digest : `str`, optional 

89 Optional digest of the corresponding part of the schema definition. 

90 

91 Notes 

92 ----- 

93 Schema digest is supposed to help with detecting unintentional schema 

94 changes in the code without upgrading schema version. Digest is 

95 constructed whom the set of table definitions and is compared to a digest 

96 defined in configuration, if two digests differ it means schema was 

97 changed. Intentional schema updates will need to update both configured 

98 schema version and schema digest. 

99 """ 

100 

101 def __init__(self, version: VersionTuple, digest: Optional[str] = None): 

102 self.version = version 

103 self.digest = digest 

104 

105 

106class ButlerVersionsManager: 

107 """Utility class to manage and verify schema version compatibility. 

108 

109 Parameters 

110 ---------- 

111 attributes : `ButlerAttributeManager` 

112 Attribute manager instance. 

113 managers : `dict` [`str`, `VersionedExtension`] 

114 Mapping of extension type as defined in configuration (e.g. 

115 "collections") to corresponding instance of manager. 

116 """ 

117 

118 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]): 

119 self._attributes = attributes 

120 self._managers: MutableMapping[str, VersionedExtension] = {} 

121 # we only care about managers implementing VersionedExtension interface 

122 for name, manager in managers.items(): 

123 if isinstance(manager, VersionedExtension): 

124 self._managers[name] = manager 

125 elif manager is not None: 

126 # All regular managers need to support versioning mechanism. 

127 _LOG.warning("extension %r does not implement VersionedExtension", name) 

128 self._emptyFlag: Optional[bool] = None 

129 

130 @classmethod 

131 def _managerConfigKey(cls, name: str) -> str: 

132 """Return key used to store manager config. 

133 

134 Parameters 

135 ---------- 

136 name : `str` 

137 Name of the namager type, e.g. "dimensions" 

138 

139 Returns 

140 ------- 

141 key : `str` 

142 Name of the key in attributes table. 

143 """ 

144 return f"config:registry.managers.{name}" 

145 

146 @classmethod 

147 def _managerVersionKey(cls, extension: VersionedExtension) -> str: 

148 """Return key used to store manager version. 

149 

150 Parameters 

151 ---------- 

152 extension : `VersionedExtension` 

153 Instance of the extension. 

154 

155 Returns 

156 ------- 

157 key : `str` 

158 Name of the key in attributes table. 

159 """ 

160 return "version:" + extension.extensionName() 

161 

162 @classmethod 

163 def _managerDigestKey(cls, extension: VersionedExtension) -> str: 

164 """Return key used to store manager schema digest. 

165 

166 Parameters 

167 ---------- 

168 extension : `VersionedExtension` 

169 Instance of the extension. 

170 

171 Returns 

172 ------- 

173 key : `str` 

174 Name of the key in attributes table. 

175 """ 

176 return "schema_digest:" + extension.extensionName() 

177 

178 @staticmethod 

179 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

180 """Compare two versions for compatibility. 

181 

182 Parameters 

183 ---------- 

184 old_version : `VersionTuple` 

185 Old schema version, typically one stored in a database. 

186 new_version : `VersionTuple` 

187 New schema version, typically version defined in configuration. 

188 update : `bool` 

189 If True then read-write access is expected. 

190 """ 

191 if old_version.major != new_version.major: 

192 # different major versions are not compatible at all 

193 return False 

194 if old_version.minor != new_version.minor: 

195 # different minor versions are backward compatible for read 

196 # access only 

197 return new_version.minor > old_version.minor and not update 

198 # patch difference does not matter 

199 return True 

200 

201 def storeManagersConfig(self) -> None: 

202 """Store configured extension names in attributes table. 

203 

204 For each extension we store a record with the key 

205 "config:registry.managers.{name}" and fully qualified class name as a 

206 value. 

207 """ 

208 for name, extension in self._managers.items(): 

209 key = self._managerConfigKey(name) 

210 value = extension.extensionName() 

211 self._attributes.set(key, value) 

212 _LOG.debug("saved manager config %s=%s", key, value) 

213 self._emptyFlag = False 

214 

215 def storeManagersVersions(self) -> None: 

216 """Store current manager versions in registry arttributes. 

217 

218 For each extension we store two records: 

219 

220 - record with the key "version:{fullExtensionName}" and version 

221 number in its string format as a value, 

222 - record with the key "schema_digest:{fullExtensionName}" and 

223 schema digest as a value. 

224 """ 

225 for extension in self._managers.values(): 

226 version = extension.currentVersion() 

227 if version: 

228 key = self._managerVersionKey(extension) 

229 value = str(version) 

230 self._attributes.set(key, value) 

231 _LOG.debug("saved manager version %s=%s", key, value) 

232 

233 digest = extension.schemaDigest() 

234 if digest is not None: 

235 key = self._managerDigestKey(extension) 

236 self._attributes.set(key, digest) 

237 _LOG.debug("saved manager schema digest %s=%s", key, digest) 

238 

239 self._emptyFlag = False 

240 

241 @property 

242 def _attributesEmpty(self) -> bool: 

243 """True if attributes table is empty.""" 

244 # There are existing repositories where attributes table was not 

245 # filled, we don't want to force schema migration in this case yet 

246 # (and we don't have tools) so we allow this as valid use case and 

247 # skip all checks but print a warning. 

248 if self._emptyFlag is None: 

249 self._emptyFlag = self._attributes.empty() 

250 if self._emptyFlag: 

251 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

252 return self._emptyFlag 

253 

254 def checkManagersConfig(self) -> None: 

255 """Compare configured manager names with stored in database. 

256 

257 Raises 

258 ------ 

259 ManagerMismatchError 

260 Raised if manager names are different. 

261 MissingManagerError 

262 Raised if database has no stored manager name. 

263 """ 

264 if self._attributesEmpty: 

265 return 

266 

267 missing = [] 

268 mismatch = [] 

269 for name, extension in self._managers.items(): 

270 key = self._managerConfigKey(name) 

271 storedMgr = self._attributes.get(key) 

272 _LOG.debug("found manager config %s=%s", key, storedMgr) 

273 if storedMgr is None: 

274 missing.append(name) 

275 continue 

276 if extension.extensionName() != storedMgr: 

277 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}") 

278 if missing: 

279 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

280 if mismatch: 

281 raise ManagerMismatchError( 

282 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

283 ) 

284 

285 def checkManagersVersions(self, writeable: bool) -> None: 

286 """Compare configured versions with the versions stored in database. 

287 

288 Parameters 

289 ---------- 

290 writeable : `bool` 

291 If ``True`` then read-write access needs to be checked. 

292 

293 Raises 

294 ------ 

295 IncompatibleVersionError 

296 Raised if versions are not compatible. 

297 MissingVersionError 

298 Raised if database has no stored version for one or more groups. 

299 """ 

300 if self._attributesEmpty: 

301 return 

302 

303 for extension in self._managers.values(): 

304 version = extension.currentVersion() 

305 if version: 

306 key = self._managerVersionKey(extension) 

307 storedVersionStr = self._attributes.get(key) 

308 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version) 

309 if storedVersionStr is None: 

310 raise MissingVersionError(f"Failed to read version number {key}") 

311 storedVersion = VersionTuple.fromString(storedVersionStr) 

312 if not self.checkCompatibility(storedVersion, version, writeable): 

313 raise IncompatibleVersionError( 

314 f"Configured version {version} is not compatible with stored version " 

315 f"{storedVersion} for extension {extension.extensionName()}" 

316 )