Coverage for python/lsst/daf/butler/registry/versions.py: 22%

95 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-28 04:40 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingVersionError", 

28 "MissingManagerError", 

29 "ManagerMismatchError", 

30] 

31 

32import logging 

33from collections.abc import Mapping, MutableMapping 

34from typing import TYPE_CHECKING, Any 

35 

36from .interfaces import VersionedExtension, VersionTuple 

37 

38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 from .interfaces import ButlerAttributeManager 

40 

41 

42_LOG = logging.getLogger(__name__) 

43 

44 

45class MissingVersionError(RuntimeError): 

46 """Exception raised when existing database is missing attributes with 

47 version numbers. 

48 """ 

49 

50 pass 

51 

52 

53class IncompatibleVersionError(RuntimeError): 

54 """Exception raised when configured version number is not compatible with 

55 database version. 

56 """ 

57 

58 pass 

59 

60 

61class MissingManagerError(RuntimeError): 

62 """Exception raised when manager name is missing from registry.""" 

63 

64 pass 

65 

66 

67class ManagerMismatchError(RuntimeError): 

68 """Exception raised when configured manager name does not match name 

69 stored in the database. 

70 """ 

71 

72 pass 

73 

74 

75class ButlerVersionsManager: 

76 """Utility class to manage and verify schema version compatibility. 

77 

78 Parameters 

79 ---------- 

80 attributes : `ButlerAttributeManager` 

81 Attribute manager instance. 

82 managers : `dict` [`str`, `VersionedExtension`] 

83 Mapping of extension type as defined in configuration (e.g. 

84 "collections") to corresponding instance of manager. 

85 """ 

86 

87 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]): 

88 self._attributes = attributes 

89 self._managers: MutableMapping[str, VersionedExtension] = {} 

90 # we only care about managers implementing VersionedExtension interface 

91 for name, manager in managers.items(): 

92 if isinstance(manager, VersionedExtension): 

93 self._managers[name] = manager 

94 elif manager is not None: 

95 # All regular managers need to support versioning mechanism. 

96 _LOG.warning("extension %r does not implement VersionedExtension", name) 

97 self._emptyFlag: bool | None = None 

98 

99 @classmethod 

100 def _managerConfigKey(cls, name: str) -> str: 

101 """Return key used to store manager config. 

102 

103 Parameters 

104 ---------- 

105 name : `str` 

106 Name of the namager type, e.g. "dimensions" 

107 

108 Returns 

109 ------- 

110 key : `str` 

111 Name of the key in attributes table. 

112 """ 

113 return f"config:registry.managers.{name}" 

114 

115 @classmethod 

116 def _managerVersionKey(cls, extension: VersionedExtension) -> str: 

117 """Return key used to store manager version. 

118 

119 Parameters 

120 ---------- 

121 extension : `VersionedExtension` 

122 Instance of the extension. 

123 

124 Returns 

125 ------- 

126 key : `str` 

127 Name of the key in attributes table. 

128 """ 

129 return "version:" + extension.extensionName() 

130 

131 @staticmethod 

132 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

133 """Compare two versions for compatibility. 

134 

135 Parameters 

136 ---------- 

137 old_version : `VersionTuple` 

138 Old schema version, typically one stored in a database. 

139 new_version : `VersionTuple` 

140 New schema version, typically version defined in configuration. 

141 update : `bool` 

142 If True then read-write access is expected. 

143 """ 

144 if old_version.major != new_version.major: 

145 # different major versions are not compatible at all 

146 return False 

147 if old_version.minor != new_version.minor: 

148 # different minor versions are backward compatible for read 

149 # access only 

150 return new_version.minor > old_version.minor and not update 

151 # patch difference does not matter 

152 return True 

153 

154 def storeManagersConfig(self) -> None: 

155 """Store configured extension names in attributes table. 

156 

157 For each extension we store a record with the key 

158 "config:registry.managers.{name}" and fully qualified class name as a 

159 value. 

160 """ 

161 for name, extension in self._managers.items(): 

162 key = self._managerConfigKey(name) 

163 value = extension.extensionName() 

164 self._attributes.set(key, value) 

165 _LOG.debug("saved manager config %s=%s", key, value) 

166 self._emptyFlag = False 

167 

168 def storeManagersVersions(self) -> None: 

169 """Store current manager versions in registry attributes. 

170 

171 For each extension we store single record with a key 

172 "version:{fullExtensionName}" and version number in its string format 

173 as a value. 

174 """ 

175 for extension in self._managers.values(): 

176 version = extension.currentVersion() 

177 if version: 

178 key = self._managerVersionKey(extension) 

179 value = str(version) 

180 self._attributes.set(key, value) 

181 _LOG.debug("saved manager version %s=%s", key, value) 

182 

183 self._emptyFlag = False 

184 

185 @property 

186 def _attributesEmpty(self) -> bool: 

187 """True if attributes table is empty.""" 

188 # There are existing repositories where attributes table was not 

189 # filled, we don't want to force schema migration in this case yet 

190 # (and we don't have tools) so we allow this as valid use case and 

191 # skip all checks but print a warning. 

192 if self._emptyFlag is None: 

193 self._emptyFlag = self._attributes.empty() 

194 if self._emptyFlag: 

195 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

196 return self._emptyFlag 

197 

198 def checkManagersConfig(self) -> None: 

199 """Compare configured manager names with stored in database. 

200 

201 Raises 

202 ------ 

203 ManagerMismatchError 

204 Raised if manager names are different. 

205 MissingManagerError 

206 Raised if database has no stored manager name. 

207 """ 

208 if self._attributesEmpty: 

209 return 

210 

211 missing = [] 

212 mismatch = [] 

213 for name, extension in self._managers.items(): 

214 key = self._managerConfigKey(name) 

215 storedMgr = self._attributes.get(key) 

216 _LOG.debug("found manager config %s=%s", key, storedMgr) 

217 if storedMgr is None: 

218 missing.append(name) 

219 continue 

220 if extension.extensionName() != storedMgr: 

221 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}") 

222 if missing: 

223 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

224 if mismatch: 

225 raise ManagerMismatchError( 

226 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

227 ) 

228 

229 def checkManagersVersions(self, writeable: bool) -> None: 

230 """Compare configured versions with the versions stored in database. 

231 

232 Parameters 

233 ---------- 

234 writeable : `bool` 

235 If ``True`` then read-write access needs to be checked. 

236 

237 Raises 

238 ------ 

239 IncompatibleVersionError 

240 Raised if versions are not compatible. 

241 MissingVersionError 

242 Raised if database has no stored version for one or more groups. 

243 """ 

244 if self._attributesEmpty: 

245 return 

246 

247 for extension in self._managers.values(): 

248 version = extension.currentVersion() 

249 if version: 

250 key = self._managerVersionKey(extension) 

251 storedVersionStr = self._attributes.get(key) 

252 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version) 

253 if storedVersionStr is None: 

254 raise MissingVersionError(f"Failed to read version number {key}") 

255 storedVersion = VersionTuple.fromString(storedVersionStr) 

256 if not self.checkCompatibility(storedVersion, version, writeable): 

257 raise IncompatibleVersionError( 

258 f"Configured version {version} is not compatible with stored version " 

259 f"{storedVersion} for extension {extension.extensionName()}" 

260 )