Coverage for python/lsst/daf/butler/registry/versions.py: 28%

90 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 10:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = [ 

31 "ButlerVersionsManager", 

32 "IncompatibleVersionError", 

33 "MissingManagerError", 

34 "ManagerMismatchError", 

35] 

36 

37import logging 

38from collections.abc import Mapping 

39from typing import TYPE_CHECKING 

40 

41from .interfaces import VersionedExtension, VersionTuple 

42 

43if TYPE_CHECKING: 

44 from .interfaces import ButlerAttributeManager 

45 

46 

47_LOG = logging.getLogger(__name__) 

48 

49 

50class IncompatibleVersionError(RuntimeError): 

51 """Exception raised when configured version number is not compatible with 

52 database version. 

53 """ 

54 

55 pass 

56 

57 

58class MissingManagerError(RuntimeError): 

59 """Exception raised when manager name is missing from registry.""" 

60 

61 pass 

62 

63 

64class ManagerMismatchError(RuntimeError): 

65 """Exception raised when configured manager name does not match name 

66 stored in the database. 

67 """ 

68 

69 pass 

70 

71 

72class ButlerVersionsManager: 

73 """Utility class to manage and verify schema version compatibility. 

74 

75 Parameters 

76 ---------- 

77 attributes : `ButlerAttributeManager` 

78 Attribute manager instance. 

79 """ 

80 

81 def __init__(self, attributes: ButlerAttributeManager): 

82 self._attributes = attributes 

83 # Maps manager type to its class name and schema version. 

84 self._cache: Mapping[str, tuple[str, VersionTuple | None]] | None = None 

85 self._emptyFlag: bool | None = None 

86 

87 @classmethod 

88 def _managerConfigKey(cls, name: str) -> str: 

89 """Return key used to store manager config. 

90 

91 Parameters 

92 ---------- 

93 name : `str` 

94 Name of the namager type, e.g. "dimensions" 

95 

96 Returns 

97 ------- 

98 key : `str` 

99 Name of the key in attributes table. 

100 """ 

101 return f"config:registry.managers.{name}" 

102 

103 @classmethod 

104 def _managerVersionKey(cls, extensionName: str) -> str: 

105 """Return key used to store manager version. 

106 

107 Parameters 

108 ---------- 

109 extensionName : `str` 

110 Extension name (e.g. its class name). 

111 

112 Returns 

113 ------- 

114 key : `str` 

115 Name of the key in attributes table. 

116 """ 

117 return f"version:{extensionName}" 

118 

119 @property 

120 def _manager_data(self) -> Mapping[str, tuple[str, VersionTuple | None]]: 

121 """Retrieve per-manager type name and schema version.""" 

122 if not self._cache: 

123 self._cache = {} 

124 

125 # Number of items in attributes table is small, read all of them 

126 # in a single query and filter later. 

127 attributes = dict(self._attributes.items()) 

128 

129 for name, value in attributes.items(): 

130 if name.startswith("config:registry.managers."): 

131 _, _, manager_type = name.rpartition(".") 

132 manager_class = value 

133 version_str = attributes.get(self._managerVersionKey(manager_class)) 

134 if version_str is None: 

135 self._cache[manager_type] = (manager_class, None) 

136 else: 

137 version = VersionTuple.fromString(version_str) 

138 self._cache[manager_type] = (manager_class, version) 

139 

140 return self._cache 

141 

142 @staticmethod 

143 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

144 """Compare two versions for compatibility. 

145 

146 Parameters 

147 ---------- 

148 old_version : `VersionTuple` 

149 Old schema version, typically one stored in a database. 

150 new_version : `VersionTuple` 

151 New schema version, typically version defined in configuration. 

152 update : `bool` 

153 If True then read-write access is expected. 

154 """ 

155 if old_version.major != new_version.major: 

156 # different major versions are not compatible at all 

157 return False 

158 if old_version.minor != new_version.minor: 

159 # different minor versions are backward compatible for read 

160 # access only 

161 return new_version.minor > old_version.minor and not update 

162 # patch difference does not matter 

163 return True 

164 

165 def storeManagersConfig(self, managers: Mapping[str, VersionedExtension]) -> None: 

166 """Store configured extension names and their versions. 

167 

168 Parameters 

169 ---------- 

170 managers : `~collections.abc.Mapping` [`str`, `type`] 

171 Collection of manager extension classes, the key is a manager type, 

172 e.g. "datasets". 

173 

174 Notes 

175 ----- 

176 For each extension we store two records: 

177 - with the key "config:registry.managers.{name}" and fully qualified 

178 class name as a value, 

179 - with the key "version:{fullExtensionName}" and version number in its 

180 string format as a value. 

181 """ 

182 for name, extension in managers.items(): 

183 key = self._managerConfigKey(name) 

184 value = extension.extensionName() 

185 self._attributes.set(key, value) 

186 _LOG.debug("saved manager config %s=%s", key, value) 

187 

188 version = extension.newSchemaVersion() 

189 if version: 

190 key = self._managerVersionKey(extension.extensionName()) 

191 value = str(version) 

192 self._attributes.set(key, value) 

193 _LOG.debug("saved manager version %s=%s", key, value) 

194 

195 @property 

196 def _attributesEmpty(self) -> bool: 

197 """True if attributes table is empty.""" 

198 # There are existing repositories where attributes table was not 

199 # filled, we don't want to force schema migration in this case yet 

200 # (and we don't have tools) so we allow this as valid use case and 

201 # skip all checks but print a warning. 

202 if self._emptyFlag is None: 

203 self._emptyFlag = self._attributes.empty() 

204 if self._emptyFlag: 

205 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

206 return self._emptyFlag 

207 

208 def checkManagersConfig(self, managers: Mapping[str, type[VersionedExtension]]) -> None: 

209 """Compare configured manager names versions with stored in database. 

210 

211 Parameters 

212 ---------- 

213 managers : `~collections.abc.Mapping` [ `str`, `type`] 

214 The configured managers to check. 

215 

216 Raises 

217 ------ 

218 ManagerMismatchError 

219 Raised if manager names are different. 

220 MissingManagerError 

221 Raised if database has no stored manager name. 

222 IncompatibleVersionError 

223 Raised if versions are not compatible. 

224 """ 

225 if self._attributesEmpty: 

226 return 

227 

228 manager_data = self._manager_data 

229 

230 missing = [] 

231 mismatch = [] 

232 for name, extension in managers.items(): 

233 try: 

234 manager_class, _ = manager_data[name] 

235 _LOG.debug("found manager config %s=%s", name, manager_class) 

236 except KeyError: 

237 missing.append(name) 

238 continue 

239 if extension.extensionName() != manager_class: 

240 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {manager_class}") 

241 if missing: 

242 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

243 if mismatch: 

244 raise ManagerMismatchError( 

245 "Configured managers do not match registry-stored names:\n" + "\n".join(mismatch) 

246 ) 

247 

248 def managerVersions(self) -> Mapping[str, VersionTuple]: 

249 """Return schema versions for each manager. 

250 

251 Returns 

252 ------- 

253 versions : `~collections.abc.Mapping` [`str`, `VersionTuple`] 

254 Mapping of managert type (e.g. "datasets") to its schema version. 

255 """ 

256 versions = {} 

257 for manager_type, (_, version) in self._manager_data.items(): 

258 if version is not None: 

259 versions[manager_type] = version 

260 return versions