Coverage for python/lsst/daf/butler/registry/versions.py: 28%

90 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-01 11:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = [ 

31 "ButlerVersionsManager", 

32 "IncompatibleVersionError", 

33 "MissingManagerError", 

34 "ManagerMismatchError", 

35] 

36 

37import logging 

38from collections.abc import Mapping 

39from typing import TYPE_CHECKING 

40 

41from .interfaces import VersionedExtension, VersionTuple 

42 

43if TYPE_CHECKING: 

44 from .interfaces import ButlerAttributeManager 

45 

46 

47_LOG = logging.getLogger(__name__) 

48 

49 

50class IncompatibleVersionError(RuntimeError): 

51 """Exception raised when configured version number is not compatible with 

52 database version. 

53 """ 

54 

55 pass 

56 

57 

58class MissingManagerError(RuntimeError): 

59 """Exception raised when manager name is missing from registry.""" 

60 

61 pass 

62 

63 

64class ManagerMismatchError(RuntimeError): 

65 """Exception raised when configured manager name does not match name 

66 stored in the database. 

67 """ 

68 

69 pass 

70 

71 

72class ButlerVersionsManager: 

73 """Utility class to manage and verify schema version compatibility. 

74 

75 Parameters 

76 ---------- 

77 attributes : `ButlerAttributeManager` 

78 Attribute manager instance. 

79 managers : `dict` [`str`, `VersionedExtension`] 

80 Mapping of extension type as defined in configuration (e.g. 

81 "collections") to corresponding instance of manager. 

82 """ 

83 

84 def __init__(self, attributes: ButlerAttributeManager): 

85 self._attributes = attributes 

86 # Maps manager type to its class name and schema version. 

87 self._cache: Mapping[str, tuple[str, VersionTuple | None]] | None = None 

88 self._emptyFlag: bool | None = None 

89 

90 @classmethod 

91 def _managerConfigKey(cls, name: str) -> str: 

92 """Return key used to store manager config. 

93 

94 Parameters 

95 ---------- 

96 name : `str` 

97 Name of the namager type, e.g. "dimensions" 

98 

99 Returns 

100 ------- 

101 key : `str` 

102 Name of the key in attributes table. 

103 """ 

104 return f"config:registry.managers.{name}" 

105 

106 @classmethod 

107 def _managerVersionKey(cls, extensionName: str) -> str: 

108 """Return key used to store manager version. 

109 

110 Parameters 

111 ---------- 

112 extensionName : `str` 

113 Extension name (e.g. its class name). 

114 

115 Returns 

116 ------- 

117 key : `str` 

118 Name of the key in attributes table. 

119 """ 

120 return f"version:{extensionName}" 

121 

122 @property 

123 def _manager_data(self) -> Mapping[str, tuple[str, VersionTuple | None]]: 

124 """Retrieve per-manager type name and schema version.""" 

125 if not self._cache: 

126 self._cache = {} 

127 

128 # Number of items in attributes table is small, read all of them 

129 # in a single query and filter later. 

130 attributes = dict(self._attributes.items()) 

131 

132 for name, value in attributes.items(): 

133 if name.startswith("config:registry.managers."): 

134 _, _, manager_type = name.rpartition(".") 

135 manager_class = value 

136 version_str = attributes.get(self._managerVersionKey(manager_class)) 

137 if version_str is None: 

138 self._cache[manager_type] = (manager_class, None) 

139 else: 

140 version = VersionTuple.fromString(version_str) 

141 self._cache[manager_type] = (manager_class, version) 

142 

143 return self._cache 

144 

145 @staticmethod 

146 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

147 """Compare two versions for compatibility. 

148 

149 Parameters 

150 ---------- 

151 old_version : `VersionTuple` 

152 Old schema version, typically one stored in a database. 

153 new_version : `VersionTuple` 

154 New schema version, typically version defined in configuration. 

155 update : `bool` 

156 If True then read-write access is expected. 

157 """ 

158 if old_version.major != new_version.major: 

159 # different major versions are not compatible at all 

160 return False 

161 if old_version.minor != new_version.minor: 

162 # different minor versions are backward compatible for read 

163 # access only 

164 return new_version.minor > old_version.minor and not update 

165 # patch difference does not matter 

166 return True 

167 

168 def storeManagersConfig(self, managers: Mapping[str, VersionedExtension]) -> None: 

169 """Store configured extension names and their versions. 

170 

171 Parmeters 

172 --------- 

173 managers: `~collections.abc.Mapping` [`str`, `type`] 

174 Collection of manager extension classes, the key is a manager type, 

175 e.g. "datasets". 

176 

177 Notes 

178 ----- 

179 For each extension we store two records: 

180 - with the key "config:registry.managers.{name}" and fully qualified 

181 class name as a value, 

182 - with the key "version:{fullExtensionName}" and version number in its 

183 string format as a value. 

184 """ 

185 for name, extension in managers.items(): 

186 key = self._managerConfigKey(name) 

187 value = extension.extensionName() 

188 self._attributes.set(key, value) 

189 _LOG.debug("saved manager config %s=%s", key, value) 

190 

191 version = extension.newSchemaVersion() 

192 if version: 

193 key = self._managerVersionKey(extension.extensionName()) 

194 value = str(version) 

195 self._attributes.set(key, value) 

196 _LOG.debug("saved manager version %s=%s", key, value) 

197 

198 @property 

199 def _attributesEmpty(self) -> bool: 

200 """True if attributes table is empty.""" 

201 # There are existing repositories where attributes table was not 

202 # filled, we don't want to force schema migration in this case yet 

203 # (and we don't have tools) so we allow this as valid use case and 

204 # skip all checks but print a warning. 

205 if self._emptyFlag is None: 

206 self._emptyFlag = self._attributes.empty() 

207 if self._emptyFlag: 

208 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

209 return self._emptyFlag 

210 

211 def checkManagersConfig(self, managers: Mapping[str, type[VersionedExtension]]) -> None: 

212 """Compare configured manager names versions with stored in database. 

213 

214 Raises 

215 ------ 

216 ManagerMismatchError 

217 Raised if manager names are different. 

218 MissingManagerError 

219 Raised if database has no stored manager name. 

220 IncompatibleVersionError 

221 Raised if versions are not compatible. 

222 """ 

223 if self._attributesEmpty: 

224 return 

225 

226 manager_data = self._manager_data 

227 

228 missing = [] 

229 mismatch = [] 

230 for name, extension in managers.items(): 

231 try: 

232 manager_class, _ = manager_data[name] 

233 _LOG.debug("found manager config %s=%s", name, manager_class) 

234 except KeyError: 

235 missing.append(name) 

236 continue 

237 if extension.extensionName() != manager_class: 

238 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {manager_class}") 

239 if missing: 

240 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

241 if mismatch: 

242 raise ManagerMismatchError( 

243 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

244 ) 

245 

246 def managerVersions(self) -> Mapping[str, VersionTuple]: 

247 """Return schema versions for each manager. 

248 

249 Returns 

250 ------- 

251 versions : `~collections.abc.Mapping` [`str`, `VersionTuple`] 

252 Mapping of managert type (e.g. "datasets") to its schema version. 

253 """ 

254 versions = {} 

255 for manager_type, (_, version) in self._manager_data.items(): 

256 if version is not None: 

257 versions[manager_type] = version 

258 return versions