Coverage for python/lsst/daf/butler/registry/versions.py: 22%

90 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-06 09:33 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ButlerVersionsManager", 

26 "IncompatibleVersionError", 

27 "MissingManagerError", 

28 "ManagerMismatchError", 

29] 

30 

31import logging 

32from collections.abc import Mapping 

33from typing import TYPE_CHECKING 

34 

35from .interfaces import VersionedExtension, VersionTuple 

36 

37if TYPE_CHECKING: 

38 from .interfaces import ButlerAttributeManager 

39 

40 

41_LOG = logging.getLogger(__name__) 

42 

43 

44class IncompatibleVersionError(RuntimeError): 

45 """Exception raised when configured version number is not compatible with 

46 database version. 

47 """ 

48 

49 pass 

50 

51 

52class MissingManagerError(RuntimeError): 

53 """Exception raised when manager name is missing from registry.""" 

54 

55 pass 

56 

57 

58class ManagerMismatchError(RuntimeError): 

59 """Exception raised when configured manager name does not match name 

60 stored in the database. 

61 """ 

62 

63 pass 

64 

65 

66class ButlerVersionsManager: 

67 """Utility class to manage and verify schema version compatibility. 

68 

69 Parameters 

70 ---------- 

71 attributes : `ButlerAttributeManager` 

72 Attribute manager instance. 

73 managers : `dict` [`str`, `VersionedExtension`] 

74 Mapping of extension type as defined in configuration (e.g. 

75 "collections") to corresponding instance of manager. 

76 """ 

77 

78 def __init__(self, attributes: ButlerAttributeManager): 

79 self._attributes = attributes 

80 # Maps manager type to its class name and schema version. 

81 self._cache: Mapping[str, tuple[str, VersionTuple | None]] | None = None 

82 self._emptyFlag: bool | None = None 

83 

84 @classmethod 

85 def _managerConfigKey(cls, name: str) -> str: 

86 """Return key used to store manager config. 

87 

88 Parameters 

89 ---------- 

90 name : `str` 

91 Name of the namager type, e.g. "dimensions" 

92 

93 Returns 

94 ------- 

95 key : `str` 

96 Name of the key in attributes table. 

97 """ 

98 return f"config:registry.managers.{name}" 

99 

100 @classmethod 

101 def _managerVersionKey(cls, extensionName: str) -> str: 

102 """Return key used to store manager version. 

103 

104 Parameters 

105 ---------- 

106 extensionName : `str` 

107 Extension name (e.g. its class name). 

108 

109 Returns 

110 ------- 

111 key : `str` 

112 Name of the key in attributes table. 

113 """ 

114 return f"version:{extensionName}" 

115 

116 @property 

117 def _manager_data(self) -> Mapping[str, tuple[str, VersionTuple | None]]: 

118 """Retrieve per-manager type name and schema version.""" 

119 if not self._cache: 

120 self._cache = {} 

121 

122 # Number of items in attributes table is small, read all of them 

123 # in a single query and filter later. 

124 attributes = {key: value for key, value in self._attributes.items()} 

125 

126 for name, value in attributes.items(): 

127 if name.startswith("config:registry.managers."): 

128 _, _, manager_type = name.rpartition(".") 

129 manager_class = value 

130 version_str = attributes.get(self._managerVersionKey(manager_class)) 

131 if version_str is None: 

132 self._cache[manager_type] = (manager_class, None) 

133 else: 

134 version = VersionTuple.fromString(version_str) 

135 self._cache[manager_type] = (manager_class, version) 

136 

137 return self._cache 

138 

139 @staticmethod 

140 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool: 

141 """Compare two versions for compatibility. 

142 

143 Parameters 

144 ---------- 

145 old_version : `VersionTuple` 

146 Old schema version, typically one stored in a database. 

147 new_version : `VersionTuple` 

148 New schema version, typically version defined in configuration. 

149 update : `bool` 

150 If True then read-write access is expected. 

151 """ 

152 if old_version.major != new_version.major: 

153 # different major versions are not compatible at all 

154 return False 

155 if old_version.minor != new_version.minor: 

156 # different minor versions are backward compatible for read 

157 # access only 

158 return new_version.minor > old_version.minor and not update 

159 # patch difference does not matter 

160 return True 

161 

162 def storeManagersConfig(self, managers: Mapping[str, VersionedExtension]) -> None: 

163 """Store configured extension names and their versions. 

164 

165 Parmeters 

166 --------- 

167 managers: `~collections.abc.Mapping` [`str`, `type`] 

168 Collection of manager extension classes, the key is a manager type, 

169 e.g. "datasets". 

170 

171 Notes 

172 ----- 

173 For each extension we store two records: 

174 - with the key "config:registry.managers.{name}" and fully qualified 

175 class name as a value, 

176 - with the key "version:{fullExtensionName}" and version number in its 

177 string format as a value. 

178 """ 

179 for name, extension in managers.items(): 

180 key = self._managerConfigKey(name) 

181 value = extension.extensionName() 

182 self._attributes.set(key, value) 

183 _LOG.debug("saved manager config %s=%s", key, value) 

184 

185 version = extension.newSchemaVersion() 

186 if version: 

187 key = self._managerVersionKey(extension.extensionName()) 

188 value = str(version) 

189 self._attributes.set(key, value) 

190 _LOG.debug("saved manager version %s=%s", key, value) 

191 

192 @property 

193 def _attributesEmpty(self) -> bool: 

194 """True if attributes table is empty.""" 

195 # There are existing repositories where attributes table was not 

196 # filled, we don't want to force schema migration in this case yet 

197 # (and we don't have tools) so we allow this as valid use case and 

198 # skip all checks but print a warning. 

199 if self._emptyFlag is None: 

200 self._emptyFlag = self._attributes.empty() 

201 if self._emptyFlag: 

202 _LOG.warning("Attributes table is empty, schema may need an upgrade.") 

203 return self._emptyFlag 

204 

205 def checkManagersConfig(self, managers: Mapping[str, type[VersionedExtension]]) -> None: 

206 """Compare configured manager names versions with stored in database. 

207 

208 Raises 

209 ------ 

210 ManagerMismatchError 

211 Raised if manager names are different. 

212 MissingManagerError 

213 Raised if database has no stored manager name. 

214 IncompatibleVersionError 

215 Raised if versions are not compatible. 

216 """ 

217 if self._attributesEmpty: 

218 return 

219 

220 manager_data = self._manager_data 

221 

222 missing = [] 

223 mismatch = [] 

224 for name, extension in managers.items(): 

225 try: 

226 manager_class, _ = manager_data[name] 

227 _LOG.debug("found manager config %s=%s", name, manager_class) 

228 except KeyError: 

229 missing.append(name) 

230 continue 

231 if extension.extensionName() != manager_class: 

232 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {manager_class}") 

233 if missing: 

234 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing)) 

235 if mismatch: 

236 raise ManagerMismatchError( 

237 "Configured managers do not match registry-stored names:\n" + "\n".join(missing) 

238 ) 

239 

240 def managerVersions(self) -> Mapping[str, VersionTuple]: 

241 """Return schema versions for each manager. 

242 

243 Returns 

244 ------- 

245 versions : `~collections.abc.Mapping` [`str`, `VersionTuple`] 

246 Mapping of managert type (e.g. "datasets") to its schema version. 

247 """ 

248 versions = {} 

249 for manager_type, (_, version) in self._manager_data.items(): 

250 if version is not None: 

251 versions[manager_type] = version 

252 return versions