Coverage for python/lsst/daf/butler/registry/versions.py: 23%
108 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-11 02:06 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-11 02:06 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30 "DigestMismatchError",
31]
33import logging
34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional
36from .interfaces import VersionedExtension, VersionTuple
38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 from .interfaces import ButlerAttributeManager
42_LOG = logging.getLogger(__name__)
45class MissingVersionError(RuntimeError):
46 """Exception raised when existing database is missing attributes with
47 version numbers.
48 """
50 pass
53class IncompatibleVersionError(RuntimeError):
54 """Exception raised when configured version number is not compatible with
55 database version.
56 """
58 pass
61class MissingManagerError(RuntimeError):
62 """Exception raised when manager name is missing from registry."""
64 pass
67class ManagerMismatchError(RuntimeError):
68 """Exception raised when configured manager name does not match name
69 stored in the database.
70 """
72 pass
75class DigestMismatchError(RuntimeError):
76 """Exception raised when schema digest is not equal to stored digest."""
78 pass
81class VersionInfo:
82 """Representation of version information as defined by configuration.
84 Parameters
85 ----------
86 version : `VersionTuple`
87 Version number in parsed format.
88 digest : `str`, optional
89 Optional digest of the corresponding part of the schema definition.
91 Notes
92 -----
93 Schema digest is supposed to help with detecting unintentional schema
94 changes in the code without upgrading schema version. Digest is
95 constructed whom the set of table definitions and is compared to a digest
96 defined in configuration, if two digests differ it means schema was
97 changed. Intentional schema updates will need to update both configured
98 schema version and schema digest.
99 """
101 def __init__(self, version: VersionTuple, digest: Optional[str] = None):
102 self.version = version
103 self.digest = digest
106class ButlerVersionsManager:
107 """Utility class to manage and verify schema version compatibility.
109 Parameters
110 ----------
111 attributes : `ButlerAttributeManager`
112 Attribute manager instance.
113 managers : `dict` [`str`, `VersionedExtension`]
114 Mapping of extension type as defined in configuration (e.g.
115 "collections") to corresponding instance of manager.
116 """
118 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]):
119 self._attributes = attributes
120 self._managers: MutableMapping[str, VersionedExtension] = {}
121 # we only care about managers implementing VersionedExtension interface
122 for name, manager in managers.items():
123 if isinstance(manager, VersionedExtension):
124 self._managers[name] = manager
125 elif manager is not None:
126 # All regular managers need to support versioning mechanism.
127 _LOG.warning("extension %r does not implement VersionedExtension", name)
128 self._emptyFlag: Optional[bool] = None
130 @classmethod
131 def _managerConfigKey(cls, name: str) -> str:
132 """Return key used to store manager config.
134 Parameters
135 ----------
136 name : `str`
137 Name of the namager type, e.g. "dimensions"
139 Returns
140 -------
141 key : `str`
142 Name of the key in attributes table.
143 """
144 return f"config:registry.managers.{name}"
146 @classmethod
147 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
148 """Return key used to store manager version.
150 Parameters
151 ----------
152 extension : `VersionedExtension`
153 Instance of the extension.
155 Returns
156 -------
157 key : `str`
158 Name of the key in attributes table.
159 """
160 return "version:" + extension.extensionName()
162 @classmethod
163 def _managerDigestKey(cls, extension: VersionedExtension) -> str:
164 """Return key used to store manager schema digest.
166 Parameters
167 ----------
168 extension : `VersionedExtension`
169 Instance of the extension.
171 Returns
172 -------
173 key : `str`
174 Name of the key in attributes table.
175 """
176 return "schema_digest:" + extension.extensionName()
178 @staticmethod
179 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
180 """Compare two versions for compatibility.
182 Parameters
183 ----------
184 old_version : `VersionTuple`
185 Old schema version, typically one stored in a database.
186 new_version : `VersionTuple`
187 New schema version, typically version defined in configuration.
188 update : `bool`
189 If True then read-write access is expected.
190 """
191 if old_version.major != new_version.major:
192 # different major versions are not compatible at all
193 return False
194 if old_version.minor != new_version.minor:
195 # different minor versions are backward compatible for read
196 # access only
197 return new_version.minor > old_version.minor and not update
198 # patch difference does not matter
199 return True
201 def storeManagersConfig(self) -> None:
202 """Store configured extension names in attributes table.
204 For each extension we store a record with the key
205 "config:registry.managers.{name}" and fully qualified class name as a
206 value.
207 """
208 for name, extension in self._managers.items():
209 key = self._managerConfigKey(name)
210 value = extension.extensionName()
211 self._attributes.set(key, value)
212 _LOG.debug("saved manager config %s=%s", key, value)
213 self._emptyFlag = False
215 def storeManagersVersions(self) -> None:
216 """Store current manager versions in registry arttributes.
218 For each extension we store two records:
220 - record with the key "version:{fullExtensionName}" and version
221 number in its string format as a value,
222 - record with the key "schema_digest:{fullExtensionName}" and
223 schema digest as a value.
224 """
225 for extension in self._managers.values():
226 version = extension.currentVersion()
227 if version:
228 key = self._managerVersionKey(extension)
229 value = str(version)
230 self._attributes.set(key, value)
231 _LOG.debug("saved manager version %s=%s", key, value)
233 digest = extension.schemaDigest()
234 if digest is not None:
235 key = self._managerDigestKey(extension)
236 self._attributes.set(key, digest)
237 _LOG.debug("saved manager schema digest %s=%s", key, digest)
239 self._emptyFlag = False
241 @property
242 def _attributesEmpty(self) -> bool:
243 """True if attributes table is empty."""
244 # There are existing repositories where attributes table was not
245 # filled, we don't want to force schema migration in this case yet
246 # (and we don't have tools) so we allow this as valid use case and
247 # skip all checks but print a warning.
248 if self._emptyFlag is None:
249 self._emptyFlag = self._attributes.empty()
250 if self._emptyFlag:
251 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
252 return self._emptyFlag
254 def checkManagersConfig(self) -> None:
255 """Compare configured manager names with stored in database.
257 Raises
258 ------
259 ManagerMismatchError
260 Raised if manager names are different.
261 MissingManagerError
262 Raised if database has no stored manager name.
263 """
264 if self._attributesEmpty:
265 return
267 missing = []
268 mismatch = []
269 for name, extension in self._managers.items():
270 key = self._managerConfigKey(name)
271 storedMgr = self._attributes.get(key)
272 _LOG.debug("found manager config %s=%s", key, storedMgr)
273 if storedMgr is None:
274 missing.append(name)
275 continue
276 if extension.extensionName() != storedMgr:
277 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}")
278 if missing:
279 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
280 if mismatch:
281 raise ManagerMismatchError(
282 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
283 )
285 def checkManagersVersions(self, writeable: bool) -> None:
286 """Compare configured versions with the versions stored in database.
288 Parameters
289 ----------
290 writeable : `bool`
291 If ``True`` then read-write access needs to be checked.
293 Raises
294 ------
295 IncompatibleVersionError
296 Raised if versions are not compatible.
297 MissingVersionError
298 Raised if database has no stored version for one or more groups.
299 """
300 if self._attributesEmpty:
301 return
303 for extension in self._managers.values():
304 version = extension.currentVersion()
305 if version:
306 key = self._managerVersionKey(extension)
307 storedVersionStr = self._attributes.get(key)
308 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
309 if storedVersionStr is None:
310 raise MissingVersionError(f"Failed to read version number {key}")
311 storedVersion = VersionTuple.fromString(storedVersionStr)
312 if not self.checkCompatibility(storedVersion, version, writeable):
313 raise IncompatibleVersionError(
314 f"Configured version {version} is not compatible with stored version "
315 f"{storedVersion} for extension {extension.extensionName()}"
316 )