Coverage for python/lsst/daf/butler/registry/versions.py: 22%
95 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-25 02:06 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-25 02:06 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30]
32import logging
33from collections.abc import Mapping, MutableMapping
34from typing import TYPE_CHECKING, Any
36from .interfaces import VersionedExtension, VersionTuple
38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 from .interfaces import ButlerAttributeManager
42_LOG = logging.getLogger(__name__)
45class MissingVersionError(RuntimeError):
46 """Exception raised when existing database is missing attributes with
47 version numbers.
48 """
50 pass
53class IncompatibleVersionError(RuntimeError):
54 """Exception raised when configured version number is not compatible with
55 database version.
56 """
58 pass
61class MissingManagerError(RuntimeError):
62 """Exception raised when manager name is missing from registry."""
64 pass
67class ManagerMismatchError(RuntimeError):
68 """Exception raised when configured manager name does not match name
69 stored in the database.
70 """
72 pass
75class ButlerVersionsManager:
76 """Utility class to manage and verify schema version compatibility.
78 Parameters
79 ----------
80 attributes : `ButlerAttributeManager`
81 Attribute manager instance.
82 managers : `dict` [`str`, `VersionedExtension`]
83 Mapping of extension type as defined in configuration (e.g.
84 "collections") to corresponding instance of manager.
85 """
87 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]):
88 self._attributes = attributes
89 self._managers: MutableMapping[str, VersionedExtension] = {}
90 # we only care about managers implementing VersionedExtension interface
91 for name, manager in managers.items():
92 if isinstance(manager, VersionedExtension):
93 self._managers[name] = manager
94 elif manager is not None:
95 # All regular managers need to support versioning mechanism.
96 _LOG.warning("extension %r does not implement VersionedExtension", name)
97 self._emptyFlag: bool | None = None
99 @classmethod
100 def _managerConfigKey(cls, name: str) -> str:
101 """Return key used to store manager config.
103 Parameters
104 ----------
105 name : `str`
106 Name of the namager type, e.g. "dimensions"
108 Returns
109 -------
110 key : `str`
111 Name of the key in attributes table.
112 """
113 return f"config:registry.managers.{name}"
115 @classmethod
116 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
117 """Return key used to store manager version.
119 Parameters
120 ----------
121 extension : `VersionedExtension`
122 Instance of the extension.
124 Returns
125 -------
126 key : `str`
127 Name of the key in attributes table.
128 """
129 return "version:" + extension.extensionName()
131 @staticmethod
132 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
133 """Compare two versions for compatibility.
135 Parameters
136 ----------
137 old_version : `VersionTuple`
138 Old schema version, typically one stored in a database.
139 new_version : `VersionTuple`
140 New schema version, typically version defined in configuration.
141 update : `bool`
142 If True then read-write access is expected.
143 """
144 if old_version.major != new_version.major:
145 # different major versions are not compatible at all
146 return False
147 if old_version.minor != new_version.minor:
148 # different minor versions are backward compatible for read
149 # access only
150 return new_version.minor > old_version.minor and not update
151 # patch difference does not matter
152 return True
154 def storeManagersConfig(self) -> None:
155 """Store configured extension names in attributes table.
157 For each extension we store a record with the key
158 "config:registry.managers.{name}" and fully qualified class name as a
159 value.
160 """
161 for name, extension in self._managers.items():
162 key = self._managerConfigKey(name)
163 value = extension.extensionName()
164 self._attributes.set(key, value)
165 _LOG.debug("saved manager config %s=%s", key, value)
166 self._emptyFlag = False
168 def storeManagersVersions(self) -> None:
169 """Store current manager versions in registry attributes.
171 For each extension we store single record with a key
172 "version:{fullExtensionName}" and version number in its string format
173 as a value.
174 """
175 for extension in self._managers.values():
176 version = extension.currentVersion()
177 if version:
178 key = self._managerVersionKey(extension)
179 value = str(version)
180 self._attributes.set(key, value)
181 _LOG.debug("saved manager version %s=%s", key, value)
183 self._emptyFlag = False
185 @property
186 def _attributesEmpty(self) -> bool:
187 """True if attributes table is empty."""
188 # There are existing repositories where attributes table was not
189 # filled, we don't want to force schema migration in this case yet
190 # (and we don't have tools) so we allow this as valid use case and
191 # skip all checks but print a warning.
192 if self._emptyFlag is None:
193 self._emptyFlag = self._attributes.empty()
194 if self._emptyFlag:
195 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
196 return self._emptyFlag
198 def checkManagersConfig(self) -> None:
199 """Compare configured manager names with stored in database.
201 Raises
202 ------
203 ManagerMismatchError
204 Raised if manager names are different.
205 MissingManagerError
206 Raised if database has no stored manager name.
207 """
208 if self._attributesEmpty:
209 return
211 missing = []
212 mismatch = []
213 for name, extension in self._managers.items():
214 key = self._managerConfigKey(name)
215 storedMgr = self._attributes.get(key)
216 _LOG.debug("found manager config %s=%s", key, storedMgr)
217 if storedMgr is None:
218 missing.append(name)
219 continue
220 if extension.extensionName() != storedMgr:
221 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}")
222 if missing:
223 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
224 if mismatch:
225 raise ManagerMismatchError(
226 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
227 )
229 def checkManagersVersions(self, writeable: bool) -> None:
230 """Compare configured versions with the versions stored in database.
232 Parameters
233 ----------
234 writeable : `bool`
235 If ``True`` then read-write access needs to be checked.
237 Raises
238 ------
239 IncompatibleVersionError
240 Raised if versions are not compatible.
241 MissingVersionError
242 Raised if database has no stored version for one or more groups.
243 """
244 if self._attributesEmpty:
245 return
247 for extension in self._managers.values():
248 version = extension.currentVersion()
249 if version:
250 key = self._managerVersionKey(extension)
251 storedVersionStr = self._attributes.get(key)
252 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
253 if storedVersionStr is None:
254 raise MissingVersionError(f"Failed to read version number {key}")
255 storedVersion = VersionTuple.fromString(storedVersionStr)
256 if not self.checkCompatibility(storedVersion, version, writeable):
257 raise IncompatibleVersionError(
258 f"Configured version {version} is not compatible with stored version "
259 f"{storedVersion} for extension {extension.extensionName()}"
260 )