Coverage for python/lsst/daf/butler/registry/versions.py: 22%
90 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 09:50 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 09:50 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingManagerError",
28 "ManagerMismatchError",
29]
31import logging
32from collections.abc import Mapping
33from typing import TYPE_CHECKING
35from .interfaces import VersionedExtension, VersionTuple
37if TYPE_CHECKING:
38 from .interfaces import ButlerAttributeManager
41_LOG = logging.getLogger(__name__)
44class IncompatibleVersionError(RuntimeError):
45 """Exception raised when configured version number is not compatible with
46 database version.
47 """
49 pass
52class MissingManagerError(RuntimeError):
53 """Exception raised when manager name is missing from registry."""
55 pass
58class ManagerMismatchError(RuntimeError):
59 """Exception raised when configured manager name does not match name
60 stored in the database.
61 """
63 pass
66class ButlerVersionsManager:
67 """Utility class to manage and verify schema version compatibility.
69 Parameters
70 ----------
71 attributes : `ButlerAttributeManager`
72 Attribute manager instance.
73 managers : `dict` [`str`, `VersionedExtension`]
74 Mapping of extension type as defined in configuration (e.g.
75 "collections") to corresponding instance of manager.
76 """
78 def __init__(self, attributes: ButlerAttributeManager):
79 self._attributes = attributes
80 # Maps manager type to its class name and schema version.
81 self._cache: Mapping[str, tuple[str, VersionTuple | None]] | None = None
82 self._emptyFlag: bool | None = None
84 @classmethod
85 def _managerConfigKey(cls, name: str) -> str:
86 """Return key used to store manager config.
88 Parameters
89 ----------
90 name : `str`
91 Name of the namager type, e.g. "dimensions"
93 Returns
94 -------
95 key : `str`
96 Name of the key in attributes table.
97 """
98 return f"config:registry.managers.{name}"
100 @classmethod
101 def _managerVersionKey(cls, extensionName: str) -> str:
102 """Return key used to store manager version.
104 Parameters
105 ----------
106 extensionName : `str`
107 Extension name (e.g. its class name).
109 Returns
110 -------
111 key : `str`
112 Name of the key in attributes table.
113 """
114 return f"version:{extensionName}"
116 @property
117 def _manager_data(self) -> Mapping[str, tuple[str, VersionTuple | None]]:
118 """Retrieve per-manager type name and schema version."""
119 if not self._cache:
120 self._cache = {}
122 # Number of items in attributes table is small, read all of them
123 # in a single query and filter later.
124 attributes = {key: value for key, value in self._attributes.items()}
126 for name, value in attributes.items():
127 if name.startswith("config:registry.managers."):
128 _, _, manager_type = name.rpartition(".")
129 manager_class = value
130 version_str = attributes.get(self._managerVersionKey(manager_class))
131 if version_str is None:
132 self._cache[manager_type] = (manager_class, None)
133 else:
134 version = VersionTuple.fromString(version_str)
135 self._cache[manager_type] = (manager_class, version)
137 return self._cache
139 @staticmethod
140 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
141 """Compare two versions for compatibility.
143 Parameters
144 ----------
145 old_version : `VersionTuple`
146 Old schema version, typically one stored in a database.
147 new_version : `VersionTuple`
148 New schema version, typically version defined in configuration.
149 update : `bool`
150 If True then read-write access is expected.
151 """
152 if old_version.major != new_version.major:
153 # different major versions are not compatible at all
154 return False
155 if old_version.minor != new_version.minor:
156 # different minor versions are backward compatible for read
157 # access only
158 return new_version.minor > old_version.minor and not update
159 # patch difference does not matter
160 return True
162 def storeManagersConfig(self, managers: Mapping[str, VersionedExtension]) -> None:
163 """Store configured extension names and their versions.
165 Parmeters
166 ---------
167 managers: `~collections.abc.Mapping` [`str`, `type`]
168 Collection of manager extension classes, the key is a manager type,
169 e.g. "datasets".
171 Notes
172 -----
173 For each extension we store two records:
174 - with the key "config:registry.managers.{name}" and fully qualified
175 class name as a value,
176 - with the key "version:{fullExtensionName}" and version number in its
177 string format as a value.
178 """
179 for name, extension in managers.items():
180 key = self._managerConfigKey(name)
181 value = extension.extensionName()
182 self._attributes.set(key, value)
183 _LOG.debug("saved manager config %s=%s", key, value)
185 version = extension.newSchemaVersion()
186 if version:
187 key = self._managerVersionKey(extension.extensionName())
188 value = str(version)
189 self._attributes.set(key, value)
190 _LOG.debug("saved manager version %s=%s", key, value)
192 @property
193 def _attributesEmpty(self) -> bool:
194 """True if attributes table is empty."""
195 # There are existing repositories where attributes table was not
196 # filled, we don't want to force schema migration in this case yet
197 # (and we don't have tools) so we allow this as valid use case and
198 # skip all checks but print a warning.
199 if self._emptyFlag is None:
200 self._emptyFlag = self._attributes.empty()
201 if self._emptyFlag:
202 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
203 return self._emptyFlag
205 def checkManagersConfig(self, managers: Mapping[str, type[VersionedExtension]]) -> None:
206 """Compare configured manager names versions with stored in database.
208 Raises
209 ------
210 ManagerMismatchError
211 Raised if manager names are different.
212 MissingManagerError
213 Raised if database has no stored manager name.
214 IncompatibleVersionError
215 Raised if versions are not compatible.
216 """
217 if self._attributesEmpty:
218 return
220 manager_data = self._manager_data
222 missing = []
223 mismatch = []
224 for name, extension in managers.items():
225 try:
226 manager_class, _ = manager_data[name]
227 _LOG.debug("found manager config %s=%s", name, manager_class)
228 except KeyError:
229 missing.append(name)
230 continue
231 if extension.extensionName() != manager_class:
232 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {manager_class}")
233 if missing:
234 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
235 if mismatch:
236 raise ManagerMismatchError(
237 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
238 )
240 def managerVersions(self) -> Mapping[str, VersionTuple]:
241 """Return schema versions for each manager.
243 Returns
244 -------
245 versions : `~collections.abc.Mapping` [`str`, `VersionTuple`]
246 Mapping of managert type (e.g. "datasets") to its schema version.
247 """
248 versions = {}
249 for manager_type, (_, version) in self._manager_data.items():
250 if version is not None:
251 versions[manager_type] = version
252 return versions