Coverage for python/lsst/daf/butler/registry/versions.py: 28%
90 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = [
31 "ButlerVersionsManager",
32 "IncompatibleVersionError",
33 "MissingManagerError",
34 "ManagerMismatchError",
35]
37import logging
38from collections.abc import Mapping
39from typing import TYPE_CHECKING
41from .interfaces import VersionedExtension, VersionTuple
43if TYPE_CHECKING:
44 from .interfaces import ButlerAttributeManager
47_LOG = logging.getLogger(__name__)
50class IncompatibleVersionError(RuntimeError):
51 """Exception raised when configured version number is not compatible with
52 database version.
53 """
55 pass
58class MissingManagerError(RuntimeError):
59 """Exception raised when manager name is missing from registry."""
61 pass
64class ManagerMismatchError(RuntimeError):
65 """Exception raised when configured manager name does not match name
66 stored in the database.
67 """
69 pass
72class ButlerVersionsManager:
73 """Utility class to manage and verify schema version compatibility.
75 Parameters
76 ----------
77 attributes : `ButlerAttributeManager`
78 Attribute manager instance.
79 managers : `dict` [`str`, `VersionedExtension`]
80 Mapping of extension type as defined in configuration (e.g.
81 "collections") to corresponding instance of manager.
82 """
84 def __init__(self, attributes: ButlerAttributeManager):
85 self._attributes = attributes
86 # Maps manager type to its class name and schema version.
87 self._cache: Mapping[str, tuple[str, VersionTuple | None]] | None = None
88 self._emptyFlag: bool | None = None
90 @classmethod
91 def _managerConfigKey(cls, name: str) -> str:
92 """Return key used to store manager config.
94 Parameters
95 ----------
96 name : `str`
97 Name of the namager type, e.g. "dimensions"
99 Returns
100 -------
101 key : `str`
102 Name of the key in attributes table.
103 """
104 return f"config:registry.managers.{name}"
106 @classmethod
107 def _managerVersionKey(cls, extensionName: str) -> str:
108 """Return key used to store manager version.
110 Parameters
111 ----------
112 extensionName : `str`
113 Extension name (e.g. its class name).
115 Returns
116 -------
117 key : `str`
118 Name of the key in attributes table.
119 """
120 return f"version:{extensionName}"
122 @property
123 def _manager_data(self) -> Mapping[str, tuple[str, VersionTuple | None]]:
124 """Retrieve per-manager type name and schema version."""
125 if not self._cache:
126 self._cache = {}
128 # Number of items in attributes table is small, read all of them
129 # in a single query and filter later.
130 attributes = dict(self._attributes.items())
132 for name, value in attributes.items():
133 if name.startswith("config:registry.managers."):
134 _, _, manager_type = name.rpartition(".")
135 manager_class = value
136 version_str = attributes.get(self._managerVersionKey(manager_class))
137 if version_str is None:
138 self._cache[manager_type] = (manager_class, None)
139 else:
140 version = VersionTuple.fromString(version_str)
141 self._cache[manager_type] = (manager_class, version)
143 return self._cache
145 @staticmethod
146 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
147 """Compare two versions for compatibility.
149 Parameters
150 ----------
151 old_version : `VersionTuple`
152 Old schema version, typically one stored in a database.
153 new_version : `VersionTuple`
154 New schema version, typically version defined in configuration.
155 update : `bool`
156 If True then read-write access is expected.
157 """
158 if old_version.major != new_version.major:
159 # different major versions are not compatible at all
160 return False
161 if old_version.minor != new_version.minor:
162 # different minor versions are backward compatible for read
163 # access only
164 return new_version.minor > old_version.minor and not update
165 # patch difference does not matter
166 return True
168 def storeManagersConfig(self, managers: Mapping[str, VersionedExtension]) -> None:
169 """Store configured extension names and their versions.
171 Parmeters
172 ---------
173 managers: `~collections.abc.Mapping` [`str`, `type`]
174 Collection of manager extension classes, the key is a manager type,
175 e.g. "datasets".
177 Notes
178 -----
179 For each extension we store two records:
180 - with the key "config:registry.managers.{name}" and fully qualified
181 class name as a value,
182 - with the key "version:{fullExtensionName}" and version number in its
183 string format as a value.
184 """
185 for name, extension in managers.items():
186 key = self._managerConfigKey(name)
187 value = extension.extensionName()
188 self._attributes.set(key, value)
189 _LOG.debug("saved manager config %s=%s", key, value)
191 version = extension.newSchemaVersion()
192 if version:
193 key = self._managerVersionKey(extension.extensionName())
194 value = str(version)
195 self._attributes.set(key, value)
196 _LOG.debug("saved manager version %s=%s", key, value)
198 @property
199 def _attributesEmpty(self) -> bool:
200 """True if attributes table is empty."""
201 # There are existing repositories where attributes table was not
202 # filled, we don't want to force schema migration in this case yet
203 # (and we don't have tools) so we allow this as valid use case and
204 # skip all checks but print a warning.
205 if self._emptyFlag is None:
206 self._emptyFlag = self._attributes.empty()
207 if self._emptyFlag:
208 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
209 return self._emptyFlag
211 def checkManagersConfig(self, managers: Mapping[str, type[VersionedExtension]]) -> None:
212 """Compare configured manager names versions with stored in database.
214 Raises
215 ------
216 ManagerMismatchError
217 Raised if manager names are different.
218 MissingManagerError
219 Raised if database has no stored manager name.
220 IncompatibleVersionError
221 Raised if versions are not compatible.
222 """
223 if self._attributesEmpty:
224 return
226 manager_data = self._manager_data
228 missing = []
229 mismatch = []
230 for name, extension in managers.items():
231 try:
232 manager_class, _ = manager_data[name]
233 _LOG.debug("found manager config %s=%s", name, manager_class)
234 except KeyError:
235 missing.append(name)
236 continue
237 if extension.extensionName() != manager_class:
238 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {manager_class}")
239 if missing:
240 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
241 if mismatch:
242 raise ManagerMismatchError(
243 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
244 )
246 def managerVersions(self) -> Mapping[str, VersionTuple]:
247 """Return schema versions for each manager.
249 Returns
250 -------
251 versions : `~collections.abc.Mapping` [`str`, `VersionTuple`]
252 Mapping of managert type (e.g. "datasets") to its schema version.
253 """
254 versions = {}
255 for manager_type, (_, version) in self._manager_data.items():
256 if version is not None:
257 versions[manager_type] = version
258 return versions