Coverage for python/lsst/daf/butler/registry/versions.py: 22%
121 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 00:10 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 00:10 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30 "DigestMismatchError",
31]
33import logging
34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional
36from deprecated.sphinx import deprecated
38from .interfaces import VersionedExtension, VersionTuple
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from .interfaces import ButlerAttributeManager
44_LOG = logging.getLogger(__name__)
47class MissingVersionError(RuntimeError):
48 """Exception raised when existing database is missing attributes with
49 version numbers.
50 """
52 pass
55class IncompatibleVersionError(RuntimeError):
56 """Exception raised when configured version number is not compatible with
57 database version.
58 """
60 pass
63class MissingManagerError(RuntimeError):
64 """Exception raised when manager name is missing from registry."""
66 pass
69class ManagerMismatchError(RuntimeError):
70 """Exception raised when configured manager name does not match name
71 stored in the database.
72 """
74 pass
77class DigestMismatchError(RuntimeError):
78 """Exception raised when schema digest is not equal to stored digest."""
80 pass
83class VersionInfo:
84 """Representation of version information as defined by configuration.
86 Parameters
87 ----------
88 version : `VersionTuple`
89 Version number in parsed format.
90 digest : `str`, optional
91 Optional digest of the corresponding part of the schema definition.
93 Notes
94 -----
95 Schema digest is supposed to help with detecting unintentional schema
96 changes in the code without upgrading schema version. Digest is
97 constructed whom the set of table definitions and is compared to a digest
98 defined in configuration, if two digests differ it means schema was
99 changed. Intentional schema updates will need to update both configured
100 schema version and schema digest.
101 """
103 def __init__(self, version: VersionTuple, digest: Optional[str] = None):
104 self.version = version
105 self.digest = digest
108class ButlerVersionsManager:
109 """Utility class to manage and verify schema version compatibility.
111 Parameters
112 ----------
113 attributes : `ButlerAttributeManager`
114 Attribute manager instance.
115 managers : `dict` [`str`, `VersionedExtension`]
116 Mapping of extension type as defined in configuration (e.g.
117 "collections") to corresponding instance of manager.
118 """
120 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]):
121 self._attributes = attributes
122 self._managers: MutableMapping[str, VersionedExtension] = {}
123 # we only care about managers implementing VersionedExtension interface
124 for name, manager in managers.items():
125 if isinstance(manager, VersionedExtension):
126 self._managers[name] = manager
127 elif manager is not None:
128 # All regular managers need to support versioning mechanism.
129 _LOG.warning("extension %r does not implement VersionedExtension", name)
130 self._emptyFlag: Optional[bool] = None
132 @classmethod
133 def _managerConfigKey(cls, name: str) -> str:
134 """Return key used to store manager config.
136 Parameters
137 ----------
138 name : `str`
139 Name of the namager type, e.g. "dimensions"
141 Returns
142 -------
143 key : `str`
144 Name of the key in attributes table.
145 """
146 return f"config:registry.managers.{name}"
148 @classmethod
149 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
150 """Return key used to store manager version.
152 Parameters
153 ----------
154 extension : `VersionedExtension`
155 Instance of the extension.
157 Returns
158 -------
159 key : `str`
160 Name of the key in attributes table.
161 """
162 return "version:" + extension.extensionName()
164 @classmethod
165 def _managerDigestKey(cls, extension: VersionedExtension) -> str:
166 """Return key used to store manager schema digest.
168 Parameters
169 ----------
170 extension : `VersionedExtension`
171 Instance of the extension.
173 Returns
174 -------
175 key : `str`
176 Name of the key in attributes table.
177 """
178 return "schema_digest:" + extension.extensionName()
180 @staticmethod
181 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
182 """Compare two versions for compatibility.
184 Parameters
185 ----------
186 old_version : `VersionTuple`
187 Old schema version, typically one stored in a database.
188 new_version : `VersionTuple`
189 New schema version, typically version defined in configuration.
190 update : `bool`
191 If True then read-write access is expected.
192 """
193 if old_version.major != new_version.major:
194 # different major versions are not compatible at all
195 return False
196 if old_version.minor != new_version.minor:
197 # different minor versions are backward compatible for read
198 # access only
199 return new_version.minor > old_version.minor and not update
200 # patch difference does not matter
201 return True
203 def storeManagersConfig(self) -> None:
204 """Store configured extension names in attributes table.
206 For each extension we store a record with the key
207 "config:registry.managers.{name}" and fully qualified class name as a
208 value.
209 """
210 for name, extension in self._managers.items():
211 key = self._managerConfigKey(name)
212 value = extension.extensionName()
213 self._attributes.set(key, value)
214 _LOG.debug("saved manager config %s=%s", key, value)
215 self._emptyFlag = False
217 def storeManagersVersions(self) -> None:
218 """Store current manager versions in registry arttributes.
220 For each extension we store two records:
222 - record with the key "version:{fullExtensionName}" and version
223 number in its string format as a value,
224 - record with the key "schema_digest:{fullExtensionName}" and
225 schema digest as a value.
226 """
227 for extension in self._managers.values():
228 version = extension.currentVersion()
229 if version:
230 key = self._managerVersionKey(extension)
231 value = str(version)
232 self._attributes.set(key, value)
233 _LOG.debug("saved manager version %s=%s", key, value)
235 digest = extension.schemaDigest()
236 if digest is not None:
237 key = self._managerDigestKey(extension)
238 self._attributes.set(key, digest)
239 _LOG.debug("saved manager schema digest %s=%s", key, digest)
241 self._emptyFlag = False
243 @property
244 def _attributesEmpty(self) -> bool:
245 """True if attributes table is empty."""
246 # There are existing repositories where attributes table was not
247 # filled, we don't want to force schema migration in this case yet
248 # (and we don't have tools) so we allow this as valid use case and
249 # skip all checks but print a warning.
250 if self._emptyFlag is None:
251 self._emptyFlag = self._attributes.empty()
252 if self._emptyFlag:
253 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
254 return self._emptyFlag
256 def checkManagersConfig(self) -> None:
257 """Compare configured manager names with stored in database.
259 Raises
260 ------
261 ManagerMismatchError
262 Raised if manager names are different.
263 MissingManagerError
264 Raised if database has no stored manager name.
265 """
266 if self._attributesEmpty:
267 return
269 missing = []
270 mismatch = []
271 for name, extension in self._managers.items():
272 key = self._managerConfigKey(name)
273 storedMgr = self._attributes.get(key)
274 _LOG.debug("found manager config %s=%s", key, storedMgr)
275 if storedMgr is None:
276 missing.append(name)
277 continue
278 if extension.extensionName() != storedMgr:
279 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}")
280 if missing:
281 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
282 if mismatch:
283 raise ManagerMismatchError(
284 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
285 )
287 def checkManagersVersions(self, writeable: bool) -> None:
288 """Compare configured versions with the versions stored in database.
290 Parameters
291 ----------
292 writeable : `bool`
293 If ``True`` then read-write access needs to be checked.
295 Raises
296 ------
297 IncompatibleVersionError
298 Raised if versions are not compatible.
299 MissingVersionError
300 Raised if database has no stored version for one or more groups.
301 """
302 if self._attributesEmpty:
303 return
305 for extension in self._managers.values():
306 version = extension.currentVersion()
307 if version:
308 key = self._managerVersionKey(extension)
309 storedVersionStr = self._attributes.get(key)
310 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
311 if storedVersionStr is None:
312 raise MissingVersionError(f"Failed to read version number {key}")
313 storedVersion = VersionTuple.fromString(storedVersionStr)
314 if not self.checkCompatibility(storedVersion, version, writeable):
315 raise IncompatibleVersionError(
316 f"Configured version {version} is not compatible with stored version "
317 f"{storedVersion} for extension {extension.extensionName()}"
318 )
320 @deprecated(reason="Schema checksums are ignored", category=FutureWarning, version="v24.0")
321 def checkManagersDigests(self) -> None:
322 """Compare current schema digests with digests stored in database.
324 Raises
325 ------
326 DigestMismatchError
327 Raised if digests are not equal.
329 Notes
330 -----
331 This method is not used currently and will probably disappear in the
332 future as we remove schema checksums.
333 """
334 if self._attributesEmpty:
335 return
337 for extension in self._managers.values():
338 digest = extension.schemaDigest()
339 if digest is not None:
340 key = self._managerDigestKey(extension)
341 storedDigest = self._attributes.get(key)
342 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest)
343 if storedDigest != digest:
344 raise DigestMismatchError(
345 f"Current schema digest '{digest}' is not the same as stored digest "
346 f"'{storedDigest}' for extension {extension.extensionName()}"
347 )