Coverage for python/lsst/daf/butler/registry/versions.py: 22%
121 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-01 10:04 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-01 10:04 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30 "DigestMismatchError",
31]
33import logging
34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional
36from deprecated.sphinx import deprecated
38from .interfaces import VersionedExtension, VersionTuple
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from .interfaces import ButlerAttributeManager
44_LOG = logging.getLogger(__name__)
47class MissingVersionError(RuntimeError):
48 """Exception raised when existing database is missing attributes with
49 version numbers.
50 """
52 pass
55class IncompatibleVersionError(RuntimeError):
56 """Exception raised when configured version number is not compatible with
57 database version.
58 """
60 pass
63class MissingManagerError(RuntimeError):
64 """Exception raised when manager name is missing from registry."""
66 pass
69class ManagerMismatchError(RuntimeError):
70 """Exception raised when configured manager name does not match name
71 stored in the database.
72 """
74 pass
77class DigestMismatchError(RuntimeError):
78 """Exception raised when schema digest is not equal to stored digest."""
80 pass
83class VersionInfo:
84 """Representation of version information as defined by configuration.
86 Parameters
87 ----------
88 version : `VersionTuple`
89 Version number in parsed format.
90 digest : `str`, optional
91 Optional digest of the corresponding part of the schema definition.
93 Notes
94 -----
95 Schema digest is supposed to help with detecting unintentional schema
96 changes in the code without upgrading schema version. Digest is
97 constructed whom the set of table definitions and is compared to a digest
98 defined in configuration, if two digests differ it means schema was
99 changed. Intentional schema updates will need to update both configured
100 schema version and schema digest.
101 """
103 def __init__(self, version: VersionTuple, digest: Optional[str] = None):
104 self.version = version
105 self.digest = digest
108class ButlerVersionsManager:
109 """Utility class to manage and verify schema version compatibility.
111 Parameters
112 ----------
113 attributes : `ButlerAttributeManager`
114 Attribute manager instance.
115 managers : `dict` [`str`, `VersionedExtension`]
116 Mapping of extension type as defined in configuration (e.g.
117 "collections") to corresponding instance of manager.
118 """
120 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]):
121 self._attributes = attributes
122 self._managers: MutableMapping[str, VersionedExtension] = {}
123 # we only care about managers implementing VersionedExtension interface
124 for name, manager in managers.items():
125 if isinstance(manager, VersionedExtension):
126 self._managers[name] = manager
127 elif manager is not None:
128 # All regular managers need to support versioning mechanism.
129 _LOG.warning("extension %r does not implement VersionedExtension", name)
130 self._emptyFlag: Optional[bool] = None
132 @classmethod
133 def _managerConfigKey(cls, name: str) -> str:
134 """Return key used to store manager config.
136 Parameters
137 ----------
138 name : `str`
139 Name of the namager type, e.g. "dimensions"
141 Returns
142 -------
143 key : `str`
144 Name of the key in attributes table.
145 """
146 return f"config:registry.managers.{name}"
148 @classmethod
149 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
150 """Return key used to store manager version.
152 Parameters
153 ----------
154 extension : `VersionedExtension`
155 Instance of the extension.
157 Returns
158 -------
159 key : `str`
160 Name of the key in attributes table.
161 """
162 return "version:" + extension.extensionName()
164 @classmethod
165 def _managerDigestKey(cls, extension: VersionedExtension) -> str:
166 """Return key used to store manager schema digest.
168 Parameters
169 ----------
170 extension : `VersionedExtension`
171 Instance of the extension.
173 Returns
174 -------
175 key : `str`
176 Name of the key in attributes table.
177 """
178 return "schema_digest:" + extension.extensionName()
180 @staticmethod
181 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
182 """Compare two versions for compatibility.
184 Parameters
185 ----------
186 old_version : `VersionTuple`
187 Old schema version, typically one stored in a database.
188 new_version : `VersionTuple`
189 New schema version, typically version defined in configuration.
190 update : `bool`
191 If True then read-write access is expected.
192 """
193 if old_version.major != new_version.major:
194 # different major versions are not compatible at all
195 return False
196 if old_version.minor != new_version.minor:
197 # different minor versions are backward compatible for read
198 # access only
199 return new_version.minor > old_version.minor and not update
200 # patch difference does not matter
201 return True
203 def storeManagersConfig(self) -> None:
204 """Store configured extension names in attributes table.
206 For each extension we store a record with the key
207 "config:registry.managers.{name}" and fully qualified class name as a
208 value.
209 """
210 for name, extension in self._managers.items():
211 key = self._managerConfigKey(name)
212 value = extension.extensionName()
213 self._attributes.set(key, value)
214 _LOG.debug("saved manager config %s=%s", key, value)
215 self._emptyFlag = False
217 def storeManagersVersions(self) -> None:
218 """Store current manager versions in registry arttributes.
220 For each extension we store two records:
222 - record with the key "version:{fullExtensionName}" and version
223 number in its string format as a value,
224 - record with the key "schema_digest:{fullExtensionName}" and
225 schema digest as a value.
226 """
227 for extension in self._managers.values():
229 version = extension.currentVersion()
230 if version:
231 key = self._managerVersionKey(extension)
232 value = str(version)
233 self._attributes.set(key, value)
234 _LOG.debug("saved manager version %s=%s", key, value)
236 digest = extension.schemaDigest()
237 if digest is not None:
238 key = self._managerDigestKey(extension)
239 self._attributes.set(key, digest)
240 _LOG.debug("saved manager schema digest %s=%s", key, digest)
242 self._emptyFlag = False
244 @property
245 def _attributesEmpty(self) -> bool:
246 """True if attributes table is empty."""
247 # There are existing repositories where attributes table was not
248 # filled, we don't want to force schema migration in this case yet
249 # (and we don't have tools) so we allow this as valid use case and
250 # skip all checks but print a warning.
251 if self._emptyFlag is None:
252 self._emptyFlag = self._attributes.empty()
253 if self._emptyFlag:
254 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
255 return self._emptyFlag
257 def checkManagersConfig(self) -> None:
258 """Compare configured manager names with stored in database.
260 Raises
261 ------
262 ManagerMismatchError
263 Raised if manager names are different.
264 MissingManagerError
265 Raised if database has no stored manager name.
266 """
267 if self._attributesEmpty:
268 return
270 missing = []
271 mismatch = []
272 for name, extension in self._managers.items():
273 key = self._managerConfigKey(name)
274 storedMgr = self._attributes.get(key)
275 _LOG.debug("found manager config %s=%s", key, storedMgr)
276 if storedMgr is None:
277 missing.append(name)
278 continue
279 if extension.extensionName() != storedMgr:
280 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}")
281 if missing:
282 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
283 if mismatch:
284 raise ManagerMismatchError(
285 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
286 )
288 def checkManagersVersions(self, writeable: bool) -> None:
289 """Compare configured versions with the versions stored in database.
291 Parameters
292 ----------
293 writeable : `bool`
294 If ``True`` then read-write access needs to be checked.
296 Raises
297 ------
298 IncompatibleVersionError
299 Raised if versions are not compatible.
300 MissingVersionError
301 Raised if database has no stored version for one or more groups.
302 """
303 if self._attributesEmpty:
304 return
306 for extension in self._managers.values():
307 version = extension.currentVersion()
308 if version:
309 key = self._managerVersionKey(extension)
310 storedVersionStr = self._attributes.get(key)
311 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
312 if storedVersionStr is None:
313 raise MissingVersionError(f"Failed to read version number {key}")
314 storedVersion = VersionTuple.fromString(storedVersionStr)
315 if not self.checkCompatibility(storedVersion, version, writeable):
316 raise IncompatibleVersionError(
317 f"Configured version {version} is not compatible with stored version "
318 f"{storedVersion} for extension {extension.extensionName()}"
319 )
321 @deprecated(reason="Schema checksums are ignored", category=FutureWarning, version="v24.0")
322 def checkManagersDigests(self) -> None:
323 """Compare current schema digests with digests stored in database.
325 Raises
326 ------
327 DigestMismatchError
328 Raised if digests are not equal.
330 Notes
331 -----
332 This method is not used currently and will probably disappear in the
333 future as we remove schema checksums.
334 """
335 if self._attributesEmpty:
336 return
338 for extension in self._managers.values():
339 digest = extension.schemaDigest()
340 if digest is not None:
341 key = self._managerDigestKey(extension)
342 storedDigest = self._attributes.get(key)
343 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest)
344 if storedDigest != digest:
345 raise DigestMismatchError(
346 f"Current schema digest '{digest}' is not the same as stored digest "
347 f"'{storedDigest}' for extension {extension.extensionName()}"
348 )