Coverage for python/lsst/daf/butler/registry/versions.py: 28%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30 "DigestMismatchError",
31]
33import logging
34from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional
36from .interfaces import VersionedExtension, VersionTuple
38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 from .interfaces import ButlerAttributeManager
42_LOG = logging.getLogger(__name__)
45class MissingVersionError(RuntimeError):
46 """Exception raised when existing database is missing attributes with
47 version numbers.
48 """
50 pass
53class IncompatibleVersionError(RuntimeError):
54 """Exception raised when configured version number is not compatible with
55 database version.
56 """
58 pass
61class MissingManagerError(RuntimeError):
62 """Exception raised when manager name is missing from registry."""
64 pass
67class ManagerMismatchError(RuntimeError):
68 """Exception raised when configured manager name does not match name
69 stored in the database.
70 """
72 pass
75class DigestMismatchError(RuntimeError):
76 """Exception raised when schema digest is not equal to stored digest."""
78 pass
81class VersionInfo:
82 """Representation of version information as defined by configuration.
84 Parameters
85 ----------
86 version : `VersionTuple`
87 Version number in parsed format.
88 digest : `str`, optional
89 Optional digest of the corresponding part of the schema definition.
91 Notes
92 -----
93 Schema digest is supposed to help with detecting unintentional schema
94 changes in the code without upgrading schema version. Digest is
95 constructed whom the set of table definitions and is compared to a digest
96 defined in configuration, if two digests differ it means schema was
97 changed. Intentional schema updates will need to update both configured
98 schema version and schema digest.
99 """
101 def __init__(self, version: VersionTuple, digest: Optional[str] = None):
102 self.version = version
103 self.digest = digest
106class ButlerVersionsManager:
107 """Utility class to manage and verify schema version compatibility.
109 Parameters
110 ----------
111 attributes : `ButlerAttributeManager`
112 Attribute manager instance.
113 managers : `dict` [`str`, `VersionedExtension`]
114 Mapping of extension type as defined in configuration (e.g.
115 "collections") to corresponding instance of manager.
116 """
118 def __init__(self, attributes: ButlerAttributeManager, managers: Mapping[str, Any]):
119 self._attributes = attributes
120 self._managers: MutableMapping[str, VersionedExtension] = {}
121 # we only care about managers implementing VersionedExtension interface
122 for name, manager in managers.items():
123 if isinstance(manager, VersionedExtension):
124 self._managers[name] = manager
125 else:
126 # All regular managers need to support versioning mechanism.
127 _LOG.warning("extension %r does not implement VersionedExtension", name)
128 self._emptyFlag: Optional[bool] = None
130 @classmethod
131 def _managerConfigKey(cls, name: str) -> str:
132 """Return key used to store manager config.
134 Parameters
135 ----------
136 name : `str`
137 Name of the namager type, e.g. "dimensions"
139 Returns
140 -------
141 key : `str`
142 Name of the key in attributes table.
143 """
144 return f"config:registry.managers.{name}"
146 @classmethod
147 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
148 """Return key used to store manager version.
150 Parameters
151 ----------
152 extension : `VersionedExtension`
153 Instance of the extension.
155 Returns
156 -------
157 key : `str`
158 Name of the key in attributes table.
159 """
160 return "version:" + extension.extensionName()
162 @classmethod
163 def _managerDigestKey(cls, extension: VersionedExtension) -> str:
164 """Return key used to store manager schema digest.
166 Parameters
167 ----------
168 extension : `VersionedExtension`
169 Instance of the extension.
171 Returns
172 -------
173 key : `str`
174 Name of the key in attributes table.
175 """
176 return "schema_digest:" + extension.extensionName()
178 @staticmethod
179 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
180 """Compare two versions for compatibility.
182 Parameters
183 ----------
184 old_version : `VersionTuple`
185 Old schema version, typically one stored in a database.
186 new_version : `VersionTuple`
187 New schema version, typically version defined in configuration.
188 update : `bool`
189 If True then read-write access is expected.
190 """
191 if old_version.major != new_version.major:
192 # different major versions are not compatible at all
193 return False
194 if old_version.minor != new_version.minor:
195 # different minor versions are backward compatible for read
196 # access only
197 return new_version.minor > old_version.minor and not update
198 # patch difference does not matter
199 return True
201 def storeManagersConfig(self) -> None:
202 """Store configured extension names in attributes table.
204 For each extension we store a record with the key
205 "config:registry.managers.{name}" and fully qualified class name as a
206 value.
207 """
208 for name, extension in self._managers.items():
209 key = self._managerConfigKey(name)
210 value = extension.extensionName()
211 self._attributes.set(key, value)
212 _LOG.debug("saved manager config %s=%s", key, value)
213 self._emptyFlag = False
215 def storeManagersVersions(self) -> None:
216 """Store current manager versions in registry arttributes.
218 For each extension we store two records:
220 - record with the key "version:{fullExtensionName}" and version
221 number in its string format as a value,
222 - record with the key "schema_digest:{fullExtensionName}" and
223 schema digest as a value.
224 """
225 for extension in self._managers.values():
227 version = extension.currentVersion()
228 if version:
229 key = self._managerVersionKey(extension)
230 value = str(version)
231 self._attributes.set(key, value)
232 _LOG.debug("saved manager version %s=%s", key, value)
234 digest = extension.schemaDigest()
235 if digest is not None:
236 key = self._managerDigestKey(extension)
237 self._attributes.set(key, digest)
238 _LOG.debug("saved manager schema digest %s=%s", key, digest)
240 self._emptyFlag = False
242 @property
243 def _attributesEmpty(self) -> bool:
244 """True if attributes table is empty."""
245 # There are existing repositories where attributes table was not
246 # filled, we don't want to force schema migration in this case yet
247 # (and we don't have tools) so we allow this as valid use case and
248 # skip all checks but print a warning.
249 if self._emptyFlag is None:
250 self._emptyFlag = self._attributes.empty()
251 if self._emptyFlag:
252 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
253 return self._emptyFlag
255 def checkManagersConfig(self) -> None:
256 """Compare configured manager names with stored in database.
258 Raises
259 ------
260 ManagerMismatchError
261 Raised if manager names are different.
262 MissingManagerError
263 Raised if database has no stored manager name.
264 """
265 if self._attributesEmpty:
266 return
268 missing = []
269 mismatch = []
270 for name, extension in self._managers.items():
271 key = self._managerConfigKey(name)
272 storedMgr = self._attributes.get(key)
273 _LOG.debug("found manager config %s=%s", key, storedMgr)
274 if storedMgr is None:
275 missing.append(name)
276 continue
277 if extension.extensionName() != storedMgr:
278 mismatch.append(f"{name}: configured {extension.extensionName()}, stored: {storedMgr}")
279 if missing:
280 raise MissingManagerError("Cannot find stored configuration for managers: " + ", ".join(missing))
281 if mismatch:
282 raise ManagerMismatchError(
283 "Configured managers do not match registry-stored names:\n" + "\n".join(missing)
284 )
286 def checkManagersVersions(self, writeable: bool) -> None:
287 """Compare configured versions with the versions stored in database.
289 Parameters
290 ----------
291 writeable : `bool`
292 If ``True`` then read-write access needs to be checked.
294 Raises
295 ------
296 IncompatibleVersionError
297 Raised if versions are not compatible.
298 MissingVersionError
299 Raised if database has no stored version for one or more groups.
300 """
301 if self._attributesEmpty:
302 return
304 for extension in self._managers.values():
305 version = extension.currentVersion()
306 if version:
307 key = self._managerVersionKey(extension)
308 storedVersionStr = self._attributes.get(key)
309 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
310 if storedVersionStr is None:
311 raise MissingVersionError(f"Failed to read version number {key}")
312 storedVersion = VersionTuple.fromString(storedVersionStr)
313 if not self.checkCompatibility(storedVersion, version, writeable):
314 raise IncompatibleVersionError(
315 f"Configured version {version} is not compatible with stored version "
316 f"{storedVersion} for extension {extension.extensionName()}"
317 )
319 def checkManagersDigests(self) -> None:
320 """Compare current schema digests with digests stored in database.
322 Raises
323 ------
324 DigestMismatchError
325 Raised if digests are not equal.
326 """
327 if self._attributesEmpty:
328 return
330 for extension in self._managers.values():
331 digest = extension.schemaDigest()
332 if digest is not None:
333 key = self._managerDigestKey(extension)
334 storedDigest = self._attributes.get(key)
335 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest)
336 if storedDigest != digest:
337 raise DigestMismatchError(
338 f"Current schema digest '{digest}' is not the same as stored digest "
339 f"'{storedDigest}' for extension {extension.extensionName()}"
340 )