Coverage for python/lsst/daf/butler/registry/versions.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "ButlerVersionsManager",
26 "IncompatibleVersionError",
27 "MissingVersionError",
28 "MissingManagerError",
29 "ManagerMismatchError",
30 "DigestMismatchError",
31]
33import logging
34from typing import (
35 Any,
36 Mapping,
37 MutableMapping,
38 Optional,
39 TYPE_CHECKING,
40)
42from .interfaces import VersionTuple, VersionedExtension
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 from .interfaces import (
46 ButlerAttributeManager,
47 )
50_LOG = logging.getLogger(__name__)
53class MissingVersionError(RuntimeError):
54 """Exception raised when existing database is missing attributes with
55 version numbers.
56 """
57 pass
60class IncompatibleVersionError(RuntimeError):
61 """Exception raised when configured version number is not compatible with
62 database version.
63 """
64 pass
67class MissingManagerError(RuntimeError):
68 """Exception raised when manager name is missing from registry.
69 """
70 pass
73class ManagerMismatchError(RuntimeError):
74 """Exception raised when configured manager name does not match name
75 stored in the database.
76 """
77 pass
80class DigestMismatchError(RuntimeError):
81 """Exception raised when schema digest is not equal to stored digest.
82 """
83 pass
86class VersionInfo:
87 """Representation of version information as defined by configuration.
89 Parameters
90 ----------
91 version : `VersionTuple`
92 Version number in parsed format.
93 digest : `str`, optional
94 Optional digest of the corresponding part of the schema definition.
96 Notes
97 -----
98 Schema digest is supposed to help with detecting unintentional schema
99 changes in the code without upgrading schema version. Digest is
100 constructed whom the set of table definitions and is compared to a digest
101 defined in configuration, if two digests differ it means schema was
102 changed. Intentional schema updates will need to update both configured
103 schema version and schema digest.
104 """
105 def __init__(self, version: VersionTuple, digest: Optional[str] = None):
106 self.version = version
107 self.digest = digest
110class ButlerVersionsManager:
111 """Utility class to manage and verify schema version compatibility.
113 Parameters
114 ----------
115 attributes : `ButlerAttributeManager`
116 Attribute manager instance.
117 managers : `dict` [`str`, `VersionedExtension`]
118 Mapping of extension type as defined in configuration (e.g.
119 "collections") to corresponding instance of manager.
120 """
121 def __init__(self, attributes: ButlerAttributeManager,
122 managers: Mapping[str, Any]):
123 self._attributes = attributes
124 self._managers: MutableMapping[str, VersionedExtension] = {}
125 # we only care about managers implementing VersionedExtension interface
126 for name, manager in managers.items():
127 if isinstance(manager, VersionedExtension):
128 self._managers[name] = manager
129 else:
130 # All regular managers need to support versioning mechanism.
131 _LOG.warning("extension %r does not implement VersionedExtension", name)
132 self._emptyFlag: Optional[bool] = None
134 @classmethod
135 def _managerConfigKey(cls, name: str) -> str:
136 """Return key used to store manager config.
138 Parameters
139 ----------
140 name : `str`
141 Name of the namager type, e.g. "dimensions"
143 Returns
144 -------
145 key : `str`
146 Name of the key in attributes table.
147 """
148 return f"config:registry.managers.{name}"
150 @classmethod
151 def _managerVersionKey(cls, extension: VersionedExtension) -> str:
152 """Return key used to store manager version.
154 Parameters
155 ----------
156 extension : `VersionedExtension`
157 Instance of the extension.
159 Returns
160 -------
161 key : `str`
162 Name of the key in attributes table.
163 """
164 return "version:" + extension.extensionName()
166 @classmethod
167 def _managerDigestKey(cls, extension: VersionedExtension) -> str:
168 """Return key used to store manager schema digest.
170 Parameters
171 ----------
172 extension : `VersionedExtension`
173 Instance of the extension.
175 Returns
176 -------
177 key : `str`
178 Name of the key in attributes table.
179 """
180 return "schema_digest:" + extension.extensionName()
182 @staticmethod
183 def checkCompatibility(old_version: VersionTuple, new_version: VersionTuple, update: bool) -> bool:
184 """Compare two versions for compatibility.
186 Parameters
187 ----------
188 old_version : `VersionTuple`
189 Old schema version, typically one stored in a database.
190 new_version : `VersionTuple`
191 New schema version, typically version defined in configuration.
192 update : `bool`
193 If True then read-write access is expected.
194 """
195 if old_version.major != new_version.major:
196 # different major versions are not compatible at all
197 return False
198 if old_version.minor != new_version.minor:
199 # different minor versions are backward compatible for read
200 # access only
201 return new_version.minor > old_version.minor and not update
202 # patch difference does not matter
203 return True
205 def storeManagersConfig(self) -> None:
206 """Store configured extension names in attributes table.
208 For each extension we store a record with the key
209 "config:registry.managers.{name}" and fully qualified class name as a
210 value.
211 """
212 for name, extension in self._managers.items():
213 key = self._managerConfigKey(name)
214 value = extension.extensionName()
215 self._attributes.set(key, value)
216 _LOG.debug("saved manager config %s=%s", key, value)
217 self._emptyFlag = False
219 def storeManagersVersions(self) -> None:
220 """Store current manager versions in registry arttributes.
222 For each extension we store two records:
224 - record with the key "version:{fullExtensionName}" and version
225 number in its string format as a value,
226 - record with the key "schema_digest:{fullExtensionName}" and
227 schema digest as a value.
228 """
229 for extension in self._managers.values():
231 version = extension.currentVersion()
232 if version:
233 key = self._managerVersionKey(extension)
234 value = str(version)
235 self._attributes.set(key, value)
236 _LOG.debug("saved manager version %s=%s", key, value)
238 digest = extension.schemaDigest()
239 if digest is not None:
240 key = self._managerDigestKey(extension)
241 self._attributes.set(key, digest)
242 _LOG.debug("saved manager schema digest %s=%s", key, digest)
244 self._emptyFlag = False
246 @property
247 def _attributesEmpty(self) -> bool:
248 """True if attributes table is empty.
249 """
250 # There are existing repositories where attributes table was not
251 # filled, we don't want to force schema migration in this case yet
252 # (and we don't have tools) so we allow this as valid use case and
253 # skip all checks but print a warning.
254 if self._emptyFlag is None:
255 self._emptyFlag = self._attributes.empty()
256 if self._emptyFlag:
257 _LOG.warning("Attributes table is empty, schema may need an upgrade.")
258 return self._emptyFlag
260 def checkManagersConfig(self) -> None:
261 """Compare configured manager names with stored in database.
263 Raises
264 ------
265 ManagerMismatchError
266 Raised if manager names are different.
267 MissingManagerError
268 Raised if database has no stored manager name.
269 """
270 if self._attributesEmpty:
271 return
273 missing = []
274 mismatch = []
275 for name, extension in self._managers.items():
276 key = self._managerConfigKey(name)
277 storedMgr = self._attributes.get(key)
278 _LOG.debug("found manager config %s=%s", key, storedMgr)
279 if storedMgr is None:
280 missing.append(name)
281 continue
282 if extension.extensionName() != storedMgr:
283 mismatch.append(
284 f"{name}: configured {extension.extensionName()}, stored: {storedMgr}"
285 )
286 if missing:
287 raise MissingManagerError(
288 "Cannot find stored configuration for managers: "
289 + ", ".join(missing)
290 )
291 if mismatch:
292 raise ManagerMismatchError(
293 "Configured managers do not match registry-stored names:\n"
294 + "\n".join(missing)
295 )
297 def checkManagersVersions(self, writeable: bool) -> None:
298 """Compare configured versions with the versions stored in database.
300 Parameters
301 ----------
302 writeable : `bool`
303 If ``True`` then read-write access needs to be checked.
305 Raises
306 ------
307 IncompatibleVersionError
308 Raised if versions are not compatible.
309 MissingVersionError
310 Raised if database has no stored version for one or more groups.
311 """
312 if self._attributesEmpty:
313 return
315 for extension in self._managers.values():
316 version = extension.currentVersion()
317 if version:
318 key = self._managerVersionKey(extension)
319 storedVersionStr = self._attributes.get(key)
320 _LOG.debug("found manager version %s=%s, current version %s", key, storedVersionStr, version)
321 if storedVersionStr is None:
322 raise MissingVersionError(f"Failed to read version number {key}")
323 storedVersion = VersionTuple.fromString(storedVersionStr)
324 if not self.checkCompatibility(storedVersion, version, writeable):
325 raise IncompatibleVersionError(
326 f"Configured version {version} is not compatible with stored version "
327 f"{storedVersion} for extension {extension.extensionName()}"
328 )
330 def checkManagersDigests(self) -> None:
331 """Compare current schema digests with digests stored in database.
333 Raises
334 ------
335 DigestMismatchError
336 Raised if digests are not equal.
337 """
338 if self._attributesEmpty:
339 return
341 for extension in self._managers.values():
342 digest = extension.schemaDigest()
343 if digest is not None:
344 key = self._managerDigestKey(extension)
345 storedDigest = self._attributes.get(key)
346 _LOG.debug("found manager schema digest %s=%s, current digest %s", key, storedDigest, digest)
347 if storedDigest != digest:
348 raise DigestMismatchError(
349 f"Current schema digest '{digest}' is not the same as stored digest "
350 f"'{storedDigest}' for extension {extension.extensionName()}"
351 )