Coverage for python/lsst/images/serialization/_frozen_schemas.py: 82%
81 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-22 17:43 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11"""Frozen JSON schema files for the serialization data models.
13Every `~lsst.images.serialization.ArchiveTree` subclass has a canonical JSON
14Schema derived from its pydantic model. These are written to git-committed
15``schemas/`` files so the published schema at
16``https://images.lsst.io/schemas/{name}-{version}`` is a stable artifact
17rather than whatever the code currently produces, and so superseded versions
18remain available after the models move on.
19"""
21from __future__ import annotations
23__all__ = (
24 "FrozenSchemaError",
25 "available_schema_classes",
26 "check_frozen_schemas",
27 "dump_schema",
28 "frozen_schema_filename",
29 "frozen_schema_path",
30 "write_frozen_schemas",
31)
33import importlib.metadata
34import json
35import re
36from pathlib import Path
37from typing import Any
39from ._asdf_utils import ArrayReferenceModel
40from ._common import ArchiveTree, is_development_version
41from ._io import (
42 _BUILTIN_SCHEMA_PROVIDERS,
43 _REGISTRY,
44 _SCHEMA_ENTRY_POINT_GROUP,
45 class_for_schema,
46 parameterize_tree,
47)
50class FrozenSchemaError(RuntimeError):
51 """A finalized frozen schema would change without a version bump."""
54def available_schema_classes(package: str = "lsst.images") -> list[type[ArchiveTree]]:
55 """Return every `~lsst.images.serialization.ArchiveTree` subclass owned
56 by ``package``, sorted by schema name.
58 Parameters
59 ----------
60 package
61 Only classes whose defining module is this package (or a
62 subpackage of it) are returned, so a package freezes and publishes
63 exactly the schemas it owns, and schema classes created elsewhere
64 (e.g. test doubles) are never picked up by accident.
66 Notes
67 -----
68 Candidate schemas come from the in-memory registry, the built-in lazy
69 providers, and the ``lsst.images.schemas`` entry point group, so an
70 external package's schemas are found even when nothing has imported
71 their modules yet.
72 """
73 entry_point_names = {
74 entry_point.name for entry_point in importlib.metadata.entry_points(group=_SCHEMA_ENTRY_POINT_GROUP)
75 }
76 classes: list[type[ArchiveTree]] = []
77 for name in sorted(set(_REGISTRY) | set(_BUILTIN_SCHEMA_PROVIDERS) | entry_point_names):
78 cls = class_for_schema(name)
79 if cls is None: 79 ↛ 80line 79 didn't jump to line 80 because the condition on line 79 was never true
80 raise RuntimeError(f"Schema {name!r} is registered but its class could not be loaded.")
81 if cls.__module__ != package and not cls.__module__.startswith(f"{package}."):
82 continue
83 classes.append(cls)
84 return classes
87def _summary_description(text: str) -> str:
88 """Return a docstring-derived description trimmed to its summary.
90 Text is kept up to (but not including) the first numpydoc section header:
91 a non-blank line immediately followed by a line of dashes at least as long
92 as it. Text with no such header (e.g. a one-line ``pydantic.Field``
93 description) is returned unchanged, so only the sectioned content of a
94 class docstring (``Notes``, ``Parameters``, ...) is dropped and the summary
95 plus extended summary are kept.
96 """
97 lines = text.split("\n")
98 for i in range(len(lines) - 1):
99 header = lines[i].strip()
100 underline = lines[i + 1].strip()
101 if header and re.fullmatch(r"-+", underline) and len(underline) >= len(header):
102 return "\n".join(lines[:i]).rstrip()
103 return text
106def _summarize_descriptions(node: Any) -> None:
107 """Trim every ``description`` in a schema tree to its summary, in place."""
108 if isinstance(node, dict):
109 description = node.get("description")
110 if isinstance(description, str):
111 node["description"] = _summary_description(description)
112 for value in node.values():
113 _summarize_descriptions(value)
114 elif isinstance(node, list):
115 for item in node:
116 _summarize_descriptions(item)
119def dump_schema(tree_cls: type[ArchiveTree]) -> dict[str, Any]:
120 """Return the JSON Schema for ``tree_cls``.
122 Parameters
123 ----------
124 tree_cls
125 Serialization model class to dump.
127 Notes
128 -----
129 Generic trees are parameterized over
130 `~lsst.images.serialization.ArrayReferenceModel`, matching the convention
131 used by ``lsst-images-admin diagram``. Model descriptions derived from
132 class docstrings are trimmed to their summary so the numpydoc sections
133 that follow it do not leak into the published schema.
134 """
135 schema = parameterize_tree(tree_cls, ArrayReferenceModel).model_json_schema()
136 # A recursive model (e.g. sum_field) produces a root that is just a $ref
137 # into $defs, with the class's json_schema_extra landing on the $def.
138 # Hoist the canonical identity to the document root so every frozen
139 # document self-identifies; $ref siblings are valid in draft 2020-12.
140 schema.setdefault("$id", f"{tree_cls.SCHEMA_URL_BASE}/{tree_cls.SCHEMA_NAME}-{tree_cls.SCHEMA_VERSION}")
141 schema.setdefault("title", tree_cls.SCHEMA_NAME)
142 # Nested ArchiveTree definitions inherit their class's $id, but $id
143 # starts a new resolution scope in draft 2020-12, which would break the
144 # root-relative "#/$defs/..." references pydantic generates inside them.
145 # Record the canonical URL under a non-reserved key instead, which
146 # validators ignore and documentation tooling can still use to identify
147 # published sub-schemas.
148 for definition in schema.get("$defs", {}).values():
149 if isinstance(definition, dict) and "$id" in definition:
150 definition["x-lsst-schema-url"] = definition.pop("$id")
151 _summarize_descriptions(schema)
152 return schema
155def frozen_schema_filename(tree_cls: type[ArchiveTree]) -> str:
156 """Return the frozen-schema filename for ``tree_cls``.
158 Parameters
159 ----------
160 tree_cls
161 Serialization model class to name the file for.
162 """
163 return f"{tree_cls.SCHEMA_NAME}-{tree_cls.SCHEMA_VERSION}.json"
166def frozen_schema_path(directory: Path, tree_cls: type[ArchiveTree]) -> Path:
167 """Return the frozen-schema file path for ``tree_cls`` under
168 ``directory``.
170 Parameters
171 ----------
172 directory
173 Directory holding the frozen schema files.
174 tree_cls
175 Serialization model class to locate the file for.
177 Notes
178 -----
179 Files are laid out as ``{name}/{name}-{version}.json``: one
180 subdirectory per schema so the directory stays navigable as versions
181 accumulate, with the full name-version filename kept so a file is
182 self-identifying when copied elsewhere.
183 """
184 return directory / tree_cls.SCHEMA_NAME / frozen_schema_filename(tree_cls)
187def _canonical_text(schema: dict[str, Any]) -> str:
188 """Return the canonical file serialization of ``schema``."""
189 return json.dumps(schema, indent=2, sort_keys=True) + "\n"
192def write_frozen_schemas(directory: Path, package: str = "lsst.images") -> list[Path]:
193 """Write the frozen schema file for every current schema.
195 Parameters
196 ----------
197 directory
198 Directory to write the ``{name}-{version}.json`` files into; created
199 if necessary.
200 package
201 Package whose schemas to freeze; see
202 `~lsst.images.serialization.available_schema_classes`.
204 Returns
205 -------
206 `list` [ `pathlib.Path` ]
207 Paths that were created or rewritten.
209 Notes
210 -----
211 Schemas at a development version (a PEP 440 ``.devN`` release) are skipped
212 and never frozen. A finalized schema is frozen only on its first write; an
213 existing frozen file is immutable, so a live-model change to it raises
214 rather than overwriting. Frozen files for superseded versions are never
215 touched, so old schema URLs keep resolving.
217 Raises
218 ------
219 FrozenSchemaError
220 If a finalized schema's frozen file exists and the live model would
221 change its content; bump ``SCHEMA_VERSION`` instead of overwriting.
222 """
223 changed: list[Path] = []
224 for cls in available_schema_classes(package):
225 if is_development_version(cls.SCHEMA_VERSION):
226 continue
227 path = frozen_schema_path(directory, cls)
228 text = _canonical_text(dump_schema(cls))
229 if path.exists():
230 if path.read_text() != text:
231 raise FrozenSchemaError(
232 f"{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION} is finalized and frozen; "
233 "bump SCHEMA_VERSION to change it rather than overwriting the frozen file."
234 )
235 continue
236 path.parent.mkdir(parents=True, exist_ok=True)
237 path.write_text(text)
238 changed.append(path)
239 return changed
242def check_frozen_schemas(directory: Path, package: str = "lsst.images") -> list[str]:
243 """Check the frozen schema files against the current models.
245 Parameters
246 ----------
247 directory
248 Directory holding the frozen ``{name}-{version}.json`` files.
249 package
250 Package whose schemas to check; see
251 `~lsst.images.serialization.available_schema_classes`.
253 Returns
254 -------
255 `list` [ `str` ]
256 One problem description per current schema whose frozen file is
257 missing or does not match the current model; empty when the frozen
258 files are up to date.
260 Notes
261 -----
262 Schemas at a development version (a PEP 440 ``.devN`` release) are
263 skipped and not reported as missing.
264 """
265 problems: list[str] = []
266 for cls in available_schema_classes(package):
267 if is_development_version(cls.SCHEMA_VERSION):
268 continue
269 path = frozen_schema_path(directory, cls)
270 if not path.exists():
271 problems.append(f"{path.relative_to(directory)}: missing")
272 elif path.read_text() != _canonical_text(dump_schema(cls)):
273 problems.append(f"{path.relative_to(directory)}: finalized schema changed; bump SCHEMA_VERSION")
274 return problems