Coverage for python/lsst/images/serialization/_common.py: 54%
126 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-22 17:43 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "SCHEMA_URL_BASE",
16 "ArchiveAccessRequiredError",
17 "ArchiveReadError",
18 "ArchiveTree",
19 "ButlerInfo",
20 "DevelopmentSchemaWarning",
21 "InvalidComponentError",
22 "InvalidParameterError",
23 "JsonRef",
24 "MetadataValue",
25 "OpaqueArchiveMetadata",
26 "is_development_version",
27 "no_header_updates",
28 "warn_for_development_schemas",
29)
31import operator
32import warnings
33from abc import ABC, abstractmethod
34from collections.abc import Iterator
35from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
37import astropy.table
38import astropy.units
39import pydantic
40from packaging.version import Version
42from .._geom import Box
43from ..utils import is_none
45try:
46 from lsst.daf.butler import DatasetProvenance, SerializedDatasetRef
47except ImportError:
48 type DatasetProvenance = Any # type: ignore[no-redef]
49 type SerializedDatasetRef = Any # type: ignore[no-redef]
51if TYPE_CHECKING:
52 import astropy.io.fits
54 from ._input_archive import InputArchive
57type MetadataValue = (
58 pydantic.StrictInt | pydantic.StrictFloat | pydantic.StrictStr | pydantic.StrictBool | None
59)
61SCHEMA_URL_BASE = "https://images.lsst.io/schemas"
62"""Base for the schema URLs of this package's own schemas, used as
63``{SCHEMA_URL_BASE}/{name}-{version}``.
65External packages providing their own schemas override
66`~lsst.images.serialization.ArchiveTree.SCHEMA_URL_BASE` instead, so their
67schema URLs are minted under a site they control.
68"""
71class ButlerInfo(pydantic.BaseModel):
72 """Information about a butler dataset."""
74 dataset: SerializedDatasetRef
75 provenance: DatasetProvenance = pydantic.Field(default_factory=DatasetProvenance)
78class JsonRef(pydantic.BaseModel, serialize_by_alias=True):
79 """Pydantic model for JSON Reference / Pointer (IETF RFC 6901).
81 Notes
82 -----
83 This model does not do any of the escaping or special-character
84 interpretation required by the spec; it assumes that's already been done,
85 so its job is *just* putting a ``$ref`` field inside another model.
86 """
88 ref: str = pydantic.Field(alias="$ref")
91class ArchiveTree(
92 pydantic.BaseModel, ABC, ser_json_inf_nan="constants", ser_json_bytes="base64", val_json_bytes="base64"
93):
94 """An intermediate base class of `pydantic.BaseModel` that should be used
95 for all objects that may be used as the top-level tree models written to
96 archives.
98 See :ref:`lsst.images-schema-versioning` for how the ``SCHEMA_NAME`` /
99 ``SCHEMA_VERSION`` / ``MIN_READ_VERSION`` constants and the
100 ``schema_version`` / ``min_read_version`` / ``schema_url`` fields are used.
101 """
103 SCHEMA_NAME: ClassVar[str]
104 SCHEMA_VERSION: ClassVar[str]
105 MIN_READ_VERSION: ClassVar[int]
106 SCHEMA_URL_BASE: ClassVar[str] = SCHEMA_URL_BASE
107 """Base for this schema's URL, as ``{SCHEMA_URL_BASE}/{name}-{version}``.
109 External packages providing their own schemas should override this (once,
110 on a shared intermediate base class) so their schema URLs are minted
111 under a documentation site they control rather than images.lsst.io."""
113 PUBLIC_TYPE: ClassVar[type]
114 """In-memory Python type produced by this tree's ``deserialize`` (e.g.
115 `dict` for a mapping return). Declared explicitly by each concrete
116 subclass and surfaced by
117 `~lsst.images.serialization.public_type_for_schema`."""
119 schema_version: str = pydantic.Field(
120 default="1.0.0",
121 description="Data-model schema version of this tree (major.minor.patch).",
122 )
123 min_read_version: int = pydantic.Field(
124 default=1,
125 description="Smallest reader major that can interpret this tree.",
126 )
127 metadata: dict[str, MetadataValue] = pydantic.Field(
128 default_factory=dict, description="Additional unstructured metadata.", exclude_if=operator.not_
129 )
130 butler_info: ButlerInfo | None = pydantic.Field(
131 default=None,
132 description="Information about the butler dataset backed by this file.",
133 exclude_if=is_none,
134 )
135 indirect: list[Any] = pydantic.Field(
136 default_factory=list,
137 description="Serialized nested objects that may be saved or read more than once.",
138 exclude_if=operator.not_,
139 )
141 @pydantic.computed_field(description="Canonical schema URL for this tree.") # type: ignore[prop-decorator]
142 @property
143 def schema_url(self) -> str:
144 """Return the schema URL of this tree's class.
146 Computed from ``SCHEMA_NAME`` and ``SCHEMA_VERSION`` ClassVars.
147 """
148 cls = type(self)
149 return f"{cls.SCHEMA_URL_BASE}/{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION}"
151 @pydantic.model_validator(mode="after")
152 def _check_and_normalize_schema_version(self) -> Self:
153 """Validate and normalise the schema version fields.
155 Compares the on-tree ``schema_version`` / ``min_read_version`` against
156 the in-code values from the subclass's ClassVars; raises if
157 incompatible, otherwise normalises the fields to the in-code values.
158 """
159 cls = type(self)
160 # ArchiveTree itself is abstract (deserialize is @abstractmethod).
161 # Subclasses that haven't yet declared SCHEMA_NAME are skipped — this
162 # matters during incremental rollout and remains a safe no-op
163 # afterwards (a class-invariants test ensures every concrete subclass
164 # has the constants).
165 if not hasattr(cls, "SCHEMA_NAME"): 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true
166 return self
167 _check_compat(
168 cls.SCHEMA_NAME,
169 self.schema_version,
170 self.min_read_version,
171 cls.SCHEMA_VERSION,
172 )
173 if self.schema_version != cls.SCHEMA_VERSION:
174 self.schema_version = cls.SCHEMA_VERSION
175 if self.min_read_version != cls.MIN_READ_VERSION: 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true
176 self.min_read_version = cls.MIN_READ_VERSION
177 return self
179 @classmethod
180 def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
181 """Inject ``$id`` and ``title`` into the subclass's JSON Schema, and
182 register the subclass in the schema-name registry.
184 Populates ``model_config['json_schema_extra']`` with values derived
185 from the subclass's ``SCHEMA_NAME`` / ``SCHEMA_VERSION`` ClassVars,
186 then registers the subclass so it can be looked up by schema name.
187 Subclasses that haven't declared the ClassVars are skipped.
188 """
189 super().__pydantic_init_subclass__(**kwargs)
190 name = cls.__dict__.get("SCHEMA_NAME")
191 version = cls.__dict__.get("SCHEMA_VERSION")
192 if name is None or version is None:
193 return
194 json_schema_extra = cls.model_config.get("json_schema_extra") or {}
195 if isinstance(json_schema_extra, dict): 195 ↛ 207line 195 didn't jump to line 207 because the condition on line 195 was always true
196 existing = dict(json_schema_extra)
197 # Always override: a subclass of a concrete schema (e.g.
198 # visit_image subclassing masked_image) inherits its parent's
199 # already-stamped values through the merged model_config, and
200 # this hook only runs when the subclass declares its own
201 # SCHEMA_NAME / SCHEMA_VERSION for these to be derived from.
202 existing["$id"] = f"{cls.SCHEMA_URL_BASE}/{name}-{version}"
203 existing["title"] = name
204 cls.model_config = {**cls.model_config, "json_schema_extra": existing}
205 # Local import to avoid the _io -> _common circular dependency at
206 # module load time.
207 from ._io import register_schema_class
209 register_schema_class(cls)
211 @abstractmethod
212 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Any:
213 """Return the in-memory object that was serialized to this tree.
215 Parameters
216 ----------
217 archive
218 The input archive to read from.
219 **kwargs
220 Additional keyword arguments specific to this type.
222 Raises
223 ------
224 ~lsst.images.serialization.InvalidParameterError
225 Raised for unsupported ``**kwargs``.
227 Notes
228 -----
229 Subclass implementations may take additional keyword-only arguments.
230 Callers that invoke this method without knowing what those might be
231 should catch `TypeError` and re-raise as
232 `~lsst.images.serialization.InvalidParameterError` if they pass
233 additional keyword arguments.
234 """
235 raise NotImplementedError()
237 def deserialize_component(self, component: str, archive: InputArchive[Any], **kwargs: Any) -> Any:
238 """Return a component in-memory object that was serialized to this
239 tree.
241 Parameters
242 ----------
243 component
244 Name of the component to read.
245 archive
246 The input archive to read from.
247 **kwargs
248 Additional keyword arguments specific to this type.
250 Raises
251 ------
252 ~lsst.images.serialization.InvalidComponentError
253 Raise if ``component`` is not recognized.
254 ~lsst.images.serialization.InvalidParameterError
255 Raised for unsupported ``**kwargs``.
257 Notes
258 -----
259 The default implementation for this method tries to get an attribute
260 with the component's name from ``self``, and then:
262 - returns `None` if it is `None`;
263 - calls `deserialize` on that object if it is also an
264 `~lsst.images.serialization.ArchiveTree`;
265 - returns it directly otherwise.
267 If there is no such attribute, it raises
268 `~lsst.images.serialization.InvalidComponentError`.
270 ``**kwargs`` are forwarded to component `deserialize` methods, but
271 are otherwise not checked. Subclasses are generally expected to
272 implement this method to do that checking and handle any components
273 for which the other will not work, and then delegate to `super` at
274 the end.
275 """
276 try:
277 component_model = getattr(self, component)
278 except AttributeError:
279 raise InvalidComponentError(
280 f"Component {component!r} is not recognized by {type(self).__name__}."
281 ) from None
282 if component_model is None:
283 return None
284 if isinstance(component_model, ArchiveTree):
285 return component_model.deserialize(archive, **kwargs)
286 return component_model
289class DevelopmentSchemaWarning(UserWarning):
290 """Warning that a file is being written with a development schema."""
293def is_development_version(version: str) -> bool:
294 """Return whether a schema version string is a PEP 440 development release.
296 Parameters
297 ----------
298 version
299 Schema version string, e.g. ``1.0.0`` or ``1.0.0.dev0``.
300 """
301 return Version(version).is_devrelease
304def _iter_archive_trees(obj: Any) -> Iterator[ArchiveTree]:
305 """Yield every `ArchiveTree` embedded in a serialized tree."""
306 if isinstance(obj, ArchiveTree):
307 yield obj
308 if isinstance(obj, pydantic.BaseModel):
309 for field_name in type(obj).model_fields:
310 yield from _iter_archive_trees(getattr(obj, field_name))
311 elif isinstance(obj, list | tuple):
312 for item in obj:
313 yield from _iter_archive_trees(item)
314 elif isinstance(obj, dict):
315 for value in obj.values():
316 yield from _iter_archive_trees(value)
319def warn_for_development_schemas(root: ArchiveTree) -> None:
320 """Emit a `DevelopmentSchemaWarning` if a serialized tree contains any
321 schema still in development.
323 Parameters
324 ----------
325 root
326 Top-level serialized tree about to be written.
327 """
328 developing = sorted(
329 {
330 tree.schema_url
331 for tree in _iter_archive_trees(root)
332 if is_development_version(type(tree).SCHEMA_VERSION)
333 }
334 )
335 if developing:
336 warnings.warn(
337 "Writing a file with development schema(s) "
338 f"{', '.join(developing)}; such files are not for production and "
339 "may not remain readable.",
340 DevelopmentSchemaWarning,
341 stacklevel=3,
342 )
345class ArchiveReadError(RuntimeError):
346 """Exception raised when the contents of an archive cannot be read."""
349class InvalidParameterError(ArchiveReadError):
350 """Exception raised by `ArchiveTree.deserialize` or
351 `ArchiveTree.deserialize_component` when passed an invalid keyword
352 argument.
353 """
356class InvalidComponentError(ArchiveReadError):
357 """Exception `ArchiveTree.deserialize_component` when passed an invalid
358 component name.
359 """
362class ArchiveAccessRequiredError(RuntimeError):
363 """Exception raised when a deserialization needs data from the file.
365 Raised by all data-access methods of
366 `~lsst.images.serialization.DetachedArchive`, signaling that the
367 requested object cannot be deserialized from the JSON tree alone.
369 Notes
370 -----
371 This deliberately does not inherit from `ArchiveReadError`: it signals
372 that a live archive is required rather than that an archive is corrupt,
373 and it must not be swallowed by ``except ArchiveReadError`` handlers.
374 """
377class OpaqueArchiveMetadata(Protocol):
378 """Interface for opaque archive metadata.
380 In addition to implementing the methods defined here, all implementations
381 must be pickleable.
382 """
384 def copy(self) -> Self | None:
385 """Copy, reference, or discard metadata when its holding object is
386 copied.
387 """
388 ...
390 def subset(self, bbox: Box) -> Self | None:
391 """Copy, reference, or discard metadata when a subset of its its
392 holding object is extracted.
394 Parameters
395 ----------
396 bbox
397 Bounding box of the subset being extracted.
398 """
399 ...
402def no_header_updates(header: astropy.io.fits.Header) -> None:
403 """Do not make any modifications to the given FITS header.
405 Parameters
406 ----------
407 header
408 FITS header that is left unchanged.
409 """
412def _parse_major(version: str) -> int:
413 """Return the integer major component of a major.minor.patch string.
415 Raises
416 ------
417 ArchiveReadError
418 If ``version`` is not a non-empty string of the form
419 ``major.minor.patch`` with integer components.
420 """
421 if not isinstance(version, str) or not version: 421 ↛ 422line 421 didn't jump to line 422 because the condition on line 421 was never true
422 raise ArchiveReadError(f"Schema version {version!r} is not a non-empty string.")
423 head = version.split(".", 1)[0]
424 try:
425 return int(head)
426 except ValueError as exc:
427 raise ArchiveReadError(f"Schema version {version!r} has non-integer major.") from exc
430def _check_compat(
431 name: str,
432 on_disk_version: str,
433 on_disk_min_read: int,
434 in_code_version: str,
435) -> None:
436 """Raise `ArchiveReadError` if a tree written with the given
437 schema_version/min_read_version cannot be read by the current code.
439 See :ref:`lsst.images-schema-versioning` for the compatibility rule.
440 """
441 in_code_major = _parse_major(in_code_version)
442 if on_disk_min_read > in_code_major:
443 raise ArchiveReadError(
444 f"{name}: tree requires reader major >= {on_disk_min_read}; this release is {in_code_version}."
445 )
448def _check_format_version(name: str, on_disk: int, in_code: int) -> None:
449 """Raise `ArchiveReadError` if a backend file's container layout
450 version is newer than this release knows how to read.
451 """
452 if on_disk > in_code:
453 raise ArchiveReadError(
454 f"{name}: on-disk container format version {on_disk} is "
455 f"newer than this release ({in_code}); cannot read."
456 )