Coverage for python/lsst/images/serialization/_common.py: 61%
176 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 09:29 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "SCHEMA_URL_BASE",
16 "ArchiveAccessRequiredError",
17 "ArchiveReadError",
18 "ArchiveTree",
19 "ButlerInfo",
20 "DevelopmentSchemaWarning",
21 "InvalidComponentError",
22 "InvalidParameterError",
23 "JsonRef",
24 "MetadataValue",
25 "OpaqueArchiveMetadata",
26 "is_development_version",
27 "no_header_updates",
28 "warn_for_development_schemas",
29)
31import operator
32import warnings
33from abc import ABC, abstractmethod
34from collections.abc import Iterator
35from copy import deepcopy
36from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
38import astropy.table
39import astropy.units
40import pydantic
41from packaging.version import Version
43from .._geom import Box
44from ..utils import is_none
45from ._external_metadata import ExternalMetadata
46from ._migrations import _MIGRATABLE_NAMES, _MIGRATIONS
48try:
49 from lsst.daf.butler import DatasetProvenance, SerializedDatasetRef
50except ImportError:
51 type DatasetProvenance = Any # type: ignore[no-redef]
52 type SerializedDatasetRef = Any # type: ignore[no-redef]
54if TYPE_CHECKING:
55 import astropy.io.fits
57 from ._input_archive import InputArchive
60type MetadataValue = (
61 pydantic.StrictInt | pydantic.StrictFloat | pydantic.StrictStr | pydantic.StrictBool | None
62)
64SCHEMA_URL_BASE = "https://images.lsst.io/schemas"
65"""Base for the schema URLs of this package's own schemas, used as
66``{SCHEMA_URL_BASE}/{name}-{version}``.
68External packages providing their own schemas override
69`~lsst.images.serialization.ArchiveTree.SCHEMA_URL_BASE` instead, so their
70schema URLs are minted under a site they control.
71"""
73_ARCHIVE_READ_CONTEXT = object()
74"""Pydantic context used when validating a tree read from an archive."""
77class ButlerInfo(pydantic.BaseModel):
78 """Information about a butler dataset."""
80 dataset: SerializedDatasetRef
81 provenance: DatasetProvenance = pydantic.Field(default_factory=DatasetProvenance)
84class JsonRef(pydantic.BaseModel, serialize_by_alias=True):
85 """Pydantic model for JSON Reference / Pointer (IETF RFC 6901).
87 Notes
88 -----
89 This model does not do any of the escaping or special-character
90 interpretation required by the spec; it assumes that's already been done,
91 so its job is *just* putting a ``$ref`` field inside another model.
92 """
94 ref: str = pydantic.Field(alias="$ref")
97class ArchiveTree(
98 pydantic.BaseModel, ABC, ser_json_inf_nan="constants", ser_json_bytes="base64", val_json_bytes="base64"
99):
100 """An intermediate base class of `pydantic.BaseModel` that should be used
101 for all objects that may be used as the top-level tree models written to
102 archives.
104 See :ref:`lsst.images-schema-versioning` for how the ``SCHEMA_NAME`` /
105 ``SCHEMA_VERSION`` / ``MIN_READ_VERSION`` constants and the
106 ``schema_version`` / ``min_read_version`` / ``schema_url`` fields are used.
107 """
109 SCHEMA_NAME: ClassVar[str]
110 SCHEMA_VERSION: ClassVar[str]
111 MIN_READ_VERSION: ClassVar[int]
112 SCHEMA_URL_BASE: ClassVar[str] = SCHEMA_URL_BASE
113 """Base for this schema's URL, as ``{SCHEMA_URL_BASE}/{name}-{version}``.
115 External packages providing their own schemas should override this (once,
116 on a shared intermediate base class) so their schema URLs are minted
117 under a documentation site they control rather than images.lsst.io."""
119 PUBLIC_TYPE: ClassVar[type]
120 """In-memory Python type produced by this tree's ``deserialize`` (e.g.
121 `dict` for a mapping return). Declared explicitly by each concrete
122 subclass and surfaced by
123 `~lsst.images.serialization.public_type_for_schema`."""
125 # These mirror the SCHEMA_VERSION / MIN_READ_VERSION class constants and
126 # are never passed on construction, so repr omits them; they say nothing
127 # about the object that its type does not already say.
128 schema_version: str = pydantic.Field(
129 default="1.0.0",
130 description="Data-model schema version of this tree (major.minor.patch).",
131 repr=False,
132 )
133 min_read_version: int = pydantic.Field(
134 default=1,
135 description="Smallest reader major that can interpret this tree.",
136 repr=False,
137 )
138 metadata: dict[str, MetadataValue] = pydantic.Field(
139 default_factory=dict, description="Additional unstructured metadata.", exclude_if=operator.not_
140 )
141 butler_info: ButlerInfo | None = pydantic.Field(
142 default=None,
143 description="Information about the butler dataset backed by this file.",
144 exclude_if=is_none,
145 )
146 # Populated by the archive machinery rather than by a caller, so repr
147 # omits it as well.
148 indirect: list[Any] = pydantic.Field(
149 default_factory=list,
150 description="Serialized nested objects that may be saved or read more than once.",
151 exclude_if=operator.not_,
152 repr=False,
153 )
155 @pydantic.computed_field(description="Canonical schema URL for this tree.") # type: ignore[prop-decorator]
156 @property
157 def schema_url(self) -> str:
158 """Return the schema URL of this tree's class.
160 Computed from ``SCHEMA_NAME`` and ``SCHEMA_VERSION`` ClassVars.
161 """
162 cls = type(self)
163 return f"{cls.SCHEMA_URL_BASE}/{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION}"
165 # The pydantic mypy plugin synthesizes a per-model model_construct
166 # signature, which this override necessarily conflicts with.
167 @classmethod # type: ignore[no-redef]
168 def model_construct( # type: ignore[override]
169 cls, _fields_set: set[str] | None = None, **values: Any
170 ) -> Self:
171 """Construct a tree without validation, stamping the in-code schema
172 version.
174 The field defaults cannot know a subclass's ``SCHEMA_VERSION`` and
175 ``MIN_READ_VERSION``, and the validator that normally sets them does
176 not run here, so they are filled in from the class constants unless
177 given explicitly.
179 Parameters
180 ----------
181 _fields_set
182 Names of the fields to record as explicitly set; passed through
183 to `pydantic.BaseModel.model_construct`.
184 **values
185 Field values for the new tree.
187 Returns
188 -------
189 `ArchiveTree`
190 The constructed tree.
191 """
192 if hasattr(cls, "SCHEMA_NAME"): 192 ↛ 195line 192 didn't jump to line 195 because the condition on line 192 was always true
193 values.setdefault("schema_version", cls.SCHEMA_VERSION)
194 values.setdefault("min_read_version", cls.MIN_READ_VERSION)
195 return super().model_construct(_fields_set, **values)
197 @pydantic.model_validator(mode="before")
198 @classmethod
199 def _migrate_from_older_major(cls, data: Any, info: pydantic.ValidationInfo) -> Any:
200 """Morph an older on-disk tree into the current shape.
202 Chains registered migrations until the tree reaches the in-code major,
203 so the ``mode="after"`` validator below sees a current-shape tree.
205 A tree validated in the archive-read context, which the backends mark
206 via ``info.context``, must carry a ``schema_version``; in-memory
207 construction is unmarked and keeps the class-constant defaults. See
208 :ref:`lsst.images-schema-versioning-migration`.
209 """
210 if not isinstance(data, dict):
211 return data
212 if not hasattr(cls, "SCHEMA_NAME"): 212 ↛ 217line 212 didn't jump to line 217 because the condition on line 212 was never true
213 # ArchiveTree itself is abstract, and a subclass that has not yet
214 # declared SCHEMA_NAME (during incremental rollout) has no
215 # SCHEMA_VERSION to parse either; skip cleanly rather than
216 # raising AttributeError, mirroring the after-validator below.
217 return data
218 name = cls.SCHEMA_NAME
219 if "schema_version" not in data:
220 if info.context is _ARCHIVE_READ_CONTEXT:
221 raise _MissingSchemaVersionError(
222 f"{name}: archive tree has no schema_version; unstamped archive data is not supported."
223 )
224 return data
225 if not _MIGRATABLE_NAMES: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 return data
227 if name not in _MIGRATABLE_NAMES:
228 # This schema has not opted into migrations at all, regardless of
229 # its major: whether some *other*, unrelated schema has a
230 # registered migration must not change what this one does with
231 # its own tree. A schema in this state falls through to ordinary
232 # validation -- in-model backfill for an additive change, or a
233 # Pydantic validation error for a real incompatibility.
234 return data
235 on_disk_major = _parse_on_disk_major(data["schema_version"])
236 in_code_major = _parse_major(cls.SCHEMA_VERSION)
237 if on_disk_major < in_code_major:
238 # A failed union candidate must not mutate the input subsequently
239 # tried by another candidate. Migrations may mutate freely, so
240 # isolate the complete JSON-like tree, not just its top-level
241 # dictionary.
242 data = deepcopy(data)
243 while on_disk_major < in_code_major:
244 try:
245 step = _MIGRATIONS[(name, on_disk_major)]
246 except KeyError:
247 raise _MigrationGapError(
248 f"{name}: no migration from major {on_disk_major} to {on_disk_major + 1}."
249 ) from None
250 data = step(data)
251 on_disk_major += 1
252 data["schema_version"] = f"{on_disk_major}.0.0"
253 return data
255 @pydantic.model_validator(mode="after")
256 def _check_and_normalize_schema_version(self) -> Self:
257 """Validate and normalise the schema version fields.
259 Compares the on-tree ``schema_version`` / ``min_read_version`` against
260 the in-code values from the subclass's ClassVars; raises if
261 incompatible, otherwise normalises the fields to the in-code values.
262 """
263 cls = type(self)
264 # ArchiveTree itself is abstract (deserialize is @abstractmethod).
265 # Subclasses that haven't yet declared SCHEMA_NAME are skipped — this
266 # matters during incremental rollout and remains a safe no-op
267 # afterwards (a class-invariants test ensures every concrete subclass
268 # has the constants).
269 if not hasattr(cls, "SCHEMA_NAME"): 269 ↛ 270line 269 didn't jump to line 270 because the condition on line 269 was never true
270 return self
271 _check_compat(
272 cls.SCHEMA_NAME,
273 self.schema_version,
274 self.min_read_version,
275 cls.SCHEMA_VERSION,
276 )
277 if self.schema_version != cls.SCHEMA_VERSION:
278 self.schema_version = cls.SCHEMA_VERSION
279 if self.min_read_version != cls.MIN_READ_VERSION:
280 self.min_read_version = cls.MIN_READ_VERSION
281 return self
283 @classmethod
284 def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
285 """Inject ``$id`` and ``title`` into the subclass's JSON Schema, and
286 register the subclass in the schema-name registry.
288 Populates ``model_config['json_schema_extra']`` with values derived
289 from the subclass's ``SCHEMA_NAME`` / ``SCHEMA_VERSION`` ClassVars,
290 then registers the subclass so it can be looked up by schema name.
291 Subclasses that haven't declared the ClassVars are skipped.
292 """
293 super().__pydantic_init_subclass__(**kwargs)
294 name = cls.__dict__.get("SCHEMA_NAME")
295 version = cls.__dict__.get("SCHEMA_VERSION")
296 if name is None or version is None:
297 return
298 json_schema_extra = cls.model_config.get("json_schema_extra") or {}
299 if isinstance(json_schema_extra, dict): 299 ↛ 311line 299 didn't jump to line 311 because the condition on line 299 was always true
300 existing = dict(json_schema_extra)
301 # Always override: a subclass of a concrete schema (e.g.
302 # visit_image subclassing masked_image) inherits its parent's
303 # already-stamped values through the merged model_config, and
304 # this hook only runs when the subclass declares its own
305 # SCHEMA_NAME / SCHEMA_VERSION for these to be derived from.
306 existing["$id"] = f"{cls.SCHEMA_URL_BASE}/{name}-{version}"
307 existing["title"] = name
308 cls.model_config = {**cls.model_config, "json_schema_extra": existing}
309 # Local import to avoid the _io -> _common circular dependency at
310 # module load time.
311 from ._io import register_schema_class
313 register_schema_class(cls)
315 @abstractmethod
316 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Any:
317 """Return the in-memory object that was serialized to this tree.
319 Parameters
320 ----------
321 archive
322 The input archive to read from.
323 **kwargs
324 Additional keyword arguments specific to this type.
326 Raises
327 ------
328 ~lsst.images.serialization.InvalidParameterError
329 Raised for unsupported ``**kwargs``.
331 Notes
332 -----
333 Subclass implementations may take additional keyword-only arguments.
334 Callers that invoke this method without knowing what those might be
335 should catch `TypeError` and re-raise as
336 `~lsst.images.serialization.InvalidParameterError` if they pass
337 additional keyword arguments.
338 """
339 raise NotImplementedError()
341 def deserialize_component(self, component: str, archive: InputArchive[Any], **kwargs: Any) -> Any:
342 """Return a component in-memory object that was serialized to this
343 tree.
345 Parameters
346 ----------
347 component
348 Name of the component to read.
349 archive
350 The input archive to read from.
351 **kwargs
352 Additional keyword arguments specific to this type.
354 Raises
355 ------
356 ~lsst.images.serialization.InvalidComponentError
357 Raise if ``component`` is not recognized.
358 ~lsst.images.serialization.InvalidParameterError
359 Raised for unsupported ``**kwargs``.
361 Notes
362 -----
363 The default implementation for this method tries to get an attribute
364 with the component's name from ``self``, and then:
366 - returns `None` if it is `None`;
367 - calls `deserialize` on that object if it is also an
368 `~lsst.images.serialization.ArchiveTree`;
369 - returns it directly otherwise.
371 If there is no such attribute, it raises
372 `~lsst.images.serialization.InvalidComponentError`.
374 ``**kwargs`` are forwarded to component `deserialize` methods, but
375 are otherwise not checked. Subclasses are generally expected to
376 implement this method to do that checking and handle any components
377 for which the other will not work, and then delegate to `super` at
378 the end.
379 """
380 try:
381 component_model = getattr(self, component)
382 except AttributeError:
383 raise InvalidComponentError(
384 f"Component {component!r} is not recognized by {type(self).__name__}."
385 ) from None
386 if component_model is None:
387 return None
388 if isinstance(component_model, ArchiveTree):
389 return component_model.deserialize(archive, **kwargs)
390 return component_model
393class DevelopmentSchemaWarning(UserWarning):
394 """Warning that a file is being written with a development schema."""
397def is_development_version(version: str) -> bool:
398 """Return whether a schema version string is a PEP 440 development release.
400 Parameters
401 ----------
402 version
403 Schema version string, e.g. ``1.0.0`` or ``1.0.0.dev0``.
404 """
405 return Version(version).is_devrelease
408def _iter_archive_trees(obj: Any) -> Iterator[ArchiveTree]:
409 """Yield every `ArchiveTree` embedded in a serialized tree."""
410 if isinstance(obj, ArchiveTree):
411 yield obj
412 if isinstance(obj, pydantic.BaseModel):
413 for field_name in type(obj).model_fields:
414 yield from _iter_archive_trees(getattr(obj, field_name))
415 elif isinstance(obj, list | tuple):
416 for item in obj:
417 yield from _iter_archive_trees(item)
418 elif isinstance(obj, dict):
419 for value in obj.values():
420 yield from _iter_archive_trees(value)
423def warn_for_development_schemas(root: ArchiveTree) -> None:
424 """Emit a `DevelopmentSchemaWarning` if a serialized tree contains any
425 schema still in development.
427 Parameters
428 ----------
429 root
430 Top-level serialized tree about to be written.
431 """
432 developing = sorted(
433 {
434 tree.schema_url
435 for tree in _iter_archive_trees(root)
436 if is_development_version(type(tree).SCHEMA_VERSION)
437 }
438 )
439 if developing:
440 warnings.warn(
441 "Writing a file with development schema(s) "
442 f"{', '.join(developing)}; such files are not for production and "
443 "may not remain readable.",
444 DevelopmentSchemaWarning,
445 stacklevel=3,
446 )
449class ArchiveReadError(RuntimeError):
450 """Exception raised when the contents of an archive cannot be read."""
453class InvalidParameterError(ArchiveReadError):
454 """Exception raised by `ArchiveTree.deserialize` or
455 `ArchiveTree.deserialize_component` when passed an invalid keyword
456 argument.
457 """
460class InvalidComponentError(ArchiveReadError):
461 """Exception `ArchiveTree.deserialize_component` when passed an invalid
462 component name.
463 """
466class _MigrationGapError(ArchiveReadError, ValueError):
467 """A migration chain has no registered step for the tree's on-disk major.
469 Raised from the ``mode="before"`` validator in
470 `ArchiveTree._migrate_from_older_major`, which pydantic-core invokes as
471 part of validating a field. Deriving only from `ArchiveReadError` (a
472 `RuntimeError`) would let this exception escape a discriminated union
473 entirely: pydantic only treats `ValueError`, `TypeError` and
474 `AssertionError` raised by a validator as "this candidate failed, try the
475 next one", so a `RuntimeError` would abort the whole union instead of
476 falling through. Deriving from both means a ``left_to_right`` (or
477 ``smart``) union tries the next variant on a migration gap exactly as it
478 would on any other validation failure, while ``except ArchiveReadError``
479 callers outside a union still catch it unchanged.
480 """
483class _MissingSchemaVersionError(ArchiveReadError, ValueError):
484 """An archive tree omitted its required ``schema_version`` stamp."""
487class _MalformedSchemaVersionError(ArchiveReadError, ValueError):
488 """An archive tree's ``schema_version`` stamp could not be parsed.
490 Also a `ValueError`, for the reason `_MigrationGapError` gives: the stamp
491 is parsed in the ``mode="before"`` validator, which runs ahead of field
492 validation for every migratable union candidate, including ones the
493 payload does not belong to. A stamp the reader cannot parse therefore has
494 to fail that one candidate rather than abort the whole union.
496 Only an *on-disk* stamp raises this. A malformed in-code
497 ``SCHEMA_VERSION`` is a programming error and keeps raising the
498 `RuntimeError`-only `ArchiveReadError`, so a union cannot swallow it.
499 """
502class ArchiveAccessRequiredError(RuntimeError):
503 """Exception raised when a deserialization needs data from the file.
505 Raised by all data-access methods of
506 `~lsst.images.serialization.DetachedArchive`, signaling that the
507 requested object cannot be deserialized from the JSON tree alone.
509 Notes
510 -----
511 This deliberately does not inherit from `ArchiveReadError`: it signals
512 that a live archive is required rather than that an archive is corrupt,
513 and it must not be swallowed by ``except ArchiveReadError`` handlers.
514 """
517class OpaqueArchiveMetadata(Protocol):
518 """Interface for opaque archive metadata.
520 In addition to implementing the methods defined here, all implementations
521 must be pickleable.
522 """
524 def copy(self) -> Self | None:
525 """Copy, reference, or discard metadata when its holding object is
526 copied.
527 """
528 ...
530 def subset(self, bbox: Box) -> Self | None:
531 """Copy, reference, or discard metadata when a subset of its its
532 holding object is extracted.
534 Parameters
535 ----------
536 bbox
537 Bounding box of the subset being extracted.
538 """
539 ...
541 def external_metadata(self) -> ExternalMetadata:
542 """Return a read-only view of the metadata carried by this object
543 that is not part of the data model.
544 """
545 ...
548def no_header_updates(header: astropy.io.fits.Header) -> None:
549 """Do not make any modifications to the given FITS header.
551 Parameters
552 ----------
553 header
554 FITS header that is left unchanged.
555 """
558def _parse_major(version: object) -> int:
559 """Return the integer major component of a major.minor.patch string.
561 Accepts any object, because callers pass values that may have come
562 straight from a file; use `_parse_on_disk_major` for those.
564 Raises
565 ------
566 ArchiveReadError
567 If ``version`` is not a non-empty string of the form
568 ``major.minor.patch`` with integer components.
569 """
570 if not isinstance(version, str) or not version:
571 raise ArchiveReadError(f"Schema version {version!r} is not a non-empty string.")
572 head = version.split(".", 1)[0]
573 try:
574 return int(head)
575 except ValueError as exc:
576 raise ArchiveReadError(f"Schema version {version!r} has non-integer major.") from exc
579def _parse_on_disk_major(version: object) -> int:
580 """Return the integer major of a version stamp that came from a file.
582 Raises
583 ------
584 _MalformedSchemaVersionError
585 If ``version`` is not a non-empty string of the form
586 ``major.minor.patch`` with integer components. Unlike `_parse_major`,
587 this is a `ValueError`, so a stamp the reader cannot parse fails one
588 union candidate instead of aborting the union.
589 """
590 try:
591 return _parse_major(version)
592 except ArchiveReadError as exc:
593 raise _MalformedSchemaVersionError(str(exc)) from None
596def _check_compat(
597 name: str,
598 on_disk_version: str,
599 on_disk_min_read: int,
600 in_code_version: str,
601) -> None:
602 """Raise `ArchiveReadError` if a tree written with the given
603 schema_version/min_read_version cannot be read by the current code.
605 See :ref:`lsst.images-schema-versioning` for the compatibility rule.
606 """
607 in_code_major = _parse_major(in_code_version)
608 if on_disk_min_read > in_code_major:
609 raise ArchiveReadError(
610 f"{name}: tree requires reader major >= {on_disk_min_read}; this release is {in_code_version}."
611 )
614def _check_format_version(name: str, on_disk: int, in_code: int) -> None:
615 """Raise `ArchiveReadError` if a backend file's container layout
616 version is newer than this release knows how to read.
617 """
618 if on_disk > in_code:
619 raise ArchiveReadError(
620 f"{name}: on-disk container format version {on_disk} is "
621 f"newer than this release ({in_code}); cannot read."
622 )