Coverage for python/lsst/images/serialization/_input

1# This file is part of lsst-images.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# Use of this source code is governed by a 3-clause BSD-style

10# license that can be found in the LICENSE file.

12from __future__ import annotations

14__all__ = ("ArchiveInfo", "DetachedArchive", "InputArchive")

16from abc import ABC, abstractmethod

17from collections.abc import Callable

18from contextlib import AbstractContextManager

19from types import EllipsisType

20from typing import IO, TYPE_CHECKING, Any, TypeVar

22import astropy.io.fits

23import astropy.table

24import astropy.units

25import numpy as np

26import pydantic

28from lsst.resources import ResourcePath, ResourcePathExpression

30from ._asdf_utils import ArrayReferenceModel, InlineArrayModel

31from ._common import (

32 ArchiveAccessRequiredError,

33 ArchiveTree,

34 OpaqueArchiveMetadata,

35 no_header_updates,

36)

37from ._tables import TableModel

39if TYPE_CHECKING:

40 from .._transforms import FrameSet

43# This pre-python-3.12 declaration is needed by Sphinx (probably the

44# autodoc-typehints plugin.

45P = TypeVar("P", bound=pydantic.BaseModel)

48class ArchiveInfo(pydantic.BaseModel, frozen=True):

49 """Basic identifying information about an on-disk archive.

51 Read from a file's headers/metadata without deserializing pixel data.

52 """

54 schema_url: str

55 """Canonical schema URL of the top-level tree."""

57 schema_name: str

58 """Schema name parsed from ``schema_url``."""

60 schema_version: str

61 """Schema version parsed from ``schema_url``."""

63 format_version: int | None

64 """Container layout version (FITS ``FMTVER`` / NDF ``FORMAT_VERSION``);

65 `None` for formats with no separate container version (JSON)."""

67 @classmethod

68 def from_schema_url(cls, schema_url: str, *, format_version: int | None) -> ArchiveInfo:

69 """Build an `ArchiveInfo` by parsing a schema URL of the form

70 ``https://{host}/.../schemas/{name}-{version}``.

72 The URL is parsed with `~lsst.resources.ResourcePath` and must be an

73 ``http(s)`` URL whose final directory is ``schemas``, so a

74 ``DATAMODL`` header written by an unrelated tool cannot steer reads

75 toward an arbitrary schema. The host is not restricted: external

76 packages mint schema URLs under their own documentation sites via

77 `ArchiveTree.SCHEMA_URL_BASE`.

79 Parameters

80 ----------

81 schema_url

82 Schema URL to parse for the schema name and version.

83 format_version

84 Container layout version, or `None` for formats with no

85 separate container version.

86 """

87 parsed = ResourcePath(schema_url)

88 segments = parsed.path.rstrip("/").split("/")

89 if (

90 parsed.scheme not in ("http", "https")

91 or not parsed.netloc

92 or len(segments) < 2

93 or segments[-2] != "schemas"

94 ):

95 raise ValueError(

96 f"Schema URL {schema_url!r} does not have the required "

97 "https://{host}/.../schemas/{name}-{version} form; "

98 "this file was not written by lsst.images."

99 )

100 tail = parsed.basename()

101 # Split on the last hyphen: schema names may contain hyphens; the

102 # version (after the final hyphen) is assumed not to.

103 name, _, version = tail.rpartition("-")

104 if not name or not version:

105 raise ValueError(f"Cannot parse schema name/version from URL {schema_url!r}.")

106 return cls(

107 schema_url=schema_url,

108 schema_name=name,

109 schema_version=version,

110 format_version=format_version,

111 )

112

113

114class InputArchive[P: pydantic.BaseModel](ABC):

115 """Abstract interface for reading from a file format.

116

117 Notes

118 -----

119 An input archive instance is assumed to be paired with a Pydantic model

120 that represents a JSON tree, with the archive used to deserialize data that

121 is not native JSON from data that is (which may just be a reference to

122 binary data stored elsewhere in the file). The archive doesn't actually

123 hold that model instance because we'd prefer to avoid making the input

124 archive generic over the model type. It is expected that most concrete

125 archive implementations will provide a method to load the paired model from

126 a file, but this is not part of the base class interface.

127 """

128

129 @classmethod

130 def get_basic_info(cls, path: ResourcePathExpression) -> ArchiveInfo:

131 """Return basic identifying information for the archive at ``path``

132 without deserializing pixel data.

133

134 Each concrete backend reads only the headers/metadata it needs.

135

136 Parameters

137 ----------

138 path

139 Path to the archive to read.

140 """

141 raise NotImplementedError(f"{cls.__name__} does not implement get_basic_info.")

142

143 @classmethod

144 def open_tree(

145 cls,

146 path: ResourcePathExpression | IO[bytes],

147 *,

148 partial: bool = True,

149 **backend_kwargs: Any,

150 ) -> AbstractContextManager[tuple[InputArchive[P], ArchiveTree, ArchiveInfo]]:

151 """Open ``path``, load and validate its top-level tree, and yield

152 ``(archive, tree, info)`` as a context manager.

153

154 Parameters

155 ----------

156 path

157 File to be opened (local or remote), or a seekable binary

158 stream containing the file's content.

159 partial

160 Whether the file should be opened for incremental reads or not.

161 Can be ignored by a backend where not relevant.

162 **backend_kwargs

163 Any keyword parameters that should be forwarded to the backend

164 open.

165

166 Raises

167 ------

168 ArchiveReadError

169 If the file's schema is not registered.

170

171 Notes

172 -----

173 Each concrete backend implements this.

174 """

175 raise NotImplementedError(f"{cls.__name__} does not implement open_tree.")

176

177 @abstractmethod

178 def deserialize_pointer[U: ArchiveTree, V](

179 self, pointer: P, model_type: type[U], deserializer: Callable[[U, InputArchive[P]], V]

180 ) -> V:

181 """Deserialize an object that was saved by

182 `~lsst.serialization.OutputArchive.serialize_pointer`.

183

184 Parameters

185 ----------

186 pointer

187 JSON Pointer model to dereference.

188 model_type

189 Pydantic model type that the pointer should dereference to.

190 deserializer

191 Callable that takes an instance of ``model_type`` and an input

192 archive, and returns the deserialized object.

193

194 Returns

195 -------

196 V

197 The deserialized object.

198

199 Notes

200 -----

201 Implementations are required to remember previously-deserialized

202 objects and return them when the same pointer is passed in multiple

203 times.

204

205 There is no ``deserialize_direct`` (to pair with

206 `~lsst.serialization.OutputArchive.serialize_direct`) because the

207 caller can just call a deserializer function directly on a sub-model

208 of its Pydantic tree.

209 """

210 raise NotImplementedError()

211

212 @abstractmethod

213 def get_frame_set(self, ref: P) -> FrameSet:

214 """Return an already-deserialized frame set from the archive.

215

216 Parameters

217 ----------

218 ref

219 Implementation-specific reference to the frame set.

220

221 Returns

222 -------

223 FrameSet

224 Loaded frame set.

225 """

226 raise NotImplementedError()

227

228 @abstractmethod

229 def get_array(

230 self,

231 model: ArrayReferenceModel | InlineArrayModel,

232 *,

233 slices: tuple[slice, ...] | EllipsisType = ...,

234 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

235 ) -> np.ndarray:

236 """Load an array from the archive.

237

238 Parameters

239 ----------

240 model

241 A Pydantic model that references or holds the array.

242 slices

243 Slices that specify a subset of the original array to read.

244 strip_header

245 A callable that strips out any FITS header cards added by the

246 ``update_header`` argument in the corresponding call to

247 `~lsst.images.serialization.OutputArchive.add_array`.

248 """

249 raise NotImplementedError()

250

251 @abstractmethod

252 def get_table(

253 self,

254 model: TableModel,

255 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

256 ) -> astropy.table.Table:

257 """Load a table from the archive.

258

259 Parameters

260 ----------

261 model

262 A Pydantic model that references or holds the table.

263 strip_header

264 A callable that strips out any FITS header cards added by the

265 ``update_header`` argument in the corresponding call to

266 `~lsst.serialization.OutputArchive.add_table`.

267

268 Returns

269 -------

270 astropy.table.Table

271 The loaded table.

272 """

273 raise NotImplementedError()

274

275 @abstractmethod

276 def get_structured_array(

277 self,

278 model: TableModel,

279 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

280 ) -> np.ndarray:

281 """Load a table from the archive as a structured array.

282

283 Parameters

284 ----------

285 model

286 A Pydantic model that references or holds the table.

287 strip_header

288 A callable that strips out any FITS header cards added by the

289 ``update_header`` argument in the corresponding call to

290 `~lsst.serialization.OutputArchive.add_structured_array`.

291

292 Returns

293 -------

294 numpy.ndarray

295 The loaded table as a structured array.

296 """

297 raise NotImplementedError()

298

299 def get_opaque_metadata(self) -> OpaqueArchiveMetadata | None:

300 """Return opaque metadata loaded from the file that should be saved if

301 another version of the object is saved to the same file format.

302

303 Returns

304 -------

305 OpaqueArchiveMetadata

306 Opaque metadata specific to this archive type that should be

307 round-tripped if it is saved in the same format.

308 """

309 return None

310

311

312class DetachedArchive(InputArchive[Any]):

313 """An input archive that is not attached to any file.

314

315 Every method that would read data from a file raises

316 `ArchiveAccessRequiredError`.

317

318 Notes

319 -----

320 Passing an instance to `ArchiveTree.deserialize_component` probes

321 whether a component can be deserialized from the tree alone: success

322 means no file access was needed, while `ArchiveAccessRequiredError`

323 means the caller must use a live archive instead. Instances hold no

324 state, so a single instance can be shared by any number of probes.

325 """

326

327 def deserialize_pointer[U: ArchiveTree, V](

328 self, pointer: Any, model_type: type[U], deserializer: Callable[[U, InputArchive[Any]], V]

329 ) -> V:

330 # Docstring inherited.

331 raise ArchiveAccessRequiredError("Dereferencing an archive pointer requires file access.")

332

333 def get_frame_set(self, ref: Any) -> FrameSet:

334 # Docstring inherited.

335 raise ArchiveAccessRequiredError("Reading a frame set requires file access.")

336

337 def get_array(

338 self,

339 model: ArrayReferenceModel | InlineArrayModel,

340 *,

341 slices: tuple[slice, ...] | EllipsisType = ...,

342 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

343 ) -> np.ndarray:

344 # Docstring inherited.

345 raise ArchiveAccessRequiredError("Reading an array requires file access.")

346

347 def get_table(

348 self,

349 model: TableModel,

350 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

351 ) -> astropy.table.Table:

352 # Docstring inherited.

353 raise ArchiveAccessRequiredError("Reading a table requires file access.")

354

355 def get_structured_array(

356 self,

357 model: TableModel,

358 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates,

359 ) -> np.ndarray:

360 # Docstring inherited.

361 raise ArchiveAccessRequiredError("Reading a structured array requires file access.")

Coverage for python/lsst/images/serialization/_input_archive.py: 28%

65 statements