Coverage for python/lsst/images/serialization/_input_archive.py: 28%

1# This file is part of lsst-images. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ("ArchiveInfo", "DetachedArchive", "InputArchive") 

15 

16from abc import ABC, abstractmethod 

17from collections.abc import Callable 

18from contextlib import AbstractContextManager 

19from types import EllipsisType 

20from typing import IO, TYPE_CHECKING, Any, TypeVar 

21 

22import astropy.io.fits 

23import astropy.table 

24import astropy.units 

25import numpy as np 

26import pydantic 

27 

28from lsst.resources import ResourcePath, ResourcePathExpression 

29 

30from ._asdf_utils import ArrayReferenceModel, InlineArrayModel 

31from ._common import ( 

32 ArchiveAccessRequiredError, 

33 ArchiveTree, 

34 OpaqueArchiveMetadata, 

35 no_header_updates, 

36) 

37from ._tables import TableModel 

38 

39if TYPE_CHECKING: 

40 from .._transforms import FrameSet 

41 

42 

43# This pre-python-3.12 declaration is needed by Sphinx (probably the 

44# autodoc-typehints plugin. 

45P = TypeVar("P", bound=pydantic.BaseModel) 

46 

47 

48class ArchiveInfo(pydantic.BaseModel, frozen=True): 

49 """Basic identifying information about an on-disk archive. 

50 

51 Read from a file's headers/metadata without deserializing pixel data. 

52 """ 

53 

54 schema_url: str 

55 """Canonical schema URL of the top-level tree.""" 

56 

57 schema_name: str 

58 """Schema name parsed from ``schema_url``.""" 

59 

60 schema_version: str 

61 """Schema version parsed from ``schema_url``.""" 

62 

63 format_version: int | None 

64 """Container layout version (FITS ``FMTVER`` / NDF ``FORMAT_VERSION``); 

65 `None` for formats with no separate container version (JSON).""" 

66 

67 @classmethod 

68 def from_schema_url(cls, schema_url: str, *, format_version: int | None) -> ArchiveInfo: 

69 """Build an `ArchiveInfo` by parsing a schema URL of the form 

70 ``https://{host}/.../schemas/{name}-{version}``. 

71 

72 The URL is parsed with `~lsst.resources.ResourcePath` and must be an 

73 ``http(s)`` URL whose final directory is ``schemas``, so a 

74 ``DATAMODL`` header written by an unrelated tool cannot steer reads 

75 toward an arbitrary schema. The host is not restricted: external 

76 packages mint schema URLs under their own documentation sites via 

77 `ArchiveTree.SCHEMA_URL_BASE`. 

78 

79 Parameters 

80 ---------- 

81 schema_url 

82 Schema URL to parse for the schema name and version. 

83 format_version 

84 Container layout version, or `None` for formats with no 

85 separate container version. 

86 """ 

87 parsed = ResourcePath(schema_url) 

88 segments = parsed.path.rstrip("/").split("/") 

89 if ( 

90 parsed.scheme not in ("http", "https") 

91 or not parsed.netloc 

92 or len(segments) < 2 

93 or segments[-2] != "schemas" 

94 ): 

95 raise ValueError( 

96 f"Schema URL {schema_url!r} does not have the required " 

97 "https://{host}/.../schemas/{name}-{version} form; " 

98 "this file was not written by lsst.images." 

99 ) 

100 tail = parsed.basename() 

101 # Split on the last hyphen: schema names may contain hyphens; the 

102 # version (after the final hyphen) is assumed not to. 

103 name, _, version = tail.rpartition("-") 

104 if not name or not version: 

105 raise ValueError(f"Cannot parse schema name/version from URL {schema_url!r}.") 

106 return cls( 

107 schema_url=schema_url, 

108 schema_name=name, 

109 schema_version=version, 

110 format_version=format_version, 

111 ) 

112 

113 

114class InputArchive[P: pydantic.BaseModel](ABC): 

115 """Abstract interface for reading from a file format. 

116 

117 Notes 

118 ----- 

119 An input archive instance is assumed to be paired with a Pydantic model 

120 that represents a JSON tree, with the archive used to deserialize data that 

121 is not native JSON from data that is (which may just be a reference to 

122 binary data stored elsewhere in the file). The archive doesn't actually 

123 hold that model instance because we'd prefer to avoid making the input 

124 archive generic over the model type. It is expected that most concrete 

125 archive implementations will provide a method to load the paired model from 

126 a file, but this is not part of the base class interface. 

127 """ 

128 

129 @classmethod 

130 def get_basic_info(cls, path: ResourcePathExpression) -> ArchiveInfo: 

131 """Return basic identifying information for the archive at ``path`` 

132 without deserializing pixel data. 

133 

134 Each concrete backend reads only the headers/metadata it needs. 

135 

136 Parameters 

137 ---------- 

138 path 

139 Path to the archive to read. 

140 """ 

141 raise NotImplementedError(f"{cls.__name__} does not implement get_basic_info.") 

142 

143 @classmethod 

144 def open_tree( 

145 cls, 

146 path: ResourcePathExpression | IO[bytes], 

147 *, 

148 partial: bool = True, 

149 **backend_kwargs: Any, 

150 ) -> AbstractContextManager[tuple[InputArchive[P], ArchiveTree, ArchiveInfo]]: 

151 """Open ``path``, load and validate its top-level tree, and yield 

152 ``(archive, tree, info)`` as a context manager. 

153 

154 Parameters 

155 ---------- 

156 path 

157 File to be opened (local or remote), or a seekable binary 

158 stream containing the file's content. 

159 partial 

160 Whether the file should be opened for incremental reads or not. 

161 Can be ignored by a backend where not relevant. 

162 **backend_kwargs 

163 Any keyword parameters that should be forwarded to the backend 

164 open. 

165 

166 Raises 

167 ------ 

168 ArchiveReadError 

169 If the file's schema is not registered. 

170 

171 Notes 

172 ----- 

173 Each concrete backend implements this. 

174 """ 

175 raise NotImplementedError(f"{cls.__name__} does not implement open_tree.") 

176 

177 @abstractmethod 

178 def deserialize_pointer[U: ArchiveTree, V]( 

179 self, pointer: P, model_type: type[U], deserializer: Callable[[U, InputArchive[P]], V] 

180 ) -> V: 

181 """Deserialize an object that was saved by 

182 `~lsst.serialization.OutputArchive.serialize_pointer`. 

183 

184 Parameters 

185 ---------- 

186 pointer 

187 JSON Pointer model to dereference. 

188 model_type 

189 Pydantic model type that the pointer should dereference to. 

190 deserializer 

191 Callable that takes an instance of ``model_type`` and an input 

192 archive, and returns the deserialized object. 

193 

194 Returns 

195 ------- 

196 V 

197 The deserialized object. 

198 

199 Notes 

200 ----- 

201 Implementations are required to remember previously-deserialized 

202 objects and return them when the same pointer is passed in multiple 

203 times. 

204 

205 There is no ``deserialize_direct`` (to pair with 

206 `~lsst.serialization.OutputArchive.serialize_direct`) because the 

207 caller can just call a deserializer function directly on a sub-model 

208 of its Pydantic tree. 

209 """ 

210 raise NotImplementedError() 

211 

212 @abstractmethod 

213 def get_frame_set(self, ref: P) -> FrameSet: 

214 """Return an already-deserialized frame set from the archive. 

215 

216 Parameters 

217 ---------- 

218 ref 

219 Implementation-specific reference to the frame set. 

220 

221 Returns 

222 ------- 

223 FrameSet 

224 Loaded frame set. 

225 """ 

226 raise NotImplementedError() 

227 

228 @abstractmethod 

229 def get_array( 

230 self, 

231 model: ArrayReferenceModel | InlineArrayModel, 

232 *, 

233 slices: tuple[slice, ...] | EllipsisType = ..., 

234 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

235 ) -> np.ndarray: 

236 """Load an array from the archive. 

237 

238 Parameters 

239 ---------- 

240 model 

241 A Pydantic model that references or holds the array. 

242 slices 

243 Slices that specify a subset of the original array to read. 

244 strip_header 

245 A callable that strips out any FITS header cards added by the 

246 ``update_header`` argument in the corresponding call to 

247 `~lsst.images.serialization.OutputArchive.add_array`. 

248 """ 

249 raise NotImplementedError() 

250 

251 @abstractmethod 

252 def get_table( 

253 self, 

254 model: TableModel, 

255 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

256 ) -> astropy.table.Table: 

257 """Load a table from the archive. 

258 

259 Parameters 

260 ---------- 

261 model 

262 A Pydantic model that references or holds the table. 

263 strip_header 

264 A callable that strips out any FITS header cards added by the 

265 ``update_header`` argument in the corresponding call to 

266 `~lsst.serialization.OutputArchive.add_table`. 

267 

268 Returns 

269 ------- 

270 astropy.table.Table 

271 The loaded table. 

272 """ 

273 raise NotImplementedError() 

274 

275 @abstractmethod 

276 def get_structured_array( 

277 self, 

278 model: TableModel, 

279 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

280 ) -> np.ndarray: 

281 """Load a table from the archive as a structured array. 

282 

283 Parameters 

284 ---------- 

285 model 

286 A Pydantic model that references or holds the table. 

287 strip_header 

288 A callable that strips out any FITS header cards added by the 

289 ``update_header`` argument in the corresponding call to 

290 `~lsst.serialization.OutputArchive.add_structured_array`. 

291 

292 Returns 

293 ------- 

294 numpy.ndarray 

295 The loaded table as a structured array. 

296 """ 

297 raise NotImplementedError() 

298 

299 def get_opaque_metadata(self) -> OpaqueArchiveMetadata | None: 

300 """Return opaque metadata loaded from the file that should be saved if 

301 another version of the object is saved to the same file format. 

302 

303 Returns 

304 ------- 

305 OpaqueArchiveMetadata 

306 Opaque metadata specific to this archive type that should be 

307 round-tripped if it is saved in the same format. 

308 """ 

309 return None 

310 

311 

312class DetachedArchive(InputArchive[Any]): 

313 """An input archive that is not attached to any file. 

314 

315 Every method that would read data from a file raises 

316 `ArchiveAccessRequiredError`. 

317 

318 Notes 

319 ----- 

320 Passing an instance to `ArchiveTree.deserialize_component` probes 

321 whether a component can be deserialized from the tree alone: success 

322 means no file access was needed, while `ArchiveAccessRequiredError` 

323 means the caller must use a live archive instead. Instances hold no 

324 state, so a single instance can be shared by any number of probes. 

325 """ 

326 

327 def deserialize_pointer[U: ArchiveTree, V]( 

328 self, pointer: Any, model_type: type[U], deserializer: Callable[[U, InputArchive[Any]], V] 

329 ) -> V: 

330 # Docstring inherited. 

331 raise ArchiveAccessRequiredError("Dereferencing an archive pointer requires file access.") 

332 

333 def get_frame_set(self, ref: Any) -> FrameSet: 

334 # Docstring inherited. 

335 raise ArchiveAccessRequiredError("Reading a frame set requires file access.") 

336 

337 def get_array( 

338 self, 

339 model: ArrayReferenceModel | InlineArrayModel, 

340 *, 

341 slices: tuple[slice, ...] | EllipsisType = ..., 

342 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

343 ) -> np.ndarray: 

344 # Docstring inherited. 

345 raise ArchiveAccessRequiredError("Reading an array requires file access.") 

346 

347 def get_table( 

348 self, 

349 model: TableModel, 

350 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

351 ) -> astropy.table.Table: 

352 # Docstring inherited. 

353 raise ArchiveAccessRequiredError("Reading a table requires file access.") 

354 

355 def get_structured_array( 

356 self, 

357 model: TableModel, 

358 strip_header: Callable[[astropy.io.fits.Header], None] = no_header_updates, 

359 ) -> np.ndarray: 

360 # Docstring inherited. 

361 raise ArchiveAccessRequiredError("Reading a structured array requires file access.")