Coverage for python/lsst/daf/butler/datastores/fileDatastoreClient.py: 78%
21 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 10:23 +0000
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 10:23 +0000
1__all__ = ("get_dataset_as_python_object", "FileDatastoreGetPayload")
3from typing import Any, Literal
5import pydantic
6from lsst.daf.butler import DatasetRef, Location
7from lsst.daf.butler.datastore.cache_manager import DatastoreDisabledCacheManager
8from lsst.daf.butler.datastore.stored_file_info import SerializedStoredFileInfo, StoredFileInfo
9from lsst.daf.butler.datastores.file_datastore.get import (
10 DatasetLocationInformation,
11 Mapping,
12 generate_datastore_get_information,
13 get_dataset_as_python_object_from_get_info,
14)
15from pydantic import AnyHttpUrl
18class FileDatastoreGetPayloadFileInfo(pydantic.BaseModel):
19 """Information required to read a single file stored in `FileDatastore`."""
21 # This is intentionally restricted to HTTP for security reasons. Allowing
22 # arbitrary URLs here would allow the server to trick the client into
23 # fetching data from any file on its local filesystem or from remote
24 # storage using credentials laying around in the environment.
25 url: AnyHttpUrl
26 """An HTTP URL that can be used to read the file."""
28 datastoreRecords: SerializedStoredFileInfo
29 """`FileDatastore` metadata records for this file."""
32class FileDatastoreGetPayload(pydantic.BaseModel):
33 """A serializable representation of the data needed for retrieving an
34 artifact and converting it to a python object.
35 """
37 datastore_type: Literal["file"]
39 file_info: list[FileDatastoreGetPayloadFileInfo]
40 """List of retrieval information for each file associated with this
41 artifact.
42 """
45def get_dataset_as_python_object(
46 ref: DatasetRef,
47 payload: FileDatastoreGetPayload,
48 *,
49 parameters: Mapping[str, Any] | None,
50) -> Any:
51 """Retrieve an artifact from storage and return it as a Python object.
53 Parameters
54 ----------
55 ref : `DatasetRef`
56 Metadata about this artifact.
57 payload : `FileDatastoreGetPayload`
58 Pre-processed information about each file associated with this
59 artifact.
60 parameters : `Mapping`[`str`, `typing.Any`]
61 `StorageClass` and `Formatter` parameters to be used when converting
62 the artifact to a Python object.
64 Returns
65 -------
66 python_object : `typing.Any`
67 The retrieved artifact, converted to a Python object.
68 """
69 fileLocations: list[DatasetLocationInformation] = [
70 (Location(None, str(file_info.url)), StoredFileInfo.from_simple(file_info.datastoreRecords))
71 for file_info in payload.file_info
72 ]
74 datastore_file_info = generate_datastore_get_information(
75 fileLocations,
76 ref=ref,
77 parameters=parameters,
78 )
79 return get_dataset_as_python_object_from_get_info(
80 datastore_file_info, ref=ref, parameters=parameters, cache_manager=DatastoreDisabledCacheManager()
81 )