Coverage for python/lsst/daf/butler/datastores/fileDatastoreClient.py: 78%

21 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-19 03:43 -0700

1__all__ = ("get_dataset_as_python_object", "FileDatastoreGetPayload") 

2 

3from typing import Any, Literal 

4 

5import pydantic 

6from lsst.daf.butler import DatasetRef, Location 

7from lsst.daf.butler.datastore.cache_manager import DatastoreDisabledCacheManager 

8from lsst.daf.butler.datastore.stored_file_info import SerializedStoredFileInfo, StoredFileInfo 

9from lsst.daf.butler.datastores.file_datastore.get import ( 

10 DatasetLocationInformation, 

11 Mapping, 

12 generate_datastore_get_information, 

13 get_dataset_as_python_object_from_get_info, 

14) 

15from pydantic import AnyHttpUrl 

16 

17 

18class FileDatastoreGetPayloadFileInfo(pydantic.BaseModel): 

19 """Information required to read a single file stored in `FileDatastore`.""" 

20 

21 # This is intentionally restricted to HTTP for security reasons. Allowing 

22 # arbitrary URLs here would allow the server to trick the client into 

23 # fetching data from any file on its local filesystem or from remote 

24 # storage using credentials laying around in the environment. 

25 url: AnyHttpUrl 

26 """An HTTP URL that can be used to read the file.""" 

27 

28 datastoreRecords: SerializedStoredFileInfo 

29 """`FileDatastore` metadata records for this file.""" 

30 

31 

32class FileDatastoreGetPayload(pydantic.BaseModel): 

33 """A serializable representation of the data needed for retrieving an 

34 artifact and converting it to a python object. 

35 """ 

36 

37 datastore_type: Literal["file"] 

38 

39 file_info: list[FileDatastoreGetPayloadFileInfo] 

40 """List of retrieval information for each file associated with this 

41 artifact. 

42 """ 

43 

44 

45def get_dataset_as_python_object( 

46 ref: DatasetRef, 

47 payload: FileDatastoreGetPayload, 

48 *, 

49 parameters: Mapping[str, Any] | None, 

50) -> Any: 

51 """Retrieve an artifact from storage and return it as a Python object. 

52 

53 Parameters 

54 ---------- 

55 ref : `DatasetRef` 

56 Metadata about this artifact. 

57 payload : `FileDatastoreGetPayload` 

58 Pre-processed information about each file associated with this 

59 artifact. 

60 parameters : `Mapping`[`str`, `typing.Any`] 

61 `StorageClass` and `Formatter` parameters to be used when converting 

62 the artifact to a Python object. 

63 

64 Returns 

65 ------- 

66 python_object : `typing.Any` 

67 The retrieved artifact, converted to a Python object. 

68 """ 

69 fileLocations: list[DatasetLocationInformation] = [ 

70 (Location(None, str(file_info.url)), StoredFileInfo.from_simple(file_info.datastoreRecords)) 

71 for file_info in payload.file_info 

72 ] 

73 

74 datastore_file_info = generate_datastore_get_information( 

75 fileLocations, 

76 ref=ref, 

77 parameters=parameters, 

78 ) 

79 return get_dataset_as_python_object_from_get_info( 

80 datastore_file_info, ref=ref, parameters=parameters, cache_manager=DatastoreDisabledCacheManager() 

81 )