Coverage for python/lsst/daf/butler/tests/server.py: 7%

67 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-03 02:47 -0700

1import os 

2from collections.abc import Iterator 

3from contextlib import contextmanager 

4from dataclasses import dataclass 

5from tempfile import TemporaryDirectory 

6 

7from fastapi import FastAPI, Request 

8from fastapi.testclient import TestClient 

9from lsst.daf.butler import Butler, Config, LabeledButlerFactory 

10from lsst.daf.butler.remote_butler import RemoteButler, RemoteButlerFactory 

11from lsst.daf.butler.remote_butler.server import create_app 

12from lsst.daf.butler.remote_butler.server._dependencies import butler_factory_dependency 

13from lsst.resources.s3utils import clean_test_environment_for_s3, getS3Client 

14 

15from ..direct_butler import DirectButler 

16from .hybrid_butler import HybridButler 

17from .server_utils import add_auth_header_check_middleware 

18 

19try: 

20 # moto v5 

21 from moto import mock_aws # type: ignore 

22except ImportError: 

23 # moto v4 and earlier 

24 from moto import mock_s3 as mock_aws # type: ignore 

25 

26__all__ = ("create_test_server", "TestServerInstance", "TEST_REPOSITORY_NAME") 

27 

28 

29TEST_REPOSITORY_NAME = "testrepo" 

30 

31 

32@dataclass(frozen=True) 

33class TestServerInstance: 

34 """Butler instances and other data associated with a temporary server 

35 instance. 

36 """ 

37 

38 config_file_path: str 

39 """Path to the Butler config file used by the server.""" 

40 client: TestClient 

41 """HTTPX client connected to the temporary server.""" 

42 remote_butler: RemoteButler 

43 """`RemoteButler` connected to the temporary server.""" 

44 remote_butler_without_error_propagation: RemoteButler 

45 """`RemoteButler` connected to the temporary server. 

46 

47 By default, the TestClient instance raises any unhandled exceptions 

48 from the server as if they had originated in the client to ease debugging. 

49 However, this can make it appear that error propagation is working 

50 correctly when in a real deployment the server exception would cause a 500 

51 Internal Server Error. This instance of the butler is set up so that any 

52 unhandled server exceptions do return a 500 status code.""" 

53 direct_butler: Butler 

54 """`DirectButler` instance connected to the same repository as the 

55 temporary server. 

56 """ 

57 hybrid_butler: HybridButler 

58 """`HybridButler` instance connected to the temporary server.""" 

59 

60 

61@contextmanager 

62def create_test_server(test_directory: str) -> Iterator[TestServerInstance]: 

63 """Create a temporary Butler server instance for testing. 

64 

65 Parameters 

66 ---------- 

67 test_directory : `str` 

68 Path to the ``tests/`` directory at the root of the repository, 

69 containing Butler test configuration files. 

70 

71 Returns 

72 ------- 

73 instance : `TestServerInstance` 

74 Object containing Butler instances connected to the server and 

75 associated information. 

76 """ 

77 # Set up a mock S3 environment using Moto. Moto also monkeypatches the 

78 # `requests` library so that any HTTP requests to presigned S3 URLs get 

79 # redirected to the mocked S3. 

80 # Note that all files are stored in memory. 

81 with clean_test_environment_for_s3(): 

82 with mock_aws(): 

83 base_config_path = os.path.join(test_directory, "config/basic/server.yaml") 

84 # Create S3 buckets used for the datastore in server.yaml. 

85 for bucket in ["mutable-bucket", "immutable-bucket"]: 

86 getS3Client().create_bucket(Bucket=bucket) 

87 

88 with TemporaryDirectory() as root: 

89 Butler.makeRepo(root, config=Config(base_config_path), forceConfigRoot=False) 

90 config_file_path = os.path.join(root, "butler.yaml") 

91 

92 app = create_app() 

93 add_auth_header_check_middleware(app) 

94 _add_root_exception_handler(app) 

95 # Override the server's Butler initialization to point at our 

96 # test repo 

97 server_butler_factory = LabeledButlerFactory({TEST_REPOSITORY_NAME: config_file_path}) 

98 # DirectButler has a dimension_record_cache object that 

99 # maintains a complete set of dimension records for governor 

100 # dimensions. These values change infrequently and are needed 

101 # for almost every DirectButler operation, so the complete set 

102 # is downloaded the first time a record is needed. 

103 # 

104 # On the server it would be expensive to do this for every 

105 # request's new DirectButler instance, so normally these are 

106 # loaded once, the first time a repository is accessed. This 

107 # is a problem for unit tests because they typically manipulate 

108 # instrument records etc during setup. So configure the 

109 # factory to disable this preloading and re-fetch the records 

110 # as needed. 

111 server_butler_factory._preload_direct_butler_cache = False 

112 app.dependency_overrides[butler_factory_dependency] = lambda: server_butler_factory 

113 

114 client = TestClient(app) 

115 client_without_error_propagation = TestClient(app, raise_server_exceptions=False) 

116 

117 remote_butler = _make_remote_butler(client) 

118 remote_butler_without_error_propagation = _make_remote_butler( 

119 client_without_error_propagation 

120 ) 

121 

122 direct_butler = Butler.from_config(config_file_path, writeable=True) 

123 assert isinstance(direct_butler, DirectButler) 

124 hybrid_butler = HybridButler(remote_butler, direct_butler) 

125 

126 yield TestServerInstance( 

127 config_file_path=config_file_path, 

128 client=client, 

129 direct_butler=direct_butler, 

130 remote_butler=remote_butler, 

131 remote_butler_without_error_propagation=remote_butler_without_error_propagation, 

132 hybrid_butler=hybrid_butler, 

133 ) 

134 

135 

136def _make_remote_butler(client: TestClient) -> RemoteButler: 

137 remote_butler_factory = RemoteButlerFactory( 

138 f"https://test.example/api/butler/repo/{TEST_REPOSITORY_NAME}", client 

139 ) 

140 return remote_butler_factory.create_butler_for_access_token("fake-access-token") 

141 

142 

143class UnhandledServerError(Exception): 

144 """Raised for unhandled exceptions within the server that would result in a 

145 500 Internal Server Error in a real deployment. This allows us to tell the 

146 difference between exceptions being propagated intentionally, and those 

147 just bubbling up implicitly from the server to the client. 

148 

149 The FastAPI TestClient by default passes unhandled exceptions up from the 

150 server to the client. This is useful behavior for unit testing because it 

151 gives you traceability from the test to the problem in the server code. 

152 However, because RemoteButler is in some ways just a proxy for the 

153 server-side Butler, we raise similar exceptions on the client and server 

154 side. Thus the default TestClient behavior can mask missing error-handling 

155 logic. 

156 """ 

157 

158 

159def _add_root_exception_handler(app: FastAPI) -> None: 

160 @app.exception_handler(Exception) 

161 async def convert_exception_types(request: Request, exc: Exception) -> None: 

162 raise UnhandledServerError("Unhandled server exception") from exc