Coverage for python / lsst / daf / butler / tests / server.py: 0%
87 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-17 08:48 +0000
1import json
2import os
3from collections.abc import Iterator
4from contextlib import closing, contextmanager
5from dataclasses import dataclass
6from tempfile import TemporaryDirectory
8from fastapi import FastAPI, Request
9from fastapi.testclient import TestClient
11from lsst.daf.butler import Butler, ButlerConfig, Config, LabeledButlerFactory
12from lsst.daf.butler.remote_butler import RemoteButler
13from lsst.daf.butler.remote_butler._factory import RemoteButlerFactory
14from lsst.daf.butler.remote_butler.server import create_app
15from lsst.daf.butler.remote_butler.server._config import ButlerServerConfig, RepositoryConfig, mock_config
16from lsst.daf.butler.remote_butler.server._dependencies import (
17 auth_delegated_token_dependency,
18 butler_factory_dependency,
19 reset_dependency_caches,
20 user_name_dependency,
21)
22from lsst.resources import ResourcePath
23from lsst.resources.s3utils import clean_test_environment_for_s3, getS3Client
25from ..direct_butler import DirectButler
26from .hybrid_butler import HybridButler
27from .postgresql import TemporaryPostgresInstance
28from .server_utils import add_auth_header_check_middleware
30try:
31 # moto v5
32 from moto import mock_aws # type: ignore
33except ImportError:
34 # moto v4 and earlier
35 from moto import mock_s3 as mock_aws # type: ignore
37__all__ = ("TEST_REPOSITORY_NAME", "TestServerInstance", "create_test_server")
40TEST_REPOSITORY_NAME = "testrepo"
43@dataclass(frozen=True)
44class TestServerInstance:
45 """Butler instances and other data associated with a temporary server
46 instance.
47 """
49 config_file_path: str
50 """Path to the Butler config file used by the server."""
51 client: TestClient
52 """HTTPX client connected to the temporary server."""
53 remote_butler: RemoteButler
54 """`RemoteButler` connected to the temporary server."""
55 remote_butler_without_error_propagation: RemoteButler
56 """`RemoteButler` connected to the temporary server.
58 By default, the TestClient instance raises any unhandled exceptions
59 from the server as if they had originated in the client to ease debugging.
60 However, this can make it appear that error propagation is working
61 correctly when in a real deployment the server exception would cause a 500
62 Internal Server Error. This instance of the butler is set up so that any
63 unhandled server exceptions do return a 500 status code."""
64 direct_butler: Butler
65 """`DirectButler` instance connected to the same repository as the
66 temporary server.
67 """
68 hybrid_butler: HybridButler
69 """`HybridButler` instance connected to the temporary server."""
70 app: FastAPI
71 """Butler server FastAPI app."""
74@contextmanager
75def create_test_server(
76 test_directory: str,
77 *,
78 postgres: TemporaryPostgresInstance | None = None,
79 server_config: ButlerServerConfig | None = None,
80) -> Iterator[TestServerInstance]:
81 """Create a temporary Butler server instance for testing.
83 Parameters
84 ----------
85 test_directory : `str`
86 Path to the ``tests/`` directory at the root of the repository,
87 containing Butler test configuration files.
88 postgres : `TemporaryPostgresInstance` | `None`
89 If provided, the Butler server will use this postgres database
90 instance. If no postgres instance is specified, the server will use a
91 a SQLite database.
92 server_config : `ButlerServerConfig`, optional
93 Configuration to use for the Butler server.
95 Returns
96 -------
97 instance : `TestServerInstance`
98 Object containing Butler instances connected to the server and
99 associated information.
100 """
101 # Set up a mock S3 environment using Moto. Moto also monkeypatches the
102 # `requests` library so that any HTTP requests to presigned S3 URLs get
103 # redirected to the mocked S3.
104 # Note that all files are stored in memory.
105 with clean_test_environment_for_s3():
106 with mock_aws():
107 base_config_path = os.path.join(test_directory, "config/basic/server.yaml")
108 # Create S3 buckets used for the datastore in server.yaml.
109 for bucket in ["mutable-bucket", "immutable-bucket"]:
110 getS3Client().create_bucket(Bucket=bucket)
112 config = Config(base_config_path)
113 if postgres is not None:
114 postgres.patch_butler_config(config)
116 with TemporaryDirectory() as root, mock_config(server_config) as server_config:
117 Butler.makeRepo(root, config=config, forceConfigRoot=False)
118 config_file_path = os.path.join(root, "butler.yaml")
120 server_config.repositories = {
121 TEST_REPOSITORY_NAME: RepositoryConfig(
122 config_uri=config_file_path, authorized_groups=["*"]
123 )
124 }
125 reset_dependency_caches()
127 app = create_app()
128 if server_config.authentication == "rubin_science_platform":
129 add_auth_header_check_middleware(app)
130 _add_root_exception_handler(app)
131 # Override the server's Butler initialization to point at our
132 # test repo
133 server_butler_factory = LabeledButlerFactory({TEST_REPOSITORY_NAME: config_file_path})
134 # DirectButler has a dimension_record_cache object that
135 # maintains a complete set of dimension records for governor
136 # dimensions. These values change infrequently and are needed
137 # for almost every DirectButler operation, so the complete set
138 # is downloaded the first time a record is needed.
139 #
140 # On the server it would be expensive to do this for every
141 # request's new DirectButler instance, so normally these are
142 # loaded once, the first time a repository is accessed. This
143 # is a problem for unit tests because they typically manipulate
144 # instrument records etc during setup. So configure the
145 # factory to disable this preloading and re-fetch the records
146 # as needed.
147 server_butler_factory._preload_unsafe_direct_butler_caches = False
148 app.dependency_overrides[butler_factory_dependency] = lambda: server_butler_factory
149 # In an actual deployment, these headers would be provided by
150 # the Gafaelfawr ingress.
151 app.dependency_overrides[user_name_dependency] = lambda: "mock-username"
152 app.dependency_overrides[auth_delegated_token_dependency] = lambda: "mock-delegated-token"
154 direct_butler = Butler.from_config(config_file_path, writeable=True)
155 assert isinstance(direct_butler, DirectButler)
156 # Using TestClient in a context manager ensures that it uses
157 # the same async event loop for all requests -- otherwise it
158 # starts a new one on each request.
159 with TestClient(app) as client, direct_butler, closing(server_butler_factory):
160 remote_butler = _make_remote_butler(client)
161 hybrid_butler = HybridButler(remote_butler, direct_butler)
163 client_without_error_propagation = TestClient(app, raise_server_exceptions=False)
164 remote_butler_without_error_propagation = _make_remote_butler(
165 client_without_error_propagation
166 )
168 yield TestServerInstance(
169 config_file_path=config_file_path,
170 client=client,
171 direct_butler=direct_butler,
172 remote_butler=remote_butler,
173 remote_butler_without_error_propagation=remote_butler_without_error_propagation,
174 hybrid_butler=hybrid_butler,
175 app=app,
176 )
179def _make_remote_butler(client: TestClient) -> RemoteButler:
180 config_endpoint = f"https://test.example/api/butler/repo/{TEST_REPOSITORY_NAME}/butler.yaml"
181 config_json = client.get(config_endpoint).read()
182 config = Config(json.loads(config_json))
183 config.configFile = ResourcePath(config_endpoint)
184 butler_config = ButlerConfig(config)
185 remote_butler_factory = RemoteButlerFactory.create_factory_from_config(butler_config, client)
186 return remote_butler_factory.create_butler_for_access_token("fake-access-token")
189class UnhandledServerError(Exception):
190 """Raised for unhandled exceptions within the server that would result in a
191 500 Internal Server Error in a real deployment. This allows us to tell the
192 difference between exceptions being propagated intentionally, and those
193 just bubbling up implicitly from the server to the client.
195 The FastAPI TestClient by default passes unhandled exceptions up from the
196 server to the client. This is useful behavior for unit testing because it
197 gives you traceability from the test to the problem in the server code.
198 However, because RemoteButler is in some ways just a proxy for the
199 server-side Butler, we raise similar exceptions on the client and server
200 side. Thus the default TestClient behavior can mask missing error-handling
201 logic.
202 """
205def _add_root_exception_handler(app: FastAPI) -> None:
206 @app.exception_handler(Exception)
207 async def convert_exception_types(request: Request, exc: Exception) -> None:
208 raise UnhandledServerError("Unhandled server exception") from exc