Coverage for tests/test_server.py: 34%
86 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 10:59 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 10:59 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import os.path
29import unittest
30import uuid
32try:
33 # Failing to import any of these should disable the tests.
34 from fastapi.testclient import TestClient
35 from lsst.daf.butler.remote_butler import RemoteButler
36 from lsst.daf.butler.remote_butler.server import Factory, app
37 from lsst.daf.butler.remote_butler.server._dependencies import factory_dependency
38except ImportError:
39 TestClient = None
40 app = None
42from unittest.mock import patch
44from lsst.daf.butler import Butler, DataCoordinate, DatasetRef, MissingDatasetTypeError, StorageClassFactory
45from lsst.daf.butler.tests import DatastoreMock
46from lsst.daf.butler.tests.utils import MetricTestRepo, makeTestTempDir, removeTestTempDir
47from lsst.resources.http import HttpResourcePath
49TESTDIR = os.path.abspath(os.path.dirname(__file__))
52def _make_remote_butler(http_client):
53 return RemoteButler(
54 config={
55 "remote_butler": {
56 # This URL is ignored because we override the HTTP client, but
57 # must be valid to satisfy the config validation
58 "url": "https://test.example"
59 }
60 },
61 http_client=http_client,
62 )
65@unittest.skipIf(TestClient is None or app is None, "FastAPI not installed.")
66class ButlerClientServerTestCase(unittest.TestCase):
67 """Test for Butler client/server."""
69 @classmethod
70 def setUpClass(cls):
71 cls.storageClassFactory = StorageClassFactory()
73 # First create a butler and populate it.
74 cls.root = makeTestTempDir(TESTDIR)
75 cls.repo = MetricTestRepo(root=cls.root, configFile=os.path.join(TESTDIR, "config/basic/butler.yaml"))
76 # Override the server's Butler initialization to point at our test repo
77 server_butler = Butler.from_config(cls.root, writeable=True)
79 # Not yet testing butler.get()
80 DatastoreMock.apply(server_butler)
82 def create_factory_dependency():
83 return Factory(butler=server_butler)
85 app.dependency_overrides[factory_dependency] = create_factory_dependency
87 # Set up the RemoteButler that will connect to the server
88 cls.client = TestClient(app)
89 cls.client.base_url = "http://test.example/api/butler/"
90 cls.butler = _make_remote_butler(cls.client)
92 # Populate the test server.
93 server_butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
94 server_butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "datasets-uuid.yaml"))
96 @classmethod
97 def tearDownClass(cls):
98 del app.dependency_overrides[factory_dependency]
99 removeTestTempDir(cls.root)
101 def test_health_check(self):
102 response = self.client.get("/")
103 self.assertEqual(response.status_code, 200)
104 self.assertEqual(response.json()["name"], "butler")
106 def test_simple(self):
107 response = self.client.get("/api/butler/v1/universe")
108 self.assertEqual(response.status_code, 200)
109 self.assertIn("namespace", response.json())
111 def test_remote_butler(self):
112 universe = self.butler.dimensions
113 self.assertEqual(universe.namespace, "daf_butler")
114 self.assertFalse(self.butler.isWriteable())
116 def test_get_dataset_type(self):
117 bias_type = self.butler.get_dataset_type("bias")
118 self.assertEqual(bias_type.name, "bias")
120 with self.assertRaises(MissingDatasetTypeError):
121 self.butler.get_dataset_type("not_bias")
123 def test_find_dataset(self):
124 storage_class = self.storageClassFactory.getStorageClass("Exposure")
126 ref = self.butler.find_dataset("bias", collections="imported_g", detector=1, instrument="Cam1")
127 self.assertIsInstance(ref, DatasetRef)
128 self.assertEqual(ref.id, uuid.UUID("e15ab039-bc8b-4135-87c5-90902a7c0b22"))
129 self.assertFalse(ref.dataId.hasRecords())
131 # Try again with variation of parameters.
132 ref_new = self.butler.find_dataset(
133 "bias",
134 {"detector": 1},
135 collections="imported_g",
136 instrument="Cam1",
137 dimension_records=True,
138 )
139 self.assertEqual(ref_new, ref)
140 self.assertTrue(ref_new.dataId.hasRecords())
142 ref_new = self.butler.find_dataset(
143 ref.datasetType,
144 DataCoordinate.standardize(detector=1, instrument="Cam1", universe=self.butler.dimensions),
145 collections="imported_g",
146 storage_class=storage_class,
147 )
148 self.assertEqual(ref_new, ref)
150 ref2 = self.butler.get_dataset(ref.id)
151 self.assertEqual(ref2, ref)
153 # Use detector name to find it.
154 ref3 = self.butler.find_dataset(
155 ref.datasetType,
156 collections="imported_g",
157 instrument="Cam1",
158 full_name="Aa",
159 )
160 self.assertEqual(ref2, ref3)
162 # Try expanded refs.
163 self.assertFalse(ref.dataId.hasRecords())
164 expanded = self.butler.get_dataset(ref.id, dimension_records=True)
165 self.assertTrue(expanded.dataId.hasRecords())
167 # The test datasets are all Exposure so storage class conversion
168 # can not be tested until we fix that. For now at least test the
169 # code paths.
170 bias = self.butler.get_dataset(ref.id, storage_class=storage_class)
171 self.assertEqual(bias.datasetType.storageClass, storage_class)
173 # Unknown dataset should not fail.
174 self.assertIsNone(self.butler.get_dataset(uuid.uuid4()))
175 self.assertIsNone(self.butler.get_dataset(uuid.uuid4(), storage_class="NumpyArray"))
177 def test_instantiate_via_butler_http_search(self):
178 """Ensure that the primary Butler constructor's automatic search logic
179 correctly locates and reads the configuration file and ends up with a
180 RemoteButler pointing to the correct URL
181 """
183 # This is kind of a fragile test. Butler's search logic does a lot of
184 # manipulations involving creating new ResourcePaths, and ResourcePath
185 # doesn't use httpx so we can't easily inject the TestClient in there.
186 # We don't have an actual valid HTTP URL to give to the constructor
187 # because the test instance of the server is accessed via ASGI.
188 #
189 # Instead we just monkeypatch the HTTPResourcePath 'read' method and
190 # hope that all ResourcePath HTTP reads during construction are going
191 # to the server under test.
192 def override_read(http_resource_path):
193 return self.client.get(http_resource_path.geturl()).content
195 with patch.object(HttpResourcePath, "read", override_read):
196 butler = Butler("https://test.example/api/butler")
197 assert isinstance(butler, RemoteButler)
198 assert str(butler._config.remote_butler.url) == "https://test.example/api/butler/"
201if __name__ == "__main__":
202 unittest.main()