Coverage for tests/test_remote_butler.py: 56%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import unittest 

30from unittest.mock import patch 

31 

32from pydantic import ValidationError 

33 

34from lsst.daf.butler import Butler 

35from lsst.daf.butler._exceptions import UnknownButlerUserError 

36from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ( 

37 determine_destination_for_retrieved_artifact, 

38) 

39from lsst.daf.butler.registry import RegistryConfig 

40from lsst.daf.butler.registry.tests import RegistryTests 

41from lsst.daf.butler.tests.postgresql import TemporaryPostgresInstance, setup_postgres_test_db 

42from lsst.resources import ResourcePath 

43 

44try: 

45 import httpx 

46 

47 from lsst.daf.butler.remote_butler import ButlerServerError, RemoteButler 

48 

49 remote_butler_import_fail_message = "" 

50except ImportError as e: 

51 # httpx is not available in rubin-env yet, so skip these tests if it's not 

52 # available 

53 RemoteButler = None 

54 remote_butler_import_fail_message = str(e) 

55 

56try: 

57 from lsst.daf.butler.tests.server import create_test_server 

58 

59 server_import_fail_message = "" 

60except ImportError as e: 

61 create_test_server = None 

62 server_import_fail_message = str(e) 

63 

64TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

65 

66 

67@unittest.skipIf( 

68 RemoteButler is None, f"Remote butler can not be imported: {remote_butler_import_fail_message}" 

69) 

70class RemoteButlerConfigTests(unittest.TestCase): 

71 """Test construction of RemoteButler via Butler()""" 

72 

73 def test_bad_config(self): 

74 with self.assertRaises(ValidationError): 

75 Butler({"cls": "lsst.daf.butler.remote_butler.RemoteButler", "remote_butler": {"url": "!"}}) 

76 

77 

78@unittest.skipIf( 

79 create_test_server is None, f"Server dependencies not installed: {server_import_fail_message}" 

80) 

81class RemoteButlerErrorHandlingTests(unittest.TestCase): 

82 """Test RemoteButler error handling.""" 

83 

84 def setUp(self): 

85 server_instance = self.enterContext(create_test_server(TESTDIR)) 

86 self.butler = server_instance.remote_butler 

87 self.mock = self.enterContext(patch.object(self.butler._connection._client, "send")) 

88 

89 def _mock_error_response(self, content: str) -> None: 

90 self.mock.return_value = httpx.Response( 

91 status_code=422, content=content, request=httpx.Request("GET", "/") 

92 ) 

93 

94 def test_internal_server_error(self): 

95 self.mock.side_effect = httpx.HTTPError("unhandled error") 

96 with self.assertRaises(ButlerServerError): 

97 self.butler.get_dataset_type("int") 

98 

99 def test_unknown_error_type(self): 

100 self.mock.return_value = httpx.Response( 

101 status_code=422, json={"error_type": "not a known error type", "detail": "an error happened"} 

102 ) 

103 with self.assertRaises(UnknownButlerUserError): 

104 self.butler.get_dataset_type("int") 

105 

106 def test_non_json_error(self): 

107 # Server returns a non-JSON body with an error 

108 self._mock_error_response("notvalidjson") 

109 with self.assertRaises(ButlerServerError): 

110 self.butler.get_dataset_type("int") 

111 

112 def test_malformed_error(self): 

113 # Server returns JSON, but not in the expected format. 

114 self._mock_error_response("{}") 

115 with self.assertRaises(ButlerServerError): 

116 self.butler.get_dataset_type("int") 

117 

118 

119class RemoteButlerMiscTests(unittest.TestCase): 

120 """Test miscellaneous RemoteButler functionality.""" 

121 

122 def test_retrieve_artifacts_security(self): 

123 # Make sure that the function used to determine output file paths for 

124 # retrieveArtifacts throws if a malicious server tries to escape its 

125 # destination directory. 

126 with self.assertRaisesRegex(ValueError, "^File path attempts to escape destination directory"): 

127 determine_destination_for_retrieved_artifact( 

128 ResourcePath("output_directory/"), 

129 ResourcePath("../something.txt", forceAbsolute=False), 

130 preserve_path=True, 

131 ) 

132 

133 # Make sure all paths are forced to relative paths, even if the server 

134 # sends an absolute path. 

135 self.assertEqual( 

136 determine_destination_for_retrieved_artifact( 

137 ResourcePath("/tmp/output_directory/"), 

138 ResourcePath("file:///not/relative.txt"), 

139 preserve_path=True, 

140 ), 

141 ResourcePath("/tmp/output_directory/not/relative.txt"), 

142 ) 

143 

144 # Test prefixing. 

145 self.assertEqual( 

146 determine_destination_for_retrieved_artifact( 

147 ResourcePath("/tmp/output_directory/"), 

148 ResourcePath("file:///not/relative.txt"), 

149 preserve_path=False, 

150 prefix="prefix-", 

151 ), 

152 ResourcePath("/tmp/output_directory/prefix-relative.txt"), 

153 ) 

154 

155 

156class RemoteButlerRegistryTests(RegistryTests): 

157 """Tests for RemoteButler's `Registry` shim.""" 

158 

159 postgres: TemporaryPostgresInstance | None 

160 

161 def setUp(self): 

162 self.server_instance = self.enterContext(create_test_server(TESTDIR, postgres=self.postgres)) 

163 

164 @classmethod 

165 def getDataDir(cls) -> str: 

166 return os.path.join(TESTDIR, "data", "registry") 

167 

168 def make_butler(self, registry_config: RegistryConfig | None = None) -> Butler: 

169 return self.server_instance.hybrid_butler 

170 

171 def testBasicTransaction(self): 

172 # RemoteButler will never support transactions. 

173 pass 

174 

175 def testNestedTransaction(self): 

176 # RemoteButler will never support transactions. 

177 pass 

178 

179 def testOpaque(self): 

180 # This tests an internal implementation detail that isn't exposed to 

181 # the client side. 

182 pass 

183 

184 def testCollectionChainPrependConcurrency(self): 

185 # This tests an implementation detail that requires access to the 

186 # collection manager object. 

187 pass 

188 

189 def testCollectionChainReplaceConcurrency(self): 

190 # This tests an implementation detail that requires access to the 

191 # collection manager object. 

192 pass 

193 

194 def testCollectionChainRemoveConcurrency(self): 

195 # This tests an implementation detail that requires access to the 

196 # collection manager object. 

197 pass 

198 

199 def testAttributeManager(self): 

200 # Tests a non-public API that isn't relevant on the client side. 

201 pass 

202 

203 

204@unittest.skipIf( 

205 create_test_server is None, f"Server dependencies not installed: {server_import_fail_message}" 

206) 

207class RemoteButlerSqliteRegistryTests(RemoteButlerRegistryTests, unittest.TestCase): 

208 """Tests for RemoteButler's registry shim, with a SQLite DB backing the 

209 server. 

210 """ 

211 

212 postgres = None 

213 

214 

215@unittest.skipIf( 

216 create_test_server is None, f"Server dependencies not installed: {server_import_fail_message}" 

217) 

218class RemoteButlerPostgresRegistryTests(RemoteButlerRegistryTests, unittest.TestCase): 

219 """Tests for RemoteButler's registry shim, with a Postgres DB backing the 

220 server. 

221 """ 

222 

223 @classmethod 

224 def setUpClass(cls): 

225 cls.postgres = cls.enterClassContext(setup_postgres_test_db()) 

226 super().setUpClass() 

227 

228 def testSkipCalibs(self): 

229 if self.postgres.server_major_version() < 16: 

230 # TODO DM-44875: This test currently fails for older Postgres. 

231 self.skipTest("TODO DM-44875") 

232 return super().testSkipCalibs() 

233 

234 

235if __name__ == "__main__": 

236 unittest.main()