Coverage for tests/test_remote_butler.py: 54%

80 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-08 02:51 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import unittest 

30from unittest.mock import patch 

31 

32from lsst.daf.butler import Butler, Registry 

33from lsst.daf.butler._exceptions import UnknownButlerUserError 

34from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ( 

35 determine_destination_for_retrieved_artifact, 

36) 

37from lsst.daf.butler.registry.tests import RegistryTests 

38from lsst.resources import ResourcePath 

39from pydantic import ValidationError 

40 

41try: 

42 import httpx 

43 from lsst.daf.butler.remote_butler import ButlerServerError, RemoteButler 

44except ImportError: 

45 # httpx is not available in rubin-env yet, so skip these tests if it's not 

46 # available 

47 RemoteButler = None 

48 

49try: 

50 from lsst.daf.butler.tests.server import create_test_server 

51except ImportError: 

52 create_test_server = None 

53 

54TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

55 

56 

57@unittest.skipIf(RemoteButler is None, "httpx is not installed") 

58class RemoteButlerConfigTests(unittest.TestCase): 

59 """Test construction of RemoteButler via Butler()""" 

60 

61 def test_bad_config(self): 

62 with self.assertRaises(ValidationError): 

63 Butler({"cls": "lsst.daf.butler.remote_butler.RemoteButler", "remote_butler": {"url": "!"}}) 

64 

65 

66@unittest.skipIf(create_test_server is None, "Server dependencies not installed") 

67class RemoteButlerErrorHandlingTests(unittest.TestCase): 

68 """Test RemoteButler error handling.""" 

69 

70 def setUp(self): 

71 server_instance = self.enterContext(create_test_server(TESTDIR)) 

72 self.butler = server_instance.remote_butler 

73 self.mock = self.enterContext(patch.object(self.butler._connection._client, "request")) 

74 

75 def _mock_error_response(self, content: str) -> None: 

76 self.mock.return_value = httpx.Response( 

77 status_code=422, content=content, request=httpx.Request("GET", "/") 

78 ) 

79 

80 def test_internal_server_error(self): 

81 self.mock.side_effect = httpx.HTTPError("unhandled error") 

82 with self.assertRaises(ButlerServerError): 

83 self.butler.get_dataset_type("int") 

84 

85 def test_unknown_error_type(self): 

86 self.mock.return_value = httpx.Response( 

87 status_code=422, json={"error_type": "not a known error type", "detail": "an error happened"} 

88 ) 

89 with self.assertRaises(UnknownButlerUserError): 

90 self.butler.get_dataset_type("int") 

91 

92 def test_non_json_error(self): 

93 # Server returns a non-JSON body with an error 

94 self._mock_error_response("notvalidjson") 

95 with self.assertRaises(ButlerServerError): 

96 self.butler.get_dataset_type("int") 

97 

98 def test_malformed_error(self): 

99 # Server returns JSON, but not in the expected format. 

100 self._mock_error_response("{}") 

101 with self.assertRaises(ButlerServerError): 

102 self.butler.get_dataset_type("int") 

103 

104 

105class RemoteButlerMiscTests(unittest.TestCase): 

106 """Test miscellaneous RemoteButler functionality.""" 

107 

108 def test_retrieve_artifacts_security(self): 

109 # Make sure that the function used to determine output file paths for 

110 # retrieveArtifacts throws if a malicious server tries to escape its 

111 # destination directory. 

112 with self.assertRaisesRegex(ValueError, "^File path attempts to escape destination directory"): 

113 determine_destination_for_retrieved_artifact( 

114 ResourcePath("output_directory/"), 

115 ResourcePath("../something.txt", forceAbsolute=False), 

116 preserve_path=True, 

117 ) 

118 

119 # Make sure all paths are forced to relative paths, even if the server 

120 # sends an absolute path. 

121 self.assertEqual( 

122 determine_destination_for_retrieved_artifact( 

123 ResourcePath("/tmp/output_directory/"), 

124 ResourcePath("file:///not/relative.txt"), 

125 preserve_path=True, 

126 ), 

127 ResourcePath("/tmp/output_directory/not/relative.txt"), 

128 ) 

129 

130 

131@unittest.skipIf(create_test_server is None, "Server dependencies not installed.") 

132class RemoteButlerRegistryTests(RegistryTests, unittest.TestCase): 

133 """Tests for RemoteButler's `Registry` shim.""" 

134 

135 supportsCollectionRegex = False 

136 

137 # RemoteButler implements registry.query methods by forwarding to the new 

138 # query system, which doesn't have the same diagnostics as the old one 

139 # and also does not support query offset. 

140 supportsDetailedQueryExplain = False 

141 supportsQueryOffset = False 

142 supportsQueryGovernorValidation = False 

143 

144 def setUp(self): 

145 self.server_instance = self.enterContext(create_test_server(TESTDIR)) 

146 

147 @classmethod 

148 def getDataDir(cls) -> str: 

149 return os.path.join(TESTDIR, "data", "registry") 

150 

151 def makeRegistry(self, share_repo_with: Registry | None = None) -> Registry: 

152 if share_repo_with is None: 

153 return self.server_instance.hybrid_butler.registry 

154 else: 

155 return self.server_instance.hybrid_butler._clone().registry 

156 

157 def testBasicTransaction(self): 

158 # RemoteButler will never support transactions. 

159 pass 

160 

161 def testNestedTransaction(self): 

162 # RemoteButler will never support transactions. 

163 pass 

164 

165 def testOpaque(self): 

166 # This tests an internal implementation detail that isn't exposed to 

167 # the client side. 

168 pass 

169 

170 def testCollectionChainPrependConcurrency(self): 

171 # This tests an implementation detail that requires access to the 

172 # collection manager object. 

173 pass 

174 

175 def testCollectionChainReplaceConcurrency(self): 

176 # This tests an implementation detail that requires access to the 

177 # collection manager object. 

178 pass 

179 

180 def testAttributeManager(self): 

181 # Tests a non-public API that isn't relevant on the client side. 

182 pass 

183 

184 

185if __name__ == "__main__": 

186 unittest.main()