Coverage for tests/test_file.py: 21%

133 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-13 09:59 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import contextlib 

13import os 

14import pathlib 

15import unittest 

16import unittest.mock 

17import urllib.parse 

18 

19from lsst.resources import ResourcePath, ResourcePathExpression 

20from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

21 

22TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

23 

24 

25class SimpleTestCase(unittest.TestCase): 

26 """Basic tests for file URIs.""" 

27 

28 def test_instance(self): 

29 for example in ( 

30 "xxx", 

31 ResourcePath("xxx"), 

32 pathlib.Path("xxx"), 

33 urllib.parse.urlparse("file:///xxx"), 

34 ): 

35 self.assertIsInstance(example, ResourcePathExpression) 

36 

37 for example in ({1, 2, 3}, 42, self): 

38 self.assertNotIsInstance(example, ResourcePathExpression) 

39 

40 

41class FileTestCase(GenericTestCase, unittest.TestCase): 

42 """File-specific generic test cases.""" 

43 

44 scheme = "file" 

45 netloc = "localhost" 

46 

47 def test_env_var(self): 

48 """Test that environment variables are expanded.""" 

49 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

50 uri = ResourcePath("${MY_TEST_DIR}/d.txt") 

51 self.assertEqual(uri.path, "/a/b/c/d.txt") 

52 self.assertEqual(uri.scheme, "file") 

53 

54 # This will not expand 

55 uri = ResourcePath("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

56 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

57 self.assertFalse(uri.scheme) 

58 

59 def test_ospath(self): 

60 """File URIs have ospath property.""" 

61 file = ResourcePath(self._make_uri("a/test.txt")) 

62 self.assertEqual(file.ospath, "/a/test.txt") 

63 self.assertEqual(file.ospath, file.path) 

64 

65 # A Schemeless URI can take unquoted files but will be quoted 

66 # when it becomes a file URI. 

67 something = "/a#/???.txt" 

68 file = ResourcePath(something, forceAbsolute=True) 

69 self.assertEqual(file.scheme, "file") 

70 self.assertEqual(file.ospath, something, "From URI: {file}") 

71 self.assertNotIn("???", file.path) 

72 

73 def test_path_lib(self): 

74 """File URIs can be created from pathlib.""" 

75 file = ResourcePath(self._make_uri("a/test.txt")) 

76 

77 path_file = pathlib.Path(file.ospath) 

78 from_path = ResourcePath(path_file) 

79 self.assertEqual(from_path.ospath, file.ospath) 

80 

81 def test_schemeless_root(self): 

82 root = ResourcePath(self._make_uri("/root")) 

83 via_root = ResourcePath("b.txt", root=root) 

84 self.assertEqual(via_root.ospath, "/root/b.txt") 

85 

86 

87TEST_UMASK = 0o0333 

88 

89 

90class FileReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase): 

91 """File tests involving reading and writing of data.""" 

92 

93 scheme = "file" 

94 netloc = "localhost" 

95 testdir = TESTDIR 

96 transfer_modes = ("move", "copy", "link", "hardlink", "symlink", "relsymlink") 

97 

98 def test_transfer_identical(self): 

99 """Test overwrite of identical files. 

100 

101 Only relevant for local files. 

102 """ 

103 dir1 = self.tmpdir.join("dir1", forceDirectory=True) 

104 dir1.mkdir() 

105 self.assertTrue(dir1.exists()) 

106 dir2 = self.tmpdir.join("dir2", forceDirectory=True) 

107 # A symlink can't include a trailing slash. 

108 dir2_ospath = dir2.ospath 

109 if dir2_ospath.endswith("/"): 

110 dir2_ospath = dir2_ospath[:-1] 

111 os.symlink(dir1.ospath, dir2_ospath) 

112 

113 # Write a test file. 

114 src_file = dir1.join("test.txt") 

115 content = "0123456" 

116 src_file.write(content.encode()) 

117 

118 # Construct URI to destination that should be identical. 

119 dest_file = dir2.join("test.txt") 

120 self.assertTrue(dest_file.exists()) 

121 self.assertNotEqual(src_file, dest_file) 

122 

123 # Transfer it over itself. 

124 dest_file.transfer_from(src_file, transfer="symlink", overwrite=True) 

125 new_content = dest_file.read().decode() 

126 self.assertEqual(content, new_content) 

127 

128 def test_local_temporary(self): 

129 """Create temporary local file if no prefix specified.""" 

130 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

131 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

132 self.assertTrue(tmp.isabs(), f"uri: {tmp}") 

133 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

134 tmp.write(b"abcd") 

135 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

136 self.assertTrue(tmp.isTemporary) 

137 self.assertTrue(tmp.isLocal) 

138 

139 # If we now ask for a local form of this temporary file 

140 # it should still be temporary and it should not be deleted 

141 # on exit. 

142 with tmp.as_local() as loc: 

143 self.assertEqual(tmp, loc) 

144 self.assertTrue(loc.isTemporary) 

145 self.assertTrue(tmp.exists()) 

146 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

147 

148 def test_transfers_from_local(self): 

149 """Extra tests for local transfers.""" 

150 target = self.tmpdir.join("a/target.txt") 

151 with ResourcePath.temporary_uri() as tmp: 

152 tmp.write(b"") 

153 self.assertTrue(tmp.isTemporary) 

154 

155 # Symlink transfers for temporary resources should 

156 # trigger a debug message. 

157 for transfer in ("symlink", "relsymlink"): 

158 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

159 target.transfer_from(tmp, transfer) 

160 target.remove() 

161 self.assertIn("Using a symlink for a temporary", "".join(cm.output)) 

162 

163 # Force the target directory to be created. 

164 target.transfer_from(tmp, "move") 

165 self.assertFalse(tmp.exists()) 

166 

167 # Temporary file now gone so transfer should not work. 

168 with self.assertRaises(FileNotFoundError): 

169 target.transfer_from(tmp, "move", overwrite=True) 

170 

171 def test_write_with_restrictive_umask(self): 

172 self._test_file_with_restrictive_umask(lambda target: target.write(b"123")) 

173 

174 def test_transfer_from_with_restrictive_umask(self): 

175 def cb(target): 

176 with ResourcePath.temporary_uri() as tmp: 

177 tmp.write(b"") 

178 target.transfer_from(tmp, "copy") 

179 

180 self._test_file_with_restrictive_umask(cb) 

181 

182 def test_mkdir_with_restrictive_umask(self): 

183 self._test_with_restrictive_umask(lambda target: target.mkdir()) 

184 

185 def test_temporary_uri_with_restrictive_umask(self): 

186 with _override_umask(TEST_UMASK): 

187 with ResourcePath.temporary_uri() as tmp: 

188 tmp.write(b"") 

189 self.assertTrue(tmp.exists()) 

190 

191 def _test_file_with_restrictive_umask(self, callback): 

192 def inner_cb(target): 

193 callback(target) 

194 

195 # Make sure the umask was respected for the file itself 

196 file_mode = os.stat(target.ospath).st_mode 

197 self.assertEqual(file_mode & TEST_UMASK, 0) 

198 

199 self._test_with_restrictive_umask(inner_cb) 

200 

201 def _test_with_restrictive_umask(self, callback): 

202 """Make sure that parent directories for a file can be created even if 

203 the user has set a process umask that restricts the write and traverse 

204 bits. 

205 """ 

206 with _override_umask(TEST_UMASK): 

207 target = self.tmpdir.join("a/b/target.txt") 

208 callback(target) 

209 self.assertTrue(target.exists()) 

210 

211 dir_b_path = os.path.dirname(target.ospath) 

212 dir_a_path = os.path.dirname(dir_b_path) 

213 for dir in [dir_a_path, dir_b_path]: 

214 # Make sure we only added the minimum permissions needed for it 

215 # to work (owner-write and owner-traverse) 

216 mode = os.stat(dir).st_mode 

217 self.assertEqual(mode & TEST_UMASK, 0o0300, f"Permissions incorrect for {dir}: {mode:o}") 

218 

219 

220@contextlib.contextmanager 

221def _override_umask(temp_umask): 

222 old = os.umask(temp_umask) 

223 try: 

224 yield 

225 finally: 

226 os.umask(old) 

227 

228 

229if __name__ == "__main__": 

230 unittest.main()