Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26import urllib.parse 

27 

28try: 

29 import boto3 

30 import botocore 

31 from moto import mock_s3 

32except ImportError: 

33 boto3 = None 

34 

35 def mock_s3(cls): 

36 """A no-op decorator in case moto mock_s3 can not be imported. 

37 """ 

38 return cls 

39 

40from lsst.daf.butler import ButlerURI 

41from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials, 

42 unsetAwsEnvCredentials) 

43 

44TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

45 

46 

47class FileURITestCase(unittest.TestCase): 

48 """Concrete tests for local files""" 

49 

50 def setUp(self): 

51 # Use a local tempdir because on macOS the temp dirs use symlinks 

52 # so relsymlink gets quite confused. 

53 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR) 

54 

55 def tearDown(self): 

56 shutil.rmtree(self.tmpdir, ignore_errors=True) 

57 

58 def testFile(self): 

59 file = os.path.join(self.tmpdir, "test.txt") 

60 uri = ButlerURI(file) 

61 self.assertFalse(uri.exists(), f"{uri} should not exist") 

62 self.assertEqual(uri.ospath, file) 

63 

64 content = "abcdefghijklmnopqrstuv\n" 

65 uri.write(content.encode()) 

66 self.assertTrue(os.path.exists(file), "File should exist locally") 

67 self.assertTrue(uri.exists(), f"{uri} should now exist") 

68 self.assertEqual(uri.read().decode(), content) 

69 

70 def testRelative(self): 

71 """Check that we can get subpaths back from two URIs""" 

72 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

73 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

74 

75 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

76 

77 not_child = ButlerURI("/a/b/dir1/file.txt") 

78 self.assertFalse(not_child.relative_to(parent)) 

79 

80 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

81 self.assertFalse(child.relative_to(not_directory)) 

82 

83 # Relative URIs 

84 parent = ButlerURI("a/b/", forceAbsolute=False) 

85 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

86 self.assertFalse(child.scheme) 

87 self.assertEqual(child.relative_to(parent), "c/d.txt") 

88 

89 # File URI and schemeless URI 

90 parent = ButlerURI("file:/a/b/c/") 

91 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

92 

93 # If the child is relative and the parent is absolute we assume 

94 # that the child is a child of the parent unless it uses ".." 

95 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

96 

97 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

98 self.assertFalse(child.relative_to(parent)) 

99 

100 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

101 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

102 

103 def testMkdir(self): 

104 tmpdir = ButlerURI(self.tmpdir) 

105 newdir = tmpdir.join("newdir/seconddir") 

106 newdir.mkdir() 

107 self.assertTrue(newdir.exists()) 

108 newfile = newdir.join("temp.txt") 

109 newfile.write("Data".encode()) 

110 self.assertTrue(newfile.exists()) 

111 

112 def testTransfer(self): 

113 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

114 content = "Content is some content\nwith something to say\n\n" 

115 src.write(content.encode()) 

116 

117 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

118 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

119 dest.transfer_from(src, transfer=mode) 

120 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

121 

122 with open(dest.ospath, "r") as fh: 

123 new_content = fh.read() 

124 self.assertEqual(new_content, content) 

125 

126 if mode in ("symlink", "relsymlink"): 

127 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

128 

129 with self.assertRaises(FileExistsError): 

130 dest.transfer_from(src, transfer=mode) 

131 

132 dest.transfer_from(src, transfer=mode, overwrite=True) 

133 

134 os.remove(dest.ospath) 

135 

136 b = src.read() 

137 self.assertEqual(b.decode(), new_content) 

138 

139 nbytes = 10 

140 subset = src.read(size=nbytes) 

141 self.assertEqual(len(subset), nbytes) 

142 self.assertEqual(subset.decode(), content[:nbytes]) 

143 

144 with self.assertRaises(ValueError): 

145 src.transfer_from(src, transfer="unknown") 

146 

147 def testResource(self): 

148 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

149 self.assertTrue(u.exists(), f"Check {u} exists") 

150 

151 content = u.read().decode() 

152 self.assertTrue(content.startswith("datastore:")) 

153 

154 truncated = u.read(size=9).decode() 

155 self.assertEqual(truncated, "datastore") 

156 

157 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

158 self.assertTrue(u.exists(), f"Check directory {d} exists") 

159 

160 j = d.join("datastore.yaml") 

161 self.assertEqual(u, j) 

162 self.assertFalse(j.dirLike) 

163 self.assertFalse(d.join("not-there.yaml").exists()) 

164 

165 def testEscapes(self): 

166 """Special characters in file paths""" 

167 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

168 self.assertFalse(src.scheme) 

169 src.write(b"Some content") 

170 self.assertTrue(src.exists()) 

171 

172 # Use the internal API to force to a file 

173 file = src._force_to_file() 

174 self.assertTrue(file.exists()) 

175 self.assertIn("???", file.ospath) 

176 self.assertNotIn("???", file.path) 

177 

178 file.updateFile("tests??.txt") 

179 self.assertNotIn("??.txt", file.path) 

180 file.write(b"Other content") 

181 self.assertEqual(file.read(), b"Other content") 

182 

183 src.updateFile("tests??.txt") 

184 self.assertIn("??.txt", src.path) 

185 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

186 

187 # File URI and schemeless URI 

188 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

189 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

190 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

191 

192 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

193 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

194 

195 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

196 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

197 

198 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

199 

200 # Schemeless so should not quote 

201 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

202 self.assertIn("???", dir.ospath) 

203 self.assertIn("???", dir.path) 

204 self.assertFalse(dir.scheme) 

205 

206 # dir.join() morphs into a file scheme 

207 new = dir.join("test_j.txt") 

208 self.assertIn("???", new.ospath, f"Checking {new}") 

209 new.write(b"Content") 

210 

211 new2name = "###/test??.txt" 

212 new2 = dir.join(new2name) 

213 self.assertIn("???", new2.ospath) 

214 new2.write(b"Content") 

215 self.assertTrue(new2.ospath.endswith(new2name)) 

216 self.assertEqual(new.read(), new2.read()) 

217 

218 fdir = dir._force_to_file() 

219 self.assertNotIn("???", fdir.path) 

220 self.assertIn("???", fdir.ospath) 

221 self.assertEqual(fdir.scheme, "file") 

222 fnew = dir.join("test_jf.txt") 

223 fnew.write(b"Content") 

224 

225 fnew2 = fdir.join(new2name) 

226 fnew2.write(b"Content") 

227 self.assertTrue(fnew2.ospath.endswith(new2name)) 

228 self.assertNotIn("###", fnew2.path) 

229 

230 self.assertEqual(fnew.read(), fnew2.read()) 

231 

232 # Test that children relative to schemeless and file schemes 

233 # still return the same unquoted name 

234 self.assertEqual(fnew2.relative_to(fdir), new2name) 

235 self.assertEqual(fnew2.relative_to(dir), new2name) 

236 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

237 self.assertEqual(new2.relative_to(dir), new2name) 

238 

239 # Check for double quoting 

240 plus_path = "/a/b/c+d/" 

241 with self.assertLogs(level="WARNING"): 

242 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

243 self.assertEqual(uri.ospath, plus_path) 

244 

245 

246@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

247@mock_s3 

248class S3URITestCase(unittest.TestCase): 

249 """Tests involving S3""" 

250 

251 bucketName = "any_bucket" 

252 """Bucket name to use in tests""" 

253 

254 def setUp(self): 

255 # Local test directory 

256 self.tmpdir = tempfile.mkdtemp() 

257 

258 # set up some fake credentials if they do not exist 

259 self.usingDummyCredentials = setAwsEnvCredentials() 

260 

261 # MOTO needs to know that we expect Bucket bucketname to exist 

262 s3 = boto3.resource("s3") 

263 s3.create_bucket(Bucket=self.bucketName) 

264 

265 def tearDown(self): 

266 s3 = boto3.resource("s3") 

267 bucket = s3.Bucket(self.bucketName) 

268 try: 

269 bucket.objects.all().delete() 

270 except botocore.exceptions.ClientError as e: 

271 if e.response["Error"]["Code"] == "404": 

272 # the key was not reachable - pass 

273 pass 

274 else: 

275 raise 

276 

277 bucket = s3.Bucket(self.bucketName) 

278 bucket.delete() 

279 

280 # unset any potentially set dummy credentials 

281 if self.usingDummyCredentials: 

282 unsetAwsEnvCredentials() 

283 

284 shutil.rmtree(self.tmpdir, ignore_errors=True) 

285 

286 def makeS3Uri(self, path): 

287 return f"s3://{self.bucketName}/{path}" 

288 

289 def testTransfer(self): 

290 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

291 content = "Content is some content\nwith something to say\n\n" 

292 src.write(content.encode()) 

293 

294 dest = ButlerURI(self.makeS3Uri("test.txt")) 

295 self.assertFalse(dest.exists()) 

296 dest.transfer_from(src, transfer="copy") 

297 self.assertTrue(dest.exists()) 

298 

299 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

300 dest2.transfer_from(dest, transfer="copy") 

301 self.assertTrue(dest2.exists()) 

302 

303 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

304 local.transfer_from(dest2, transfer="copy") 

305 with open(local.ospath, "r") as fd: 

306 new_content = fd.read() 

307 self.assertEqual(new_content, content) 

308 

309 with self.assertRaises(ValueError): 

310 dest2.transfer_from(local, transfer="symlink") 

311 

312 b = dest.read() 

313 self.assertEqual(b.decode(), new_content) 

314 

315 nbytes = 10 

316 subset = dest.read(size=nbytes) 

317 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

318 self.assertEqual(subset.decode(), content[:nbytes]) 

319 

320 with self.assertRaises(FileExistsError): 

321 dest.transfer_from(src, transfer="copy") 

322 

323 dest.transfer_from(src, transfer="copy", overwrite=True) 

324 

325 def testWrite(self): 

326 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

327 content = "abcdefghijklmnopqrstuv\n" 

328 s3write.write(content.encode()) 

329 self.assertEqual(s3write.read().decode(), content) 

330 

331 def testRelative(self): 

332 """Check that we can get subpaths back from two URIs""" 

333 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

334 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

335 

336 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

337 

338 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

339 self.assertFalse(not_child.relative_to(parent)) 

340 

341 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

342 self.assertFalse(child.relative_to(not_s3)) 

343 

344 def testQuoting(self): 

345 """Check that quoting works.""" 

346 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

347 subpath = "rootdir/dir1+/file?.txt" 

348 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

349 

350 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

351 self.assertEqual(child.basename(), "file?.txt") 

352 self.assertEqual(child.relativeToPathRoot, subpath) 

353 self.assertIn("%", child.path) 

354 self.assertEqual(child.unquoted_path, "/" + subpath) 

355 

356 

357if __name__ == "__main__": 357 ↛ 358line 357 didn't jump to line 358, because the condition on line 357 was never true

358 unittest.main()