Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26import urllib.parse 

27import responses 

28 

29try: 

30 import boto3 

31 import botocore 

32 from moto import mock_s3 

33except ImportError: 

34 boto3 = None 

35 

36 def mock_s3(cls): 

37 """A no-op decorator in case moto mock_s3 can not be imported. 

38 """ 

39 return cls 

40 

41from lsst.daf.butler import ButlerURI 

42from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials, 

43 unsetAwsEnvCredentials) 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FileURITestCase(unittest.TestCase): 

49 """Concrete tests for local files""" 

50 

51 def setUp(self): 

52 # Use a local tempdir because on macOS the temp dirs use symlinks 

53 # so relsymlink gets quite confused. 

54 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR) 

55 

56 def tearDown(self): 

57 shutil.rmtree(self.tmpdir, ignore_errors=True) 

58 

59 def testFile(self): 

60 file = os.path.join(self.tmpdir, "test.txt") 

61 uri = ButlerURI(file) 

62 self.assertFalse(uri.exists(), f"{uri} should not exist") 

63 self.assertEqual(uri.ospath, file) 

64 

65 content = "abcdefghijklmnopqrstuv\n" 

66 uri.write(content.encode()) 

67 self.assertTrue(os.path.exists(file), "File should exist locally") 

68 self.assertTrue(uri.exists(), f"{uri} should now exist") 

69 self.assertEqual(uri.read().decode(), content) 

70 

71 def testRelative(self): 

72 """Check that we can get subpaths back from two URIs""" 

73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

75 

76 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

77 

78 not_child = ButlerURI("/a/b/dir1/file.txt") 

79 self.assertFalse(not_child.relative_to(parent)) 

80 

81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

82 self.assertFalse(child.relative_to(not_directory)) 

83 

84 # Relative URIs 

85 parent = ButlerURI("a/b/", forceAbsolute=False) 

86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

87 self.assertFalse(child.scheme) 

88 self.assertEqual(child.relative_to(parent), "c/d.txt") 

89 

90 # File URI and schemeless URI 

91 parent = ButlerURI("file:/a/b/c/") 

92 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

93 

94 # If the child is relative and the parent is absolute we assume 

95 # that the child is a child of the parent unless it uses ".." 

96 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

97 

98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

99 self.assertFalse(child.relative_to(parent)) 

100 

101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

102 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

103 

104 def testMkdir(self): 

105 tmpdir = ButlerURI(self.tmpdir) 

106 newdir = tmpdir.join("newdir/seconddir") 

107 newdir.mkdir() 

108 self.assertTrue(newdir.exists()) 

109 newfile = newdir.join("temp.txt") 

110 newfile.write("Data".encode()) 

111 self.assertTrue(newfile.exists()) 

112 

113 def testTransfer(self): 

114 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

115 content = "Content is some content\nwith something to say\n\n" 

116 src.write(content.encode()) 

117 

118 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

119 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

120 dest.transfer_from(src, transfer=mode) 

121 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

122 

123 with open(dest.ospath, "r") as fh: 

124 new_content = fh.read() 

125 self.assertEqual(new_content, content) 

126 

127 if mode in ("symlink", "relsymlink"): 

128 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

129 

130 with self.assertRaises(FileExistsError): 

131 dest.transfer_from(src, transfer=mode) 

132 

133 dest.transfer_from(src, transfer=mode, overwrite=True) 

134 

135 os.remove(dest.ospath) 

136 

137 b = src.read() 

138 self.assertEqual(b.decode(), new_content) 

139 

140 nbytes = 10 

141 subset = src.read(size=nbytes) 

142 self.assertEqual(len(subset), nbytes) 

143 self.assertEqual(subset.decode(), content[:nbytes]) 

144 

145 with self.assertRaises(ValueError): 

146 src.transfer_from(src, transfer="unknown") 

147 

148 def testResource(self): 

149 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

150 self.assertTrue(u.exists(), f"Check {u} exists") 

151 

152 content = u.read().decode() 

153 self.assertTrue(content.startswith("datastore:")) 

154 

155 truncated = u.read(size=9).decode() 

156 self.assertEqual(truncated, "datastore") 

157 

158 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

159 self.assertTrue(u.exists(), f"Check directory {d} exists") 

160 

161 j = d.join("datastore.yaml") 

162 self.assertEqual(u, j) 

163 self.assertFalse(j.dirLike) 

164 self.assertFalse(d.join("not-there.yaml").exists()) 

165 

166 def testEscapes(self): 

167 """Special characters in file paths""" 

168 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

169 self.assertFalse(src.scheme) 

170 src.write(b"Some content") 

171 self.assertTrue(src.exists()) 

172 

173 # Use the internal API to force to a file 

174 file = src._force_to_file() 

175 self.assertTrue(file.exists()) 

176 self.assertIn("???", file.ospath) 

177 self.assertNotIn("???", file.path) 

178 

179 file.updateFile("tests??.txt") 

180 self.assertNotIn("??.txt", file.path) 

181 file.write(b"Other content") 

182 self.assertEqual(file.read(), b"Other content") 

183 

184 src.updateFile("tests??.txt") 

185 self.assertIn("??.txt", src.path) 

186 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

187 

188 # File URI and schemeless URI 

189 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

190 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

191 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

192 

193 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

194 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

195 

196 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

197 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

198 

199 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

200 

201 # Schemeless so should not quote 

202 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

203 self.assertIn("???", dir.ospath) 

204 self.assertIn("???", dir.path) 

205 self.assertFalse(dir.scheme) 

206 

207 # dir.join() morphs into a file scheme 

208 new = dir.join("test_j.txt") 

209 self.assertIn("???", new.ospath, f"Checking {new}") 

210 new.write(b"Content") 

211 

212 new2name = "###/test??.txt" 

213 new2 = dir.join(new2name) 

214 self.assertIn("???", new2.ospath) 

215 new2.write(b"Content") 

216 self.assertTrue(new2.ospath.endswith(new2name)) 

217 self.assertEqual(new.read(), new2.read()) 

218 

219 fdir = dir._force_to_file() 

220 self.assertNotIn("???", fdir.path) 

221 self.assertIn("???", fdir.ospath) 

222 self.assertEqual(fdir.scheme, "file") 

223 fnew = dir.join("test_jf.txt") 

224 fnew.write(b"Content") 

225 

226 fnew2 = fdir.join(new2name) 

227 fnew2.write(b"Content") 

228 self.assertTrue(fnew2.ospath.endswith(new2name)) 

229 self.assertNotIn("###", fnew2.path) 

230 

231 self.assertEqual(fnew.read(), fnew2.read()) 

232 

233 # Test that children relative to schemeless and file schemes 

234 # still return the same unquoted name 

235 self.assertEqual(fnew2.relative_to(fdir), new2name) 

236 self.assertEqual(fnew2.relative_to(dir), new2name) 

237 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

238 self.assertEqual(new2.relative_to(dir), new2name) 

239 

240 # Check for double quoting 

241 plus_path = "/a/b/c+d/" 

242 with self.assertLogs(level="WARNING"): 

243 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

244 self.assertEqual(uri.ospath, plus_path) 

245 

246 

247@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

248@mock_s3 

249class S3URITestCase(unittest.TestCase): 

250 """Tests involving S3""" 

251 

252 bucketName = "any_bucket" 

253 """Bucket name to use in tests""" 

254 

255 def setUp(self): 

256 # Local test directory 

257 self.tmpdir = tempfile.mkdtemp() 

258 

259 # set up some fake credentials if they do not exist 

260 self.usingDummyCredentials = setAwsEnvCredentials() 

261 

262 # MOTO needs to know that we expect Bucket bucketname to exist 

263 s3 = boto3.resource("s3") 

264 s3.create_bucket(Bucket=self.bucketName) 

265 

266 def tearDown(self): 

267 s3 = boto3.resource("s3") 

268 bucket = s3.Bucket(self.bucketName) 

269 try: 

270 bucket.objects.all().delete() 

271 except botocore.exceptions.ClientError as e: 

272 if e.response["Error"]["Code"] == "404": 

273 # the key was not reachable - pass 

274 pass 

275 else: 

276 raise 

277 

278 bucket = s3.Bucket(self.bucketName) 

279 bucket.delete() 

280 

281 # unset any potentially set dummy credentials 

282 if self.usingDummyCredentials: 

283 unsetAwsEnvCredentials() 

284 

285 shutil.rmtree(self.tmpdir, ignore_errors=True) 

286 

287 def makeS3Uri(self, path): 

288 return f"s3://{self.bucketName}/{path}" 

289 

290 def testTransfer(self): 

291 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

292 content = "Content is some content\nwith something to say\n\n" 

293 src.write(content.encode()) 

294 

295 dest = ButlerURI(self.makeS3Uri("test.txt")) 

296 self.assertFalse(dest.exists()) 

297 dest.transfer_from(src, transfer="copy") 

298 self.assertTrue(dest.exists()) 

299 

300 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

301 dest2.transfer_from(dest, transfer="copy") 

302 self.assertTrue(dest2.exists()) 

303 

304 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

305 local.transfer_from(dest2, transfer="copy") 

306 with open(local.ospath, "r") as fd: 

307 new_content = fd.read() 

308 self.assertEqual(new_content, content) 

309 

310 with self.assertRaises(ValueError): 

311 dest2.transfer_from(local, transfer="symlink") 

312 

313 b = dest.read() 

314 self.assertEqual(b.decode(), new_content) 

315 

316 nbytes = 10 

317 subset = dest.read(size=nbytes) 

318 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

319 self.assertEqual(subset.decode(), content[:nbytes]) 

320 

321 with self.assertRaises(FileExistsError): 

322 dest.transfer_from(src, transfer="copy") 

323 

324 dest.transfer_from(src, transfer="copy", overwrite=True) 

325 

326 def testWrite(self): 

327 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

328 content = "abcdefghijklmnopqrstuv\n" 

329 s3write.write(content.encode()) 

330 self.assertEqual(s3write.read().decode(), content) 

331 

332 def testRelative(self): 

333 """Check that we can get subpaths back from two URIs""" 

334 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

335 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

336 

337 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

338 

339 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

340 self.assertFalse(not_child.relative_to(parent)) 

341 

342 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

343 self.assertFalse(child.relative_to(not_s3)) 

344 

345 def testQuoting(self): 

346 """Check that quoting works.""" 

347 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

348 subpath = "rootdir/dir1+/file?.txt" 

349 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

350 

351 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

352 self.assertEqual(child.basename(), "file?.txt") 

353 self.assertEqual(child.relativeToPathRoot, subpath) 

354 self.assertIn("%", child.path) 

355 self.assertEqual(child.unquoted_path, "/" + subpath) 

356 

357 

358class WebdavURITestCase(unittest.TestCase): 

359 

360 def setUp(self): 

361 serverRoot = "www.not-exists.orgx" 

362 existingFolderName = "existingFolder" 

363 existingFileName = "existingFile" 

364 notExistingFileName = "notExistingFile" 

365 

366 self.existingFileButlerURI = ButlerURI( 

367 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

368 self.notExistingFileButlerURI = ButlerURI( 

369 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

370 self.existingFolderButlerURI = ButlerURI( 

371 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

372 self.notExistingFolderButlerURI = ButlerURI( 

373 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

374 

375 # Need to declare the options 

376 responses.add(responses.OPTIONS, 

377 self.existingFileButlerURI.geturl(), 

378 headers={'not': '1024'}, status=200) 

379 responses.add(responses.OPTIONS, 

380 self.notExistingFileButlerURI.geturl(), 

381 headers={'not': '1024'}, status=200) 

382 responses.add(responses.OPTIONS, 

383 self.notExistingFolderButlerURI.geturl(), 

384 headers={'not': '1024'}, status=200) 

385 responses.add(responses.OPTIONS, 

386 self.existingFolderButlerURI.geturl(), 

387 headers={'not': '1024'}, status=200) 

388 

389 # Used by ButlerHttpURI.exists() 

390 responses.add(responses.HEAD, 

391 self.existingFileButlerURI.geturl(), 

392 status=200, headers={'Content-Length': '1024'}) 

393 responses.add(responses.HEAD, 

394 self.notExistingFileButlerURI.geturl(), 

395 status=404) 

396 

397 # Used by ButlerHttpURI.read() 

398 responses.add(responses.GET, 

399 self.existingFileButlerURI.geturl(), 

400 status=200, 

401 body=str.encode("It works!")) 

402 responses.add(responses.GET, 

403 self.notExistingFileButlerURI.geturl(), 

404 status=404) 

405 

406 # Used by ButlerHttpURI.write() 

407 responses.add(responses.PUT, 

408 self.existingFileButlerURI.geturl(), 

409 status=200) 

410 

411 # Used by ButlerHttpURI.transfer_from() 

412 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

413 method="COPY", 

414 headers={"Destination": self.existingFileButlerURI.geturl()}, 

415 status=200)) 

416 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

417 method="COPY", 

418 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

419 status=200)) 

420 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

421 method="MOVE", 

422 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

423 status=200)) 

424 

425 # Used by ButlerHttpURI.remove() 

426 responses.add(responses.DELETE, 

427 self.existingFileButlerURI.geturl(), 

428 status=200) 

429 responses.add(responses.DELETE, 

430 self.notExistingFileButlerURI.geturl(), 

431 status=404) 

432 

433 # Used by ButlerHttpURI.mkdir() 

434 responses.add(responses.HEAD, 

435 self.existingFolderButlerURI.geturl(), 

436 status=200, headers={'Content-Length': '1024'}) 

437 responses.add(responses.HEAD, 

438 self.notExistingFolderButlerURI.geturl(), 

439 status=404) 

440 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

441 method="MKCOL", 

442 status=201)) 

443 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

444 method="MKCOL", 

445 status=403)) 

446 

447 @responses.activate 

448 def testExists(self): 

449 

450 self.assertTrue(self.existingFileButlerURI.exists()) 

451 self.assertFalse(self.notExistingFileButlerURI.exists()) 

452 

453 @responses.activate 

454 def testRemove(self): 

455 

456 self.assertIsNone(self.existingFileButlerURI.remove()) 

457 with self.assertRaises(FileNotFoundError): 

458 self.notExistingFileButlerURI.remove() 

459 

460 @responses.activate 

461 def testMkdir(self): 

462 

463 # The mock means that we can't check this now exists 

464 self.notExistingFolderButlerURI.mkdir() 

465 

466 # This should do nothing 

467 self.existingFolderButlerURI.mkdir() 

468 

469 with self.assertRaises(ValueError): 

470 self.notExistingFileButlerURI.mkdir() 

471 

472 @responses.activate 

473 def testRead(self): 

474 

475 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

476 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

477 with self.assertRaises(FileNotFoundError): 

478 self.notExistingFileButlerURI.read() 

479 

480 @responses.activate 

481 def testWrite(self): 

482 

483 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

484 with self.assertRaises(FileExistsError): 

485 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

486 

487 @responses.activate 

488 def testTransfer(self): 

489 

490 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

491 src=self.existingFileButlerURI)) 

492 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

493 src=self.existingFileButlerURI, 

494 transfer="move")) 

495 with self.assertRaises(FileExistsError): 

496 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

497 with self.assertRaises(ValueError): 

498 self.notExistingFileButlerURI.transfer_from( 

499 src=self.existingFileButlerURI, 

500 transfer="unsupported") 

501 

502 

503if __name__ == "__main__": 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 unittest.main()