Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import unittest 

25import urllib.parse 

26import responses 

27 

28try: 

29 import boto3 

30 import botocore 

31 from moto import mock_s3 

32except ImportError: 

33 boto3 = None 

34 

35 def mock_s3(cls): 

36 """A no-op decorator in case moto mock_s3 can not be imported. 

37 """ 

38 return cls 

39 

40from lsst.daf.butler import ButlerURI 

41from lsst.daf.butler.core._butlerUri.s3utils import (setAwsEnvCredentials, 

42 unsetAwsEnvCredentials) 

43from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FileURITestCase(unittest.TestCase): 

49 """Concrete tests for local files""" 

50 

51 def setUp(self): 

52 # Use a local tempdir because on macOS the temp dirs use symlinks 

53 # so relsymlink gets quite confused. 

54 self.tmpdir = makeTestTempDir(TESTDIR) 

55 

56 def tearDown(self): 

57 removeTestTempDir(self.tmpdir) 

58 

59 def testFile(self): 

60 file = os.path.join(self.tmpdir, "test.txt") 

61 uri = ButlerURI(file) 

62 self.assertFalse(uri.exists(), f"{uri} should not exist") 

63 self.assertEqual(uri.ospath, file) 

64 

65 content = "abcdefghijklmnopqrstuv\n" 

66 uri.write(content.encode()) 

67 self.assertTrue(os.path.exists(file), "File should exist locally") 

68 self.assertTrue(uri.exists(), f"{uri} should now exist") 

69 self.assertEqual(uri.read().decode(), content) 

70 

71 def testRelative(self): 

72 """Check that we can get subpaths back from two URIs""" 

73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

75 

76 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

77 

78 not_child = ButlerURI("/a/b/dir1/file.txt") 

79 self.assertFalse(not_child.relative_to(parent)) 

80 

81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

82 self.assertFalse(child.relative_to(not_directory)) 

83 

84 # Relative URIs 

85 parent = ButlerURI("a/b/", forceAbsolute=False) 

86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

87 self.assertFalse(child.scheme) 

88 self.assertEqual(child.relative_to(parent), "c/d.txt") 

89 

90 # File URI and schemeless URI 

91 parent = ButlerURI("file:/a/b/c/") 

92 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

93 

94 # If the child is relative and the parent is absolute we assume 

95 # that the child is a child of the parent unless it uses ".." 

96 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

97 

98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

99 self.assertFalse(child.relative_to(parent)) 

100 

101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

102 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

103 

104 def testEnvVar(self): 

105 """Test that environment variables are expanded.""" 

106 

107 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

108 uri = ButlerURI("${MY_TEST_DIR}/d.txt") 

109 self.assertEqual(uri.path, "/a/b/c/d.txt") 

110 self.assertEqual(uri.scheme, "file") 

111 

112 # This will not expand 

113 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

114 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

115 self.assertFalse(uri.scheme) 

116 

117 def testMkdir(self): 

118 tmpdir = ButlerURI(self.tmpdir) 

119 newdir = tmpdir.join("newdir/seconddir") 

120 newdir.mkdir() 

121 self.assertTrue(newdir.exists()) 

122 newfile = newdir.join("temp.txt") 

123 newfile.write("Data".encode()) 

124 self.assertTrue(newfile.exists()) 

125 

126 def testTransfer(self): 

127 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

128 content = "Content is some content\nwith something to say\n\n" 

129 src.write(content.encode()) 

130 

131 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

132 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

133 dest.transfer_from(src, transfer=mode) 

134 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

135 

136 with open(dest.ospath, "r") as fh: 

137 new_content = fh.read() 

138 self.assertEqual(new_content, content) 

139 

140 if mode in ("symlink", "relsymlink"): 

141 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

142 

143 with self.assertRaises(FileExistsError): 

144 dest.transfer_from(src, transfer=mode) 

145 

146 dest.transfer_from(src, transfer=mode, overwrite=True) 

147 

148 os.remove(dest.ospath) 

149 

150 b = src.read() 

151 self.assertEqual(b.decode(), new_content) 

152 

153 nbytes = 10 

154 subset = src.read(size=nbytes) 

155 self.assertEqual(len(subset), nbytes) 

156 self.assertEqual(subset.decode(), content[:nbytes]) 

157 

158 with self.assertRaises(ValueError): 

159 src.transfer_from(src, transfer="unknown") 

160 

161 def testResource(self): 

162 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

163 self.assertTrue(u.exists(), f"Check {u} exists") 

164 

165 content = u.read().decode() 

166 self.assertTrue(content.startswith("datastore:")) 

167 

168 truncated = u.read(size=9).decode() 

169 self.assertEqual(truncated, "datastore") 

170 

171 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

172 self.assertTrue(u.exists(), f"Check directory {d} exists") 

173 

174 j = d.join("datastore.yaml") 

175 self.assertEqual(u, j) 

176 self.assertFalse(j.dirLike) 

177 self.assertFalse(d.join("not-there.yaml").exists()) 

178 

179 def testEscapes(self): 

180 """Special characters in file paths""" 

181 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

182 self.assertFalse(src.scheme) 

183 src.write(b"Some content") 

184 self.assertTrue(src.exists()) 

185 

186 # Use the internal API to force to a file 

187 file = src._force_to_file() 

188 self.assertTrue(file.exists()) 

189 self.assertIn("???", file.ospath) 

190 self.assertNotIn("???", file.path) 

191 

192 file.updateFile("tests??.txt") 

193 self.assertNotIn("??.txt", file.path) 

194 file.write(b"Other content") 

195 self.assertEqual(file.read(), b"Other content") 

196 

197 src.updateFile("tests??.txt") 

198 self.assertIn("??.txt", src.path) 

199 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

200 

201 # File URI and schemeless URI 

202 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

203 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

204 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

205 

206 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

207 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

208 

209 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

210 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

211 

212 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

213 

214 # Schemeless so should not quote 

215 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

216 self.assertIn("???", dir.ospath) 

217 self.assertIn("???", dir.path) 

218 self.assertFalse(dir.scheme) 

219 

220 # dir.join() morphs into a file scheme 

221 new = dir.join("test_j.txt") 

222 self.assertIn("???", new.ospath, f"Checking {new}") 

223 new.write(b"Content") 

224 

225 new2name = "###/test??.txt" 

226 new2 = dir.join(new2name) 

227 self.assertIn("???", new2.ospath) 

228 new2.write(b"Content") 

229 self.assertTrue(new2.ospath.endswith(new2name)) 

230 self.assertEqual(new.read(), new2.read()) 

231 

232 fdir = dir._force_to_file() 

233 self.assertNotIn("???", fdir.path) 

234 self.assertIn("???", fdir.ospath) 

235 self.assertEqual(fdir.scheme, "file") 

236 fnew = dir.join("test_jf.txt") 

237 fnew.write(b"Content") 

238 

239 fnew2 = fdir.join(new2name) 

240 fnew2.write(b"Content") 

241 self.assertTrue(fnew2.ospath.endswith(new2name)) 

242 self.assertNotIn("###", fnew2.path) 

243 

244 self.assertEqual(fnew.read(), fnew2.read()) 

245 

246 # Test that children relative to schemeless and file schemes 

247 # still return the same unquoted name 

248 self.assertEqual(fnew2.relative_to(fdir), new2name) 

249 self.assertEqual(fnew2.relative_to(dir), new2name) 

250 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

251 self.assertEqual(new2.relative_to(dir), new2name) 

252 

253 # Check for double quoting 

254 plus_path = "/a/b/c+d/" 

255 with self.assertLogs(level="WARNING"): 

256 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

257 self.assertEqual(uri.ospath, plus_path) 

258 

259 # Check that # is not escaped for schemeless URIs 

260 hash_path = "/a/b#/c&d#xyz" 

261 hpos = hash_path.rfind("#") 

262 uri = ButlerURI(hash_path) 

263 self.assertEqual(uri.ospath, hash_path[:hpos]) 

264 self.assertEqual(uri.fragment, hash_path[hpos + 1:]) 

265 

266 

267@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

268@mock_s3 

269class S3URITestCase(unittest.TestCase): 

270 """Tests involving S3""" 

271 

272 bucketName = "any_bucket" 

273 """Bucket name to use in tests""" 

274 

275 def setUp(self): 

276 # Local test directory 

277 self.tmpdir = makeTestTempDir(TESTDIR) 

278 

279 # set up some fake credentials if they do not exist 

280 self.usingDummyCredentials = setAwsEnvCredentials() 

281 

282 # MOTO needs to know that we expect Bucket bucketname to exist 

283 s3 = boto3.resource("s3") 

284 s3.create_bucket(Bucket=self.bucketName) 

285 

286 def tearDown(self): 

287 s3 = boto3.resource("s3") 

288 bucket = s3.Bucket(self.bucketName) 

289 try: 

290 bucket.objects.all().delete() 

291 except botocore.exceptions.ClientError as e: 

292 if e.response["Error"]["Code"] == "404": 

293 # the key was not reachable - pass 

294 pass 

295 else: 

296 raise 

297 

298 bucket = s3.Bucket(self.bucketName) 

299 bucket.delete() 

300 

301 # unset any potentially set dummy credentials 

302 if self.usingDummyCredentials: 

303 unsetAwsEnvCredentials() 

304 

305 shutil.rmtree(self.tmpdir, ignore_errors=True) 

306 

307 def makeS3Uri(self, path): 

308 return f"s3://{self.bucketName}/{path}" 

309 

310 def testTransfer(self): 

311 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

312 content = "Content is some content\nwith something to say\n\n" 

313 src.write(content.encode()) 

314 

315 dest = ButlerURI(self.makeS3Uri("test.txt")) 

316 self.assertFalse(dest.exists()) 

317 dest.transfer_from(src, transfer="copy") 

318 self.assertTrue(dest.exists()) 

319 

320 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

321 dest2.transfer_from(dest, transfer="copy") 

322 self.assertTrue(dest2.exists()) 

323 

324 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

325 local.transfer_from(dest2, transfer="copy") 

326 with open(local.ospath, "r") as fd: 

327 new_content = fd.read() 

328 self.assertEqual(new_content, content) 

329 

330 with self.assertRaises(ValueError): 

331 dest2.transfer_from(local, transfer="symlink") 

332 

333 b = dest.read() 

334 self.assertEqual(b.decode(), new_content) 

335 

336 nbytes = 10 

337 subset = dest.read(size=nbytes) 

338 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

339 self.assertEqual(subset.decode(), content[:nbytes]) 

340 

341 with self.assertRaises(FileExistsError): 

342 dest.transfer_from(src, transfer="copy") 

343 

344 dest.transfer_from(src, transfer="copy", overwrite=True) 

345 

346 def testWrite(self): 

347 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

348 content = "abcdefghijklmnopqrstuv\n" 

349 s3write.write(content.encode()) 

350 self.assertEqual(s3write.read().decode(), content) 

351 

352 def testRelative(self): 

353 """Check that we can get subpaths back from two URIs""" 

354 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

355 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

356 

357 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

358 

359 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

360 self.assertFalse(not_child.relative_to(parent)) 

361 

362 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

363 self.assertFalse(child.relative_to(not_s3)) 

364 

365 def testQuoting(self): 

366 """Check that quoting works.""" 

367 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

368 subpath = "rootdir/dir1+/file?.txt" 

369 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

370 

371 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

372 self.assertEqual(child.basename(), "file?.txt") 

373 self.assertEqual(child.relativeToPathRoot, subpath) 

374 self.assertIn("%", child.path) 

375 self.assertEqual(child.unquoted_path, "/" + subpath) 

376 

377 

378# Mock required environment variables during tests 

379@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

380 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join( 

381 TESTDIR, "config/testConfigs/webdav/token"), 

382 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"}) 

383class WebdavURITestCase(unittest.TestCase): 

384 

385 def setUp(self): 

386 serverRoot = "www.not-exists.orgx" 

387 existingFolderName = "existingFolder" 

388 existingFileName = "existingFile" 

389 notExistingFileName = "notExistingFile" 

390 

391 self.baseURL = ButlerURI( 

392 f"https://{serverRoot}", forceDirectory=True) 

393 self.existingFileButlerURI = ButlerURI( 

394 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

395 self.notExistingFileButlerURI = ButlerURI( 

396 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

397 self.existingFolderButlerURI = ButlerURI( 

398 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

399 self.notExistingFolderButlerURI = ButlerURI( 

400 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

401 

402 # Need to declare the options 

403 responses.add(responses.OPTIONS, 

404 self.baseURL.geturl(), 

405 status=200, headers={"DAV": "1,2,3"}) 

406 

407 # Used by ButlerHttpURI.exists() 

408 responses.add(responses.HEAD, 

409 self.existingFileButlerURI.geturl(), 

410 status=200, headers={'Content-Length': '1024'}) 

411 responses.add(responses.HEAD, 

412 self.notExistingFileButlerURI.geturl(), 

413 status=404) 

414 

415 # Used by ButlerHttpURI.read() 

416 responses.add(responses.GET, 

417 self.existingFileButlerURI.geturl(), 

418 status=200, 

419 body=str.encode("It works!")) 

420 responses.add(responses.GET, 

421 self.notExistingFileButlerURI.geturl(), 

422 status=404) 

423 

424 # Used by ButlerHttpURI.write() 

425 responses.add(responses.PUT, 

426 self.existingFileButlerURI.geturl(), 

427 status=201) 

428 

429 # Used by ButlerHttpURI.transfer_from() 

430 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

431 method="COPY", 

432 headers={"Destination": self.existingFileButlerURI.geturl()}, 

433 status=201)) 

434 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

435 method="COPY", 

436 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

437 status=201)) 

438 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

439 method="MOVE", 

440 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

441 status=201)) 

442 

443 # Used by ButlerHttpURI.remove() 

444 responses.add(responses.DELETE, 

445 self.existingFileButlerURI.geturl(), 

446 status=200) 

447 responses.add(responses.DELETE, 

448 self.notExistingFileButlerURI.geturl(), 

449 status=404) 

450 

451 # Used by ButlerHttpURI.mkdir() 

452 responses.add(responses.HEAD, 

453 self.existingFolderButlerURI.geturl(), 

454 status=200, headers={'Content-Length': '1024'}) 

455 responses.add(responses.HEAD, 

456 self.baseURL.geturl(), 

457 status=200, headers={'Content-Length': '1024'}) 

458 responses.add(responses.HEAD, 

459 self.notExistingFolderButlerURI.geturl(), 

460 status=404) 

461 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

462 method="MKCOL", 

463 status=201)) 

464 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

465 method="MKCOL", 

466 status=403)) 

467 

468 @responses.activate 

469 def testExists(self): 

470 

471 self.assertTrue(self.existingFileButlerURI.exists()) 

472 self.assertFalse(self.notExistingFileButlerURI.exists()) 

473 

474 @responses.activate 

475 def testRemove(self): 

476 

477 self.assertIsNone(self.existingFileButlerURI.remove()) 

478 with self.assertRaises(FileNotFoundError): 

479 self.notExistingFileButlerURI.remove() 

480 

481 @responses.activate 

482 def testMkdir(self): 

483 

484 # The mock means that we can't check this now exists 

485 self.notExistingFolderButlerURI.mkdir() 

486 

487 # This should do nothing 

488 self.existingFolderButlerURI.mkdir() 

489 

490 with self.assertRaises(ValueError): 

491 self.notExistingFileButlerURI.mkdir() 

492 

493 @responses.activate 

494 def testRead(self): 

495 

496 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

497 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

498 with self.assertRaises(FileNotFoundError): 

499 self.notExistingFileButlerURI.read() 

500 

501 @responses.activate 

502 def testWrite(self): 

503 

504 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

505 with self.assertRaises(FileExistsError): 

506 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

507 

508 @responses.activate 

509 def testTransfer(self): 

510 

511 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

512 src=self.existingFileButlerURI)) 

513 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

514 src=self.existingFileButlerURI, 

515 transfer="move")) 

516 with self.assertRaises(FileExistsError): 

517 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

518 with self.assertRaises(ValueError): 

519 self.notExistingFileButlerURI.transfer_from( 

520 src=self.existingFileButlerURI, 

521 transfer="unsupported") 

522 

523 def testParent(self): 

524 

525 self.assertEqual(self.existingFolderButlerURI.geturl(), 

526 self.notExistingFileButlerURI.parent().geturl()) 

527 self.assertEqual(self.baseURL.geturl(), 

528 self.baseURL.parent().geturl()) 

529 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

530 self.existingFileButlerURI.dirname().geturl()) 

531 

532 

533if __name__ == "__main__": 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true

534 unittest.main()