Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import unittest 

25import urllib.parse 

26import responses 

27 

28try: 

29 import boto3 

30 import botocore 

31 from moto import mock_s3 

32except ImportError: 

33 boto3 = None 

34 

35 def mock_s3(cls): 

36 """A no-op decorator in case moto mock_s3 can not be imported. 

37 """ 

38 return cls 

39 

40from lsst.daf.butler import ButlerURI 

41from lsst.daf.butler.core._butlerUri.s3utils import (setAwsEnvCredentials, 

42 unsetAwsEnvCredentials) 

43from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FileURITestCase(unittest.TestCase): 

49 """Concrete tests for local files""" 

50 

51 def setUp(self): 

52 # Use a local tempdir because on macOS the temp dirs use symlinks 

53 # so relsymlink gets quite confused. 

54 self.tmpdir = makeTestTempDir(TESTDIR) 

55 

56 def tearDown(self): 

57 removeTestTempDir(self.tmpdir) 

58 

59 def testFile(self): 

60 file = os.path.join(self.tmpdir, "test.txt") 

61 uri = ButlerURI(file) 

62 self.assertFalse(uri.exists(), f"{uri} should not exist") 

63 self.assertEqual(uri.ospath, file) 

64 

65 content = "abcdefghijklmnopqrstuv\n" 

66 uri.write(content.encode()) 

67 self.assertTrue(os.path.exists(file), "File should exist locally") 

68 self.assertTrue(uri.exists(), f"{uri} should now exist") 

69 self.assertEqual(uri.read().decode(), content) 

70 

71 def testRelative(self): 

72 """Check that we can get subpaths back from two URIs""" 

73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

75 

76 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

77 

78 not_child = ButlerURI("/a/b/dir1/file.txt") 

79 self.assertFalse(not_child.relative_to(parent)) 

80 

81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

82 self.assertFalse(child.relative_to(not_directory)) 

83 

84 # Relative URIs 

85 parent = ButlerURI("a/b/", forceAbsolute=False) 

86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

87 self.assertFalse(child.scheme) 

88 self.assertEqual(child.relative_to(parent), "c/d.txt") 

89 

90 # File URI and schemeless URI 

91 parent = ButlerURI("file:/a/b/c/") 

92 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

93 

94 # If the child is relative and the parent is absolute we assume 

95 # that the child is a child of the parent unless it uses ".." 

96 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

97 

98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

99 self.assertFalse(child.relative_to(parent)) 

100 

101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

102 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

103 

104 def testEnvVar(self): 

105 """Test that environment variables are expanded.""" 

106 

107 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

108 uri = ButlerURI("${MY_TEST_DIR}/d.txt") 

109 self.assertEqual(uri.path, "/a/b/c/d.txt") 

110 self.assertEqual(uri.scheme, "file") 

111 

112 # This will not expand 

113 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

114 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

115 self.assertFalse(uri.scheme) 

116 

117 def testMkdir(self): 

118 tmpdir = ButlerURI(self.tmpdir) 

119 newdir = tmpdir.join("newdir/seconddir") 

120 newdir.mkdir() 

121 self.assertTrue(newdir.exists()) 

122 newfile = newdir.join("temp.txt") 

123 newfile.write("Data".encode()) 

124 self.assertTrue(newfile.exists()) 

125 

126 def testTransfer(self): 

127 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

128 content = "Content is some content\nwith something to say\n\n" 

129 src.write(content.encode()) 

130 

131 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

132 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

133 dest.transfer_from(src, transfer=mode) 

134 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

135 

136 with open(dest.ospath, "r") as fh: 

137 new_content = fh.read() 

138 self.assertEqual(new_content, content) 

139 

140 if mode in ("symlink", "relsymlink"): 

141 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

142 

143 with self.assertRaises(FileExistsError): 

144 dest.transfer_from(src, transfer=mode) 

145 

146 dest.transfer_from(src, transfer=mode, overwrite=True) 

147 

148 os.remove(dest.ospath) 

149 

150 b = src.read() 

151 self.assertEqual(b.decode(), new_content) 

152 

153 nbytes = 10 

154 subset = src.read(size=nbytes) 

155 self.assertEqual(len(subset), nbytes) 

156 self.assertEqual(subset.decode(), content[:nbytes]) 

157 

158 with self.assertRaises(ValueError): 

159 src.transfer_from(src, transfer="unknown") 

160 

161 def testResource(self): 

162 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

163 self.assertTrue(u.exists(), f"Check {u} exists") 

164 

165 content = u.read().decode() 

166 self.assertTrue(content.startswith("datastore:")) 

167 

168 truncated = u.read(size=9).decode() 

169 self.assertEqual(truncated, "datastore") 

170 

171 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

172 self.assertTrue(u.exists(), f"Check directory {d} exists") 

173 

174 j = d.join("datastore.yaml") 

175 self.assertEqual(u, j) 

176 self.assertFalse(j.dirLike) 

177 self.assertFalse(d.join("not-there.yaml").exists()) 

178 

179 def testEscapes(self): 

180 """Special characters in file paths""" 

181 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

182 self.assertFalse(src.scheme) 

183 src.write(b"Some content") 

184 self.assertTrue(src.exists()) 

185 

186 # Use the internal API to force to a file 

187 file = src._force_to_file() 

188 self.assertTrue(file.exists()) 

189 self.assertIn("???", file.ospath) 

190 self.assertNotIn("???", file.path) 

191 

192 file.updateFile("tests??.txt") 

193 self.assertNotIn("??.txt", file.path) 

194 file.write(b"Other content") 

195 self.assertEqual(file.read(), b"Other content") 

196 

197 src.updateFile("tests??.txt") 

198 self.assertIn("??.txt", src.path) 

199 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

200 

201 # File URI and schemeless URI 

202 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

203 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

204 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

205 

206 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

207 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

208 

209 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

210 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

211 

212 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

213 

214 # Schemeless so should not quote 

215 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

216 self.assertIn("???", dir.ospath) 

217 self.assertIn("???", dir.path) 

218 self.assertFalse(dir.scheme) 

219 

220 # dir.join() morphs into a file scheme 

221 new = dir.join("test_j.txt") 

222 self.assertIn("???", new.ospath, f"Checking {new}") 

223 new.write(b"Content") 

224 

225 new2name = "###/test??.txt" 

226 new2 = dir.join(new2name) 

227 self.assertIn("???", new2.ospath) 

228 new2.write(b"Content") 

229 self.assertTrue(new2.ospath.endswith(new2name)) 

230 self.assertEqual(new.read(), new2.read()) 

231 

232 fdir = dir._force_to_file() 

233 self.assertNotIn("???", fdir.path) 

234 self.assertIn("???", fdir.ospath) 

235 self.assertEqual(fdir.scheme, "file") 

236 fnew = dir.join("test_jf.txt") 

237 fnew.write(b"Content") 

238 

239 fnew2 = fdir.join(new2name) 

240 fnew2.write(b"Content") 

241 self.assertTrue(fnew2.ospath.endswith(new2name)) 

242 self.assertNotIn("###", fnew2.path) 

243 

244 self.assertEqual(fnew.read(), fnew2.read()) 

245 

246 # Test that children relative to schemeless and file schemes 

247 # still return the same unquoted name 

248 self.assertEqual(fnew2.relative_to(fdir), new2name) 

249 self.assertEqual(fnew2.relative_to(dir), new2name) 

250 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

251 self.assertEqual(new2.relative_to(dir), new2name) 

252 

253 # Check for double quoting 

254 plus_path = "/a/b/c+d/" 

255 with self.assertLogs(level="WARNING"): 

256 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

257 self.assertEqual(uri.ospath, plus_path) 

258 

259 

260@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

261@mock_s3 

262class S3URITestCase(unittest.TestCase): 

263 """Tests involving S3""" 

264 

265 bucketName = "any_bucket" 

266 """Bucket name to use in tests""" 

267 

268 def setUp(self): 

269 # Local test directory 

270 self.tmpdir = makeTestTempDir(TESTDIR) 

271 

272 # set up some fake credentials if they do not exist 

273 self.usingDummyCredentials = setAwsEnvCredentials() 

274 

275 # MOTO needs to know that we expect Bucket bucketname to exist 

276 s3 = boto3.resource("s3") 

277 s3.create_bucket(Bucket=self.bucketName) 

278 

279 def tearDown(self): 

280 s3 = boto3.resource("s3") 

281 bucket = s3.Bucket(self.bucketName) 

282 try: 

283 bucket.objects.all().delete() 

284 except botocore.exceptions.ClientError as e: 

285 if e.response["Error"]["Code"] == "404": 

286 # the key was not reachable - pass 

287 pass 

288 else: 

289 raise 

290 

291 bucket = s3.Bucket(self.bucketName) 

292 bucket.delete() 

293 

294 # unset any potentially set dummy credentials 

295 if self.usingDummyCredentials: 

296 unsetAwsEnvCredentials() 

297 

298 shutil.rmtree(self.tmpdir, ignore_errors=True) 

299 

300 def makeS3Uri(self, path): 

301 return f"s3://{self.bucketName}/{path}" 

302 

303 def testTransfer(self): 

304 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

305 content = "Content is some content\nwith something to say\n\n" 

306 src.write(content.encode()) 

307 

308 dest = ButlerURI(self.makeS3Uri("test.txt")) 

309 self.assertFalse(dest.exists()) 

310 dest.transfer_from(src, transfer="copy") 

311 self.assertTrue(dest.exists()) 

312 

313 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

314 dest2.transfer_from(dest, transfer="copy") 

315 self.assertTrue(dest2.exists()) 

316 

317 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

318 local.transfer_from(dest2, transfer="copy") 

319 with open(local.ospath, "r") as fd: 

320 new_content = fd.read() 

321 self.assertEqual(new_content, content) 

322 

323 with self.assertRaises(ValueError): 

324 dest2.transfer_from(local, transfer="symlink") 

325 

326 b = dest.read() 

327 self.assertEqual(b.decode(), new_content) 

328 

329 nbytes = 10 

330 subset = dest.read(size=nbytes) 

331 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

332 self.assertEqual(subset.decode(), content[:nbytes]) 

333 

334 with self.assertRaises(FileExistsError): 

335 dest.transfer_from(src, transfer="copy") 

336 

337 dest.transfer_from(src, transfer="copy", overwrite=True) 

338 

339 def testWrite(self): 

340 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

341 content = "abcdefghijklmnopqrstuv\n" 

342 s3write.write(content.encode()) 

343 self.assertEqual(s3write.read().decode(), content) 

344 

345 def testRelative(self): 

346 """Check that we can get subpaths back from two URIs""" 

347 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

348 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

349 

350 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

351 

352 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

353 self.assertFalse(not_child.relative_to(parent)) 

354 

355 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

356 self.assertFalse(child.relative_to(not_s3)) 

357 

358 def testQuoting(self): 

359 """Check that quoting works.""" 

360 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

361 subpath = "rootdir/dir1+/file?.txt" 

362 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

363 

364 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

365 self.assertEqual(child.basename(), "file?.txt") 

366 self.assertEqual(child.relativeToPathRoot, subpath) 

367 self.assertIn("%", child.path) 

368 self.assertEqual(child.unquoted_path, "/" + subpath) 

369 

370 

371# Mock required environment variables during tests 

372@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

373 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join( 

374 TESTDIR, "config/testConfigs/webdav/token"), 

375 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"}) 

376class WebdavURITestCase(unittest.TestCase): 

377 

378 def setUp(self): 

379 serverRoot = "www.not-exists.orgx" 

380 existingFolderName = "existingFolder" 

381 existingFileName = "existingFile" 

382 notExistingFileName = "notExistingFile" 

383 

384 self.baseURL = ButlerURI( 

385 f"https://{serverRoot}", forceDirectory=True) 

386 self.existingFileButlerURI = ButlerURI( 

387 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

388 self.notExistingFileButlerURI = ButlerURI( 

389 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

390 self.existingFolderButlerURI = ButlerURI( 

391 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

392 self.notExistingFolderButlerURI = ButlerURI( 

393 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

394 

395 # Need to declare the options 

396 responses.add(responses.OPTIONS, 

397 self.baseURL.geturl(), 

398 status=200, headers={"DAV": "1,2,3"}) 

399 

400 # Used by ButlerHttpURI.exists() 

401 responses.add(responses.HEAD, 

402 self.existingFileButlerURI.geturl(), 

403 status=200, headers={'Content-Length': '1024'}) 

404 responses.add(responses.HEAD, 

405 self.notExistingFileButlerURI.geturl(), 

406 status=404) 

407 

408 # Used by ButlerHttpURI.read() 

409 responses.add(responses.GET, 

410 self.existingFileButlerURI.geturl(), 

411 status=200, 

412 body=str.encode("It works!")) 

413 responses.add(responses.GET, 

414 self.notExistingFileButlerURI.geturl(), 

415 status=404) 

416 

417 # Used by ButlerHttpURI.write() 

418 responses.add(responses.PUT, 

419 self.existingFileButlerURI.geturl(), 

420 status=201) 

421 

422 # Used by ButlerHttpURI.transfer_from() 

423 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

424 method="COPY", 

425 headers={"Destination": self.existingFileButlerURI.geturl()}, 

426 status=201)) 

427 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

428 method="COPY", 

429 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

430 status=201)) 

431 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

432 method="MOVE", 

433 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

434 status=201)) 

435 

436 # Used by ButlerHttpURI.remove() 

437 responses.add(responses.DELETE, 

438 self.existingFileButlerURI.geturl(), 

439 status=200) 

440 responses.add(responses.DELETE, 

441 self.notExistingFileButlerURI.geturl(), 

442 status=404) 

443 

444 # Used by ButlerHttpURI.mkdir() 

445 responses.add(responses.HEAD, 

446 self.existingFolderButlerURI.geturl(), 

447 status=200, headers={'Content-Length': '1024'}) 

448 responses.add(responses.HEAD, 

449 self.baseURL.geturl(), 

450 status=200, headers={'Content-Length': '1024'}) 

451 responses.add(responses.HEAD, 

452 self.notExistingFolderButlerURI.geturl(), 

453 status=404) 

454 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

455 method="MKCOL", 

456 status=201)) 

457 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

458 method="MKCOL", 

459 status=403)) 

460 

461 @responses.activate 

462 def testExists(self): 

463 

464 self.assertTrue(self.existingFileButlerURI.exists()) 

465 self.assertFalse(self.notExistingFileButlerURI.exists()) 

466 

467 @responses.activate 

468 def testRemove(self): 

469 

470 self.assertIsNone(self.existingFileButlerURI.remove()) 

471 with self.assertRaises(FileNotFoundError): 

472 self.notExistingFileButlerURI.remove() 

473 

474 @responses.activate 

475 def testMkdir(self): 

476 

477 # The mock means that we can't check this now exists 

478 self.notExistingFolderButlerURI.mkdir() 

479 

480 # This should do nothing 

481 self.existingFolderButlerURI.mkdir() 

482 

483 with self.assertRaises(ValueError): 

484 self.notExistingFileButlerURI.mkdir() 

485 

486 @responses.activate 

487 def testRead(self): 

488 

489 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

490 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

491 with self.assertRaises(FileNotFoundError): 

492 self.notExistingFileButlerURI.read() 

493 

494 @responses.activate 

495 def testWrite(self): 

496 

497 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

498 with self.assertRaises(FileExistsError): 

499 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

500 

501 @responses.activate 

502 def testTransfer(self): 

503 

504 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

505 src=self.existingFileButlerURI)) 

506 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

507 src=self.existingFileButlerURI, 

508 transfer="move")) 

509 with self.assertRaises(FileExistsError): 

510 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

511 with self.assertRaises(ValueError): 

512 self.notExistingFileButlerURI.transfer_from( 

513 src=self.existingFileButlerURI, 

514 transfer="unsupported") 

515 

516 def testParent(self): 

517 

518 self.assertEqual(self.existingFolderButlerURI.geturl(), 

519 self.notExistingFileButlerURI.parent().geturl()) 

520 self.assertEqual(self.baseURL.geturl(), 

521 self.baseURL.parent().geturl()) 

522 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

523 self.existingFileButlerURI.dirname().geturl()) 

524 

525 

526if __name__ == "__main__": 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true

527 unittest.main()