Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26import urllib.parse 

27import responses 

28 

29try: 

30 import boto3 

31 import botocore 

32 from moto import mock_s3 

33except ImportError: 

34 boto3 = None 

35 

36 def mock_s3(cls): 

37 """A no-op decorator in case moto mock_s3 can not be imported. 

38 """ 

39 return cls 

40 

41from lsst.daf.butler import ButlerURI 

42from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials, 

43 unsetAwsEnvCredentials) 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FileURITestCase(unittest.TestCase): 

49 """Concrete tests for local files""" 

50 

51 def setUp(self): 

52 # Use a local tempdir because on macOS the temp dirs use symlinks 

53 # so relsymlink gets quite confused. 

54 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR) 

55 

56 def tearDown(self): 

57 shutil.rmtree(self.tmpdir, ignore_errors=True) 

58 

59 def testFile(self): 

60 file = os.path.join(self.tmpdir, "test.txt") 

61 uri = ButlerURI(file) 

62 self.assertFalse(uri.exists(), f"{uri} should not exist") 

63 self.assertEqual(uri.ospath, file) 

64 

65 content = "abcdefghijklmnopqrstuv\n" 

66 uri.write(content.encode()) 

67 self.assertTrue(os.path.exists(file), "File should exist locally") 

68 self.assertTrue(uri.exists(), f"{uri} should now exist") 

69 self.assertEqual(uri.read().decode(), content) 

70 

71 def testRelative(self): 

72 """Check that we can get subpaths back from two URIs""" 

73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

75 

76 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

77 

78 not_child = ButlerURI("/a/b/dir1/file.txt") 

79 self.assertFalse(not_child.relative_to(parent)) 

80 

81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

82 self.assertFalse(child.relative_to(not_directory)) 

83 

84 # Relative URIs 

85 parent = ButlerURI("a/b/", forceAbsolute=False) 

86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

87 self.assertFalse(child.scheme) 

88 self.assertEqual(child.relative_to(parent), "c/d.txt") 

89 

90 # File URI and schemeless URI 

91 parent = ButlerURI("file:/a/b/c/") 

92 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

93 

94 # If the child is relative and the parent is absolute we assume 

95 # that the child is a child of the parent unless it uses ".." 

96 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

97 

98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

99 self.assertFalse(child.relative_to(parent)) 

100 

101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

102 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

103 

104 def testMkdir(self): 

105 tmpdir = ButlerURI(self.tmpdir) 

106 newdir = tmpdir.join("newdir/seconddir") 

107 newdir.mkdir() 

108 self.assertTrue(newdir.exists()) 

109 newfile = newdir.join("temp.txt") 

110 newfile.write("Data".encode()) 

111 self.assertTrue(newfile.exists()) 

112 

113 def testTransfer(self): 

114 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

115 content = "Content is some content\nwith something to say\n\n" 

116 src.write(content.encode()) 

117 

118 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

119 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

120 dest.transfer_from(src, transfer=mode) 

121 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

122 

123 with open(dest.ospath, "r") as fh: 

124 new_content = fh.read() 

125 self.assertEqual(new_content, content) 

126 

127 if mode in ("symlink", "relsymlink"): 

128 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

129 

130 with self.assertRaises(FileExistsError): 

131 dest.transfer_from(src, transfer=mode) 

132 

133 dest.transfer_from(src, transfer=mode, overwrite=True) 

134 

135 os.remove(dest.ospath) 

136 

137 b = src.read() 

138 self.assertEqual(b.decode(), new_content) 

139 

140 nbytes = 10 

141 subset = src.read(size=nbytes) 

142 self.assertEqual(len(subset), nbytes) 

143 self.assertEqual(subset.decode(), content[:nbytes]) 

144 

145 with self.assertRaises(ValueError): 

146 src.transfer_from(src, transfer="unknown") 

147 

148 def testResource(self): 

149 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

150 self.assertTrue(u.exists(), f"Check {u} exists") 

151 

152 content = u.read().decode() 

153 self.assertTrue(content.startswith("datastore:")) 

154 

155 truncated = u.read(size=9).decode() 

156 self.assertEqual(truncated, "datastore") 

157 

158 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

159 self.assertTrue(u.exists(), f"Check directory {d} exists") 

160 

161 j = d.join("datastore.yaml") 

162 self.assertEqual(u, j) 

163 self.assertFalse(j.dirLike) 

164 self.assertFalse(d.join("not-there.yaml").exists()) 

165 

166 def testEscapes(self): 

167 """Special characters in file paths""" 

168 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

169 self.assertFalse(src.scheme) 

170 src.write(b"Some content") 

171 self.assertTrue(src.exists()) 

172 

173 # Use the internal API to force to a file 

174 file = src._force_to_file() 

175 self.assertTrue(file.exists()) 

176 self.assertIn("???", file.ospath) 

177 self.assertNotIn("???", file.path) 

178 

179 file.updateFile("tests??.txt") 

180 self.assertNotIn("??.txt", file.path) 

181 file.write(b"Other content") 

182 self.assertEqual(file.read(), b"Other content") 

183 

184 src.updateFile("tests??.txt") 

185 self.assertIn("??.txt", src.path) 

186 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

187 

188 # File URI and schemeless URI 

189 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

190 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

191 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

192 

193 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

194 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

195 

196 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

197 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

198 

199 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

200 

201 # Schemeless so should not quote 

202 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

203 self.assertIn("???", dir.ospath) 

204 self.assertIn("???", dir.path) 

205 self.assertFalse(dir.scheme) 

206 

207 # dir.join() morphs into a file scheme 

208 new = dir.join("test_j.txt") 

209 self.assertIn("???", new.ospath, f"Checking {new}") 

210 new.write(b"Content") 

211 

212 new2name = "###/test??.txt" 

213 new2 = dir.join(new2name) 

214 self.assertIn("???", new2.ospath) 

215 new2.write(b"Content") 

216 self.assertTrue(new2.ospath.endswith(new2name)) 

217 self.assertEqual(new.read(), new2.read()) 

218 

219 fdir = dir._force_to_file() 

220 self.assertNotIn("???", fdir.path) 

221 self.assertIn("???", fdir.ospath) 

222 self.assertEqual(fdir.scheme, "file") 

223 fnew = dir.join("test_jf.txt") 

224 fnew.write(b"Content") 

225 

226 fnew2 = fdir.join(new2name) 

227 fnew2.write(b"Content") 

228 self.assertTrue(fnew2.ospath.endswith(new2name)) 

229 self.assertNotIn("###", fnew2.path) 

230 

231 self.assertEqual(fnew.read(), fnew2.read()) 

232 

233 # Test that children relative to schemeless and file schemes 

234 # still return the same unquoted name 

235 self.assertEqual(fnew2.relative_to(fdir), new2name) 

236 self.assertEqual(fnew2.relative_to(dir), new2name) 

237 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

238 self.assertEqual(new2.relative_to(dir), new2name) 

239 

240 # Check for double quoting 

241 plus_path = "/a/b/c+d/" 

242 with self.assertLogs(level="WARNING"): 

243 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

244 self.assertEqual(uri.ospath, plus_path) 

245 

246 

247@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

248@mock_s3 

249class S3URITestCase(unittest.TestCase): 

250 """Tests involving S3""" 

251 

252 bucketName = "any_bucket" 

253 """Bucket name to use in tests""" 

254 

255 def setUp(self): 

256 # Local test directory 

257 self.tmpdir = tempfile.mkdtemp() 

258 

259 # set up some fake credentials if they do not exist 

260 self.usingDummyCredentials = setAwsEnvCredentials() 

261 

262 # MOTO needs to know that we expect Bucket bucketname to exist 

263 s3 = boto3.resource("s3") 

264 s3.create_bucket(Bucket=self.bucketName) 

265 

266 def tearDown(self): 

267 s3 = boto3.resource("s3") 

268 bucket = s3.Bucket(self.bucketName) 

269 try: 

270 bucket.objects.all().delete() 

271 except botocore.exceptions.ClientError as e: 

272 if e.response["Error"]["Code"] == "404": 

273 # the key was not reachable - pass 

274 pass 

275 else: 

276 raise 

277 

278 bucket = s3.Bucket(self.bucketName) 

279 bucket.delete() 

280 

281 # unset any potentially set dummy credentials 

282 if self.usingDummyCredentials: 

283 unsetAwsEnvCredentials() 

284 

285 shutil.rmtree(self.tmpdir, ignore_errors=True) 

286 

287 def makeS3Uri(self, path): 

288 return f"s3://{self.bucketName}/{path}" 

289 

290 def testTransfer(self): 

291 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

292 content = "Content is some content\nwith something to say\n\n" 

293 src.write(content.encode()) 

294 

295 dest = ButlerURI(self.makeS3Uri("test.txt")) 

296 self.assertFalse(dest.exists()) 

297 dest.transfer_from(src, transfer="copy") 

298 self.assertTrue(dest.exists()) 

299 

300 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

301 dest2.transfer_from(dest, transfer="copy") 

302 self.assertTrue(dest2.exists()) 

303 

304 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

305 local.transfer_from(dest2, transfer="copy") 

306 with open(local.ospath, "r") as fd: 

307 new_content = fd.read() 

308 self.assertEqual(new_content, content) 

309 

310 with self.assertRaises(ValueError): 

311 dest2.transfer_from(local, transfer="symlink") 

312 

313 b = dest.read() 

314 self.assertEqual(b.decode(), new_content) 

315 

316 nbytes = 10 

317 subset = dest.read(size=nbytes) 

318 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

319 self.assertEqual(subset.decode(), content[:nbytes]) 

320 

321 with self.assertRaises(FileExistsError): 

322 dest.transfer_from(src, transfer="copy") 

323 

324 dest.transfer_from(src, transfer="copy", overwrite=True) 

325 

326 def testWrite(self): 

327 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

328 content = "abcdefghijklmnopqrstuv\n" 

329 s3write.write(content.encode()) 

330 self.assertEqual(s3write.read().decode(), content) 

331 

332 def testRelative(self): 

333 """Check that we can get subpaths back from two URIs""" 

334 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

335 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

336 

337 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

338 

339 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

340 self.assertFalse(not_child.relative_to(parent)) 

341 

342 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

343 self.assertFalse(child.relative_to(not_s3)) 

344 

345 def testQuoting(self): 

346 """Check that quoting works.""" 

347 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

348 subpath = "rootdir/dir1+/file?.txt" 

349 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

350 

351 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

352 self.assertEqual(child.basename(), "file?.txt") 

353 self.assertEqual(child.relativeToPathRoot, subpath) 

354 self.assertIn("%", child.path) 

355 self.assertEqual(child.unquoted_path, "/" + subpath) 

356 

357 

358# Mock required environment variables during tests 

359@unittest.mock.patch.dict(os.environ, {"WEBDAV_AUTH_METHOD": "TOKEN", 

360 "WEBDAV_BEARER_TOKEN": "XXXXXX"}) 

361class WebdavURITestCase(unittest.TestCase): 

362 

363 def setUp(self): 

364 serverRoot = "www.not-exists.orgx" 

365 existingFolderName = "existingFolder" 

366 existingFileName = "existingFile" 

367 notExistingFileName = "notExistingFile" 

368 

369 self.baseURL = ButlerURI( 

370 f"https://{serverRoot}", forceDirectory=True) 

371 self.existingFileButlerURI = ButlerURI( 

372 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

373 self.notExistingFileButlerURI = ButlerURI( 

374 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

375 self.existingFolderButlerURI = ButlerURI( 

376 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

377 self.notExistingFolderButlerURI = ButlerURI( 

378 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

379 

380 # Need to declare the options 

381 responses.add(responses.OPTIONS, 

382 self.baseURL.geturl(), 

383 status=200, headers={"DAV": "1,2,3"}) 

384 

385 # Used by ButlerHttpURI.exists() 

386 responses.add(responses.HEAD, 

387 self.existingFileButlerURI.geturl(), 

388 status=200, headers={'Content-Length': '1024'}) 

389 responses.add(responses.HEAD, 

390 self.notExistingFileButlerURI.geturl(), 

391 status=404) 

392 

393 # Used by ButlerHttpURI.read() 

394 responses.add(responses.GET, 

395 self.existingFileButlerURI.geturl(), 

396 status=200, 

397 body=str.encode("It works!")) 

398 responses.add(responses.GET, 

399 self.notExistingFileButlerURI.geturl(), 

400 status=404) 

401 

402 # Used by ButlerHttpURI.write() 

403 responses.add(responses.PUT, 

404 self.existingFileButlerURI.geturl(), 

405 status=200) 

406 

407 # Used by ButlerHttpURI.transfer_from() 

408 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

409 method="COPY", 

410 headers={"Destination": self.existingFileButlerURI.geturl()}, 

411 status=200)) 

412 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

413 method="COPY", 

414 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

415 status=200)) 

416 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

417 method="MOVE", 

418 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

419 status=200)) 

420 

421 # Used by ButlerHttpURI.remove() 

422 responses.add(responses.DELETE, 

423 self.existingFileButlerURI.geturl(), 

424 status=200) 

425 responses.add(responses.DELETE, 

426 self.notExistingFileButlerURI.geturl(), 

427 status=404) 

428 

429 # Used by ButlerHttpURI.mkdir() 

430 responses.add(responses.HEAD, 

431 self.existingFolderButlerURI.geturl(), 

432 status=200, headers={'Content-Length': '1024'}) 

433 responses.add(responses.HEAD, 

434 self.baseURL.geturl(), 

435 status=200, headers={'Content-Length': '1024'}) 

436 responses.add(responses.HEAD, 

437 self.notExistingFolderButlerURI.geturl(), 

438 status=404) 

439 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

440 method="MKCOL", 

441 status=201)) 

442 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

443 method="MKCOL", 

444 status=403)) 

445 

446 @responses.activate 

447 def testExists(self): 

448 

449 self.assertTrue(self.existingFileButlerURI.exists()) 

450 self.assertFalse(self.notExistingFileButlerURI.exists()) 

451 

452 @responses.activate 

453 def testRemove(self): 

454 

455 self.assertIsNone(self.existingFileButlerURI.remove()) 

456 with self.assertRaises(FileNotFoundError): 

457 self.notExistingFileButlerURI.remove() 

458 

459 @responses.activate 

460 def testMkdir(self): 

461 

462 # The mock means that we can't check this now exists 

463 self.notExistingFolderButlerURI.mkdir() 

464 

465 # This should do nothing 

466 self.existingFolderButlerURI.mkdir() 

467 

468 with self.assertRaises(ValueError): 

469 self.notExistingFileButlerURI.mkdir() 

470 

471 @responses.activate 

472 def testRead(self): 

473 

474 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

475 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

476 with self.assertRaises(FileNotFoundError): 

477 self.notExistingFileButlerURI.read() 

478 

479 @responses.activate 

480 def testWrite(self): 

481 

482 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

483 with self.assertRaises(FileExistsError): 

484 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

485 

486 @responses.activate 

487 def testTransfer(self): 

488 

489 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

490 src=self.existingFileButlerURI)) 

491 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

492 src=self.existingFileButlerURI, 

493 transfer="move")) 

494 with self.assertRaises(FileExistsError): 

495 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

496 with self.assertRaises(ValueError): 

497 self.notExistingFileButlerURI.transfer_from( 

498 src=self.existingFileButlerURI, 

499 transfer="unsupported") 

500 

501 def testParent(self): 

502 

503 self.assertEqual(self.existingFolderButlerURI.geturl(), 

504 self.notExistingFileButlerURI.parent().geturl()) 

505 self.assertEqual(self.baseURL.geturl(), 

506 self.baseURL.parent().geturl()) 

507 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

508 self.existingFileButlerURI.dirname().geturl()) 

509 

510 

511if __name__ == "__main__": 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true

512 unittest.main()