Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26import urllib.parse 

27import responses 

28 

29try: 

30 import boto3 

31 import botocore 

32 from moto import mock_s3 

33except ImportError: 

34 boto3 = None 

35 

36 def mock_s3(cls): 

37 """A no-op decorator in case moto mock_s3 can not be imported. 

38 """ 

39 return cls 

40 

41from lsst.daf.butler import ButlerURI 

42from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials, 

43 unsetAwsEnvCredentials) 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FileURITestCase(unittest.TestCase): 

49 """Concrete tests for local files""" 

50 

51 def setUp(self): 

52 # Use a local tempdir because on macOS the temp dirs use symlinks 

53 # so relsymlink gets quite confused. 

54 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR) 

55 

56 def tearDown(self): 

57 shutil.rmtree(self.tmpdir, ignore_errors=True) 

58 

59 def testFile(self): 

60 file = os.path.join(self.tmpdir, "test.txt") 

61 uri = ButlerURI(file) 

62 self.assertFalse(uri.exists(), f"{uri} should not exist") 

63 self.assertEqual(uri.ospath, file) 

64 

65 content = "abcdefghijklmnopqrstuv\n" 

66 uri.write(content.encode()) 

67 self.assertTrue(os.path.exists(file), "File should exist locally") 

68 self.assertTrue(uri.exists(), f"{uri} should now exist") 

69 self.assertEqual(uri.read().decode(), content) 

70 

71 def testRelative(self): 

72 """Check that we can get subpaths back from two URIs""" 

73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

75 

76 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

77 

78 not_child = ButlerURI("/a/b/dir1/file.txt") 

79 self.assertFalse(not_child.relative_to(parent)) 

80 

81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

82 self.assertFalse(child.relative_to(not_directory)) 

83 

84 # Relative URIs 

85 parent = ButlerURI("a/b/", forceAbsolute=False) 

86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

87 self.assertFalse(child.scheme) 

88 self.assertEqual(child.relative_to(parent), "c/d.txt") 

89 

90 # File URI and schemeless URI 

91 parent = ButlerURI("file:/a/b/c/") 

92 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

93 

94 # If the child is relative and the parent is absolute we assume 

95 # that the child is a child of the parent unless it uses ".." 

96 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

97 

98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

99 self.assertFalse(child.relative_to(parent)) 

100 

101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

102 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

103 

104 def testMkdir(self): 

105 tmpdir = ButlerURI(self.tmpdir) 

106 newdir = tmpdir.join("newdir/seconddir") 

107 newdir.mkdir() 

108 self.assertTrue(newdir.exists()) 

109 newfile = newdir.join("temp.txt") 

110 newfile.write("Data".encode()) 

111 self.assertTrue(newfile.exists()) 

112 

113 def testTransfer(self): 

114 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

115 content = "Content is some content\nwith something to say\n\n" 

116 src.write(content.encode()) 

117 

118 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

119 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

120 dest.transfer_from(src, transfer=mode) 

121 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

122 

123 with open(dest.ospath, "r") as fh: 

124 new_content = fh.read() 

125 self.assertEqual(new_content, content) 

126 

127 if mode in ("symlink", "relsymlink"): 

128 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

129 

130 with self.assertRaises(FileExistsError): 

131 dest.transfer_from(src, transfer=mode) 

132 

133 dest.transfer_from(src, transfer=mode, overwrite=True) 

134 

135 os.remove(dest.ospath) 

136 

137 b = src.read() 

138 self.assertEqual(b.decode(), new_content) 

139 

140 nbytes = 10 

141 subset = src.read(size=nbytes) 

142 self.assertEqual(len(subset), nbytes) 

143 self.assertEqual(subset.decode(), content[:nbytes]) 

144 

145 with self.assertRaises(ValueError): 

146 src.transfer_from(src, transfer="unknown") 

147 

148 def testResource(self): 

149 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

150 self.assertTrue(u.exists(), f"Check {u} exists") 

151 

152 content = u.read().decode() 

153 self.assertTrue(content.startswith("datastore:")) 

154 

155 truncated = u.read(size=9).decode() 

156 self.assertEqual(truncated, "datastore") 

157 

158 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

159 self.assertTrue(u.exists(), f"Check directory {d} exists") 

160 

161 j = d.join("datastore.yaml") 

162 self.assertEqual(u, j) 

163 self.assertFalse(j.dirLike) 

164 self.assertFalse(d.join("not-there.yaml").exists()) 

165 

166 def testEscapes(self): 

167 """Special characters in file paths""" 

168 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

169 self.assertFalse(src.scheme) 

170 src.write(b"Some content") 

171 self.assertTrue(src.exists()) 

172 

173 # Use the internal API to force to a file 

174 file = src._force_to_file() 

175 self.assertTrue(file.exists()) 

176 self.assertIn("???", file.ospath) 

177 self.assertNotIn("???", file.path) 

178 

179 file.updateFile("tests??.txt") 

180 self.assertNotIn("??.txt", file.path) 

181 file.write(b"Other content") 

182 self.assertEqual(file.read(), b"Other content") 

183 

184 src.updateFile("tests??.txt") 

185 self.assertIn("??.txt", src.path) 

186 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

187 

188 # File URI and schemeless URI 

189 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

190 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

191 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

192 

193 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

194 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

195 

196 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

197 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

198 

199 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

200 

201 # Schemeless so should not quote 

202 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

203 self.assertIn("???", dir.ospath) 

204 self.assertIn("???", dir.path) 

205 self.assertFalse(dir.scheme) 

206 

207 # dir.join() morphs into a file scheme 

208 new = dir.join("test_j.txt") 

209 self.assertIn("???", new.ospath, f"Checking {new}") 

210 new.write(b"Content") 

211 

212 new2name = "###/test??.txt" 

213 new2 = dir.join(new2name) 

214 self.assertIn("???", new2.ospath) 

215 new2.write(b"Content") 

216 self.assertTrue(new2.ospath.endswith(new2name)) 

217 self.assertEqual(new.read(), new2.read()) 

218 

219 fdir = dir._force_to_file() 

220 self.assertNotIn("???", fdir.path) 

221 self.assertIn("???", fdir.ospath) 

222 self.assertEqual(fdir.scheme, "file") 

223 fnew = dir.join("test_jf.txt") 

224 fnew.write(b"Content") 

225 

226 fnew2 = fdir.join(new2name) 

227 fnew2.write(b"Content") 

228 self.assertTrue(fnew2.ospath.endswith(new2name)) 

229 self.assertNotIn("###", fnew2.path) 

230 

231 self.assertEqual(fnew.read(), fnew2.read()) 

232 

233 # Test that children relative to schemeless and file schemes 

234 # still return the same unquoted name 

235 self.assertEqual(fnew2.relative_to(fdir), new2name) 

236 self.assertEqual(fnew2.relative_to(dir), new2name) 

237 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

238 self.assertEqual(new2.relative_to(dir), new2name) 

239 

240 # Check for double quoting 

241 plus_path = "/a/b/c+d/" 

242 with self.assertLogs(level="WARNING"): 

243 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

244 self.assertEqual(uri.ospath, plus_path) 

245 

246 

247@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

248@mock_s3 

249class S3URITestCase(unittest.TestCase): 

250 """Tests involving S3""" 

251 

252 bucketName = "any_bucket" 

253 """Bucket name to use in tests""" 

254 

255 def setUp(self): 

256 # Local test directory 

257 self.tmpdir = tempfile.mkdtemp() 

258 

259 # set up some fake credentials if they do not exist 

260 self.usingDummyCredentials = setAwsEnvCredentials() 

261 

262 # MOTO needs to know that we expect Bucket bucketname to exist 

263 s3 = boto3.resource("s3") 

264 s3.create_bucket(Bucket=self.bucketName) 

265 

266 def tearDown(self): 

267 s3 = boto3.resource("s3") 

268 bucket = s3.Bucket(self.bucketName) 

269 try: 

270 bucket.objects.all().delete() 

271 except botocore.exceptions.ClientError as e: 

272 if e.response["Error"]["Code"] == "404": 

273 # the key was not reachable - pass 

274 pass 

275 else: 

276 raise 

277 

278 bucket = s3.Bucket(self.bucketName) 

279 bucket.delete() 

280 

281 # unset any potentially set dummy credentials 

282 if self.usingDummyCredentials: 

283 unsetAwsEnvCredentials() 

284 

285 shutil.rmtree(self.tmpdir, ignore_errors=True) 

286 

287 def makeS3Uri(self, path): 

288 return f"s3://{self.bucketName}/{path}" 

289 

290 def testTransfer(self): 

291 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

292 content = "Content is some content\nwith something to say\n\n" 

293 src.write(content.encode()) 

294 

295 dest = ButlerURI(self.makeS3Uri("test.txt")) 

296 self.assertFalse(dest.exists()) 

297 dest.transfer_from(src, transfer="copy") 

298 self.assertTrue(dest.exists()) 

299 

300 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

301 dest2.transfer_from(dest, transfer="copy") 

302 self.assertTrue(dest2.exists()) 

303 

304 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

305 local.transfer_from(dest2, transfer="copy") 

306 with open(local.ospath, "r") as fd: 

307 new_content = fd.read() 

308 self.assertEqual(new_content, content) 

309 

310 with self.assertRaises(ValueError): 

311 dest2.transfer_from(local, transfer="symlink") 

312 

313 b = dest.read() 

314 self.assertEqual(b.decode(), new_content) 

315 

316 nbytes = 10 

317 subset = dest.read(size=nbytes) 

318 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

319 self.assertEqual(subset.decode(), content[:nbytes]) 

320 

321 with self.assertRaises(FileExistsError): 

322 dest.transfer_from(src, transfer="copy") 

323 

324 dest.transfer_from(src, transfer="copy", overwrite=True) 

325 

326 def testWrite(self): 

327 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

328 content = "abcdefghijklmnopqrstuv\n" 

329 s3write.write(content.encode()) 

330 self.assertEqual(s3write.read().decode(), content) 

331 

332 def testRelative(self): 

333 """Check that we can get subpaths back from two URIs""" 

334 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

335 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

336 

337 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

338 

339 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

340 self.assertFalse(not_child.relative_to(parent)) 

341 

342 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

343 self.assertFalse(child.relative_to(not_s3)) 

344 

345 def testQuoting(self): 

346 """Check that quoting works.""" 

347 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

348 subpath = "rootdir/dir1+/file?.txt" 

349 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

350 

351 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

352 self.assertEqual(child.basename(), "file?.txt") 

353 self.assertEqual(child.relativeToPathRoot, subpath) 

354 self.assertIn("%", child.path) 

355 self.assertEqual(child.unquoted_path, "/" + subpath) 

356 

357 

358# Mock required environment variables during tests 

359@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

360 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join( 

361 TESTDIR, "config/testConfigs/webdav/token"), 

362 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"}) 

363class WebdavURITestCase(unittest.TestCase): 

364 

365 def setUp(self): 

366 serverRoot = "www.not-exists.orgx" 

367 existingFolderName = "existingFolder" 

368 existingFileName = "existingFile" 

369 notExistingFileName = "notExistingFile" 

370 

371 self.baseURL = ButlerURI( 

372 f"https://{serverRoot}", forceDirectory=True) 

373 self.existingFileButlerURI = ButlerURI( 

374 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

375 self.notExistingFileButlerURI = ButlerURI( 

376 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

377 self.existingFolderButlerURI = ButlerURI( 

378 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

379 self.notExistingFolderButlerURI = ButlerURI( 

380 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

381 

382 # Need to declare the options 

383 responses.add(responses.OPTIONS, 

384 self.baseURL.geturl(), 

385 status=200, headers={"DAV": "1,2,3"}) 

386 

387 # Used by ButlerHttpURI.exists() 

388 responses.add(responses.HEAD, 

389 self.existingFileButlerURI.geturl(), 

390 status=200, headers={'Content-Length': '1024'}) 

391 responses.add(responses.HEAD, 

392 self.notExistingFileButlerURI.geturl(), 

393 status=404) 

394 

395 # Used by ButlerHttpURI.read() 

396 responses.add(responses.GET, 

397 self.existingFileButlerURI.geturl(), 

398 status=200, 

399 body=str.encode("It works!")) 

400 responses.add(responses.GET, 

401 self.notExistingFileButlerURI.geturl(), 

402 status=404) 

403 

404 # Used by ButlerHttpURI.write() 

405 responses.add(responses.PUT, 

406 self.existingFileButlerURI.geturl(), 

407 status=201) 

408 

409 # Used by ButlerHttpURI.transfer_from() 

410 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

411 method="COPY", 

412 headers={"Destination": self.existingFileButlerURI.geturl()}, 

413 status=201)) 

414 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

415 method="COPY", 

416 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

417 status=201)) 

418 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

419 method="MOVE", 

420 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

421 status=201)) 

422 

423 # Used by ButlerHttpURI.remove() 

424 responses.add(responses.DELETE, 

425 self.existingFileButlerURI.geturl(), 

426 status=200) 

427 responses.add(responses.DELETE, 

428 self.notExistingFileButlerURI.geturl(), 

429 status=404) 

430 

431 # Used by ButlerHttpURI.mkdir() 

432 responses.add(responses.HEAD, 

433 self.existingFolderButlerURI.geturl(), 

434 status=200, headers={'Content-Length': '1024'}) 

435 responses.add(responses.HEAD, 

436 self.baseURL.geturl(), 

437 status=200, headers={'Content-Length': '1024'}) 

438 responses.add(responses.HEAD, 

439 self.notExistingFolderButlerURI.geturl(), 

440 status=404) 

441 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

442 method="MKCOL", 

443 status=201)) 

444 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

445 method="MKCOL", 

446 status=403)) 

447 

448 @responses.activate 

449 def testExists(self): 

450 

451 self.assertTrue(self.existingFileButlerURI.exists()) 

452 self.assertFalse(self.notExistingFileButlerURI.exists()) 

453 

454 @responses.activate 

455 def testRemove(self): 

456 

457 self.assertIsNone(self.existingFileButlerURI.remove()) 

458 with self.assertRaises(FileNotFoundError): 

459 self.notExistingFileButlerURI.remove() 

460 

461 @responses.activate 

462 def testMkdir(self): 

463 

464 # The mock means that we can't check this now exists 

465 self.notExistingFolderButlerURI.mkdir() 

466 

467 # This should do nothing 

468 self.existingFolderButlerURI.mkdir() 

469 

470 with self.assertRaises(ValueError): 

471 self.notExistingFileButlerURI.mkdir() 

472 

473 @responses.activate 

474 def testRead(self): 

475 

476 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

477 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

478 with self.assertRaises(FileNotFoundError): 

479 self.notExistingFileButlerURI.read() 

480 

481 @responses.activate 

482 def testWrite(self): 

483 

484 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

485 with self.assertRaises(FileExistsError): 

486 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

487 

488 @responses.activate 

489 def testTransfer(self): 

490 

491 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

492 src=self.existingFileButlerURI)) 

493 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

494 src=self.existingFileButlerURI, 

495 transfer="move")) 

496 with self.assertRaises(FileExistsError): 

497 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

498 with self.assertRaises(ValueError): 

499 self.notExistingFileButlerURI.transfer_from( 

500 src=self.existingFileButlerURI, 

501 transfer="unsupported") 

502 

503 def testParent(self): 

504 

505 self.assertEqual(self.existingFolderButlerURI.geturl(), 

506 self.notExistingFileButlerURI.parent().geturl()) 

507 self.assertEqual(self.baseURL.geturl(), 

508 self.baseURL.parent().geturl()) 

509 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

510 self.existingFileButlerURI.dirname().geturl()) 

511 

512 

513if __name__ == "__main__": 513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true

514 unittest.main()