Coverage for tests/test_uri.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import shutil
24import unittest
25import urllib.parse
26import responses
28try:
29 import boto3
30 import botocore
31 from moto import mock_s3
32except ImportError:
33 boto3 = None
35 def mock_s3(cls):
36 """A no-op decorator in case moto mock_s3 can not be imported.
37 """
38 return cls
40from lsst.daf.butler import ButlerURI
41from lsst.daf.butler.core._butlerUri.s3utils import (setAwsEnvCredentials,
42 unsetAwsEnvCredentials)
43from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
45TESTDIR = os.path.abspath(os.path.dirname(__file__))
48class FileURITestCase(unittest.TestCase):
49 """Concrete tests for local files"""
51 def setUp(self):
52 # Use a local tempdir because on macOS the temp dirs use symlinks
53 # so relsymlink gets quite confused.
54 self.tmpdir = makeTestTempDir(TESTDIR)
56 def tearDown(self):
57 removeTestTempDir(self.tmpdir)
59 def testFile(self):
60 file = os.path.join(self.tmpdir, "test.txt")
61 uri = ButlerURI(file)
62 self.assertFalse(uri.exists(), f"{uri} should not exist")
63 self.assertEqual(uri.ospath, file)
65 content = "abcdefghijklmnopqrstuv\n"
66 uri.write(content.encode())
67 self.assertTrue(os.path.exists(file), "File should exist locally")
68 self.assertTrue(uri.exists(), f"{uri} should now exist")
69 self.assertEqual(uri.read().decode(), content)
71 def testRelative(self):
72 """Check that we can get subpaths back from two URIs"""
73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True)
76 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
78 not_child = ButlerURI("/a/b/dir1/file.txt")
79 self.assertFalse(not_child.relative_to(parent))
81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
82 self.assertFalse(child.relative_to(not_directory))
84 # Relative URIs
85 parent = ButlerURI("a/b/", forceAbsolute=False)
86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False)
87 self.assertFalse(child.scheme)
88 self.assertEqual(child.relative_to(parent), "c/d.txt")
90 # File URI and schemeless URI
91 parent = ButlerURI("file:/a/b/c/")
92 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
94 # If the child is relative and the parent is absolute we assume
95 # that the child is a child of the parent unless it uses ".."
96 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False)
99 self.assertFalse(child.relative_to(parent))
101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
102 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
104 def testEnvVar(self):
105 """Test that environment variables are expanded."""
107 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}):
108 uri = ButlerURI("${MY_TEST_DIR}/d.txt")
109 self.assertEqual(uri.path, "/a/b/c/d.txt")
110 self.assertEqual(uri.scheme, "file")
112 # This will not expand
113 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False)
114 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt")
115 self.assertFalse(uri.scheme)
117 def testMkdir(self):
118 tmpdir = ButlerURI(self.tmpdir)
119 newdir = tmpdir.join("newdir/seconddir")
120 newdir.mkdir()
121 self.assertTrue(newdir.exists())
122 newfile = newdir.join("temp.txt")
123 newfile.write("Data".encode())
124 self.assertTrue(newfile.exists())
126 def testTransfer(self):
127 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
128 content = "Content is some content\nwith something to say\n\n"
129 src.write(content.encode())
131 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
132 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
133 dest.transfer_from(src, transfer=mode)
134 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
136 with open(dest.ospath, "r") as fh:
137 new_content = fh.read()
138 self.assertEqual(new_content, content)
140 if mode in ("symlink", "relsymlink"):
141 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
143 with self.assertRaises(FileExistsError):
144 dest.transfer_from(src, transfer=mode)
146 dest.transfer_from(src, transfer=mode, overwrite=True)
148 os.remove(dest.ospath)
150 b = src.read()
151 self.assertEqual(b.decode(), new_content)
153 nbytes = 10
154 subset = src.read(size=nbytes)
155 self.assertEqual(len(subset), nbytes)
156 self.assertEqual(subset.decode(), content[:nbytes])
158 with self.assertRaises(ValueError):
159 src.transfer_from(src, transfer="unknown")
161 def testResource(self):
162 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml")
163 self.assertTrue(u.exists(), f"Check {u} exists")
165 content = u.read().decode()
166 self.assertTrue(content.startswith("datastore:"))
168 truncated = u.read(size=9).decode()
169 self.assertEqual(truncated, "datastore")
171 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True)
172 self.assertTrue(u.exists(), f"Check directory {d} exists")
174 j = d.join("datastore.yaml")
175 self.assertEqual(u, j)
176 self.assertFalse(j.dirLike)
177 self.assertFalse(d.join("not-there.yaml").exists())
179 def testEscapes(self):
180 """Special characters in file paths"""
181 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
182 self.assertFalse(src.scheme)
183 src.write(b"Some content")
184 self.assertTrue(src.exists())
186 # Use the internal API to force to a file
187 file = src._force_to_file()
188 self.assertTrue(file.exists())
189 self.assertIn("???", file.ospath)
190 self.assertNotIn("???", file.path)
192 file.updateFile("tests??.txt")
193 self.assertNotIn("??.txt", file.path)
194 file.write(b"Other content")
195 self.assertEqual(file.read(), b"Other content")
197 src.updateFile("tests??.txt")
198 self.assertIn("??.txt", src.path)
199 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
201 # File URI and schemeless URI
202 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
203 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
204 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
206 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
207 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
209 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
210 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
212 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
214 # Schemeless so should not quote
215 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
216 self.assertIn("???", dir.ospath)
217 self.assertIn("???", dir.path)
218 self.assertFalse(dir.scheme)
220 # dir.join() morphs into a file scheme
221 new = dir.join("test_j.txt")
222 self.assertIn("???", new.ospath, f"Checking {new}")
223 new.write(b"Content")
225 new2name = "###/test??.txt"
226 new2 = dir.join(new2name)
227 self.assertIn("???", new2.ospath)
228 new2.write(b"Content")
229 self.assertTrue(new2.ospath.endswith(new2name))
230 self.assertEqual(new.read(), new2.read())
232 fdir = dir._force_to_file()
233 self.assertNotIn("???", fdir.path)
234 self.assertIn("???", fdir.ospath)
235 self.assertEqual(fdir.scheme, "file")
236 fnew = dir.join("test_jf.txt")
237 fnew.write(b"Content")
239 fnew2 = fdir.join(new2name)
240 fnew2.write(b"Content")
241 self.assertTrue(fnew2.ospath.endswith(new2name))
242 self.assertNotIn("###", fnew2.path)
244 self.assertEqual(fnew.read(), fnew2.read())
246 # Test that children relative to schemeless and file schemes
247 # still return the same unquoted name
248 self.assertEqual(fnew2.relative_to(fdir), new2name)
249 self.assertEqual(fnew2.relative_to(dir), new2name)
250 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
251 self.assertEqual(new2.relative_to(dir), new2name)
253 # Check for double quoting
254 plus_path = "/a/b/c+d/"
255 with self.assertLogs(level="WARNING"):
256 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
257 self.assertEqual(uri.ospath, plus_path)
259 # Check that # is not escaped for schemeless URIs
260 hash_path = "/a/b#/c&d#xyz"
261 hpos = hash_path.rfind("#")
262 uri = ButlerURI(hash_path)
263 self.assertEqual(uri.ospath, hash_path[:hpos])
264 self.assertEqual(uri.fragment, hash_path[hpos + 1:])
267@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
268@mock_s3
269class S3URITestCase(unittest.TestCase):
270 """Tests involving S3"""
272 bucketName = "any_bucket"
273 """Bucket name to use in tests"""
275 def setUp(self):
276 # Local test directory
277 self.tmpdir = makeTestTempDir(TESTDIR)
279 # set up some fake credentials if they do not exist
280 self.usingDummyCredentials = setAwsEnvCredentials()
282 # MOTO needs to know that we expect Bucket bucketname to exist
283 s3 = boto3.resource("s3")
284 s3.create_bucket(Bucket=self.bucketName)
286 def tearDown(self):
287 s3 = boto3.resource("s3")
288 bucket = s3.Bucket(self.bucketName)
289 try:
290 bucket.objects.all().delete()
291 except botocore.exceptions.ClientError as e:
292 if e.response["Error"]["Code"] == "404":
293 # the key was not reachable - pass
294 pass
295 else:
296 raise
298 bucket = s3.Bucket(self.bucketName)
299 bucket.delete()
301 # unset any potentially set dummy credentials
302 if self.usingDummyCredentials:
303 unsetAwsEnvCredentials()
305 shutil.rmtree(self.tmpdir, ignore_errors=True)
307 def makeS3Uri(self, path):
308 return f"s3://{self.bucketName}/{path}"
310 def testTransfer(self):
311 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
312 content = "Content is some content\nwith something to say\n\n"
313 src.write(content.encode())
315 dest = ButlerURI(self.makeS3Uri("test.txt"))
316 self.assertFalse(dest.exists())
317 dest.transfer_from(src, transfer="copy")
318 self.assertTrue(dest.exists())
320 dest2 = ButlerURI(self.makeS3Uri("copied.txt"))
321 dest2.transfer_from(dest, transfer="copy")
322 self.assertTrue(dest2.exists())
324 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt"))
325 local.transfer_from(dest2, transfer="copy")
326 with open(local.ospath, "r") as fd:
327 new_content = fd.read()
328 self.assertEqual(new_content, content)
330 with self.assertRaises(ValueError):
331 dest2.transfer_from(local, transfer="symlink")
333 b = dest.read()
334 self.assertEqual(b.decode(), new_content)
336 nbytes = 10
337 subset = dest.read(size=nbytes)
338 self.assertEqual(len(subset), nbytes) # Extra byte comes back
339 self.assertEqual(subset.decode(), content[:nbytes])
341 with self.assertRaises(FileExistsError):
342 dest.transfer_from(src, transfer="copy")
344 dest.transfer_from(src, transfer="copy", overwrite=True)
346 def testWrite(self):
347 s3write = ButlerURI(self.makeS3Uri("created.txt"))
348 content = "abcdefghijklmnopqrstuv\n"
349 s3write.write(content.encode())
350 self.assertEqual(s3write.read().decode(), content)
352 def testRelative(self):
353 """Check that we can get subpaths back from two URIs"""
354 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
355 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt"))
357 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
359 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt"))
360 self.assertFalse(not_child.relative_to(parent))
362 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
363 self.assertFalse(child.relative_to(not_s3))
365 def testQuoting(self):
366 """Check that quoting works."""
367 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
368 subpath = "rootdir/dir1+/file?.txt"
369 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath)))
371 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
372 self.assertEqual(child.basename(), "file?.txt")
373 self.assertEqual(child.relativeToPathRoot, subpath)
374 self.assertIn("%", child.path)
375 self.assertEqual(child.unquoted_path, "/" + subpath)
378# Mock required environment variables during tests
379@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN",
380 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join(
381 TESTDIR, "config/testConfigs/webdav/token"),
382 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"})
383class WebdavURITestCase(unittest.TestCase):
385 def setUp(self):
386 serverRoot = "www.not-exists.orgx"
387 existingFolderName = "existingFolder"
388 existingFileName = "existingFile"
389 notExistingFileName = "notExistingFile"
391 self.baseURL = ButlerURI(
392 f"https://{serverRoot}", forceDirectory=True)
393 self.existingFileButlerURI = ButlerURI(
394 f"https://{serverRoot}/{existingFolderName}/{existingFileName}")
395 self.notExistingFileButlerURI = ButlerURI(
396 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}")
397 self.existingFolderButlerURI = ButlerURI(
398 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True)
399 self.notExistingFolderButlerURI = ButlerURI(
400 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True)
402 # Need to declare the options
403 responses.add(responses.OPTIONS,
404 self.baseURL.geturl(),
405 status=200, headers={"DAV": "1,2,3"})
407 # Used by ButlerHttpURI.exists()
408 responses.add(responses.HEAD,
409 self.existingFileButlerURI.geturl(),
410 status=200, headers={'Content-Length': '1024'})
411 responses.add(responses.HEAD,
412 self.notExistingFileButlerURI.geturl(),
413 status=404)
415 # Used by ButlerHttpURI.read()
416 responses.add(responses.GET,
417 self.existingFileButlerURI.geturl(),
418 status=200,
419 body=str.encode("It works!"))
420 responses.add(responses.GET,
421 self.notExistingFileButlerURI.geturl(),
422 status=404)
424 # Used by ButlerHttpURI.write()
425 responses.add(responses.PUT,
426 self.existingFileButlerURI.geturl(),
427 status=201)
429 # Used by ButlerHttpURI.transfer_from()
430 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
431 method="COPY",
432 headers={"Destination": self.existingFileButlerURI.geturl()},
433 status=201))
434 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
435 method="COPY",
436 headers={"Destination": self.notExistingFileButlerURI.geturl()},
437 status=201))
438 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
439 method="MOVE",
440 headers={"Destination": self.notExistingFileButlerURI.geturl()},
441 status=201))
443 # Used by ButlerHttpURI.remove()
444 responses.add(responses.DELETE,
445 self.existingFileButlerURI.geturl(),
446 status=200)
447 responses.add(responses.DELETE,
448 self.notExistingFileButlerURI.geturl(),
449 status=404)
451 # Used by ButlerHttpURI.mkdir()
452 responses.add(responses.HEAD,
453 self.existingFolderButlerURI.geturl(),
454 status=200, headers={'Content-Length': '1024'})
455 responses.add(responses.HEAD,
456 self.baseURL.geturl(),
457 status=200, headers={'Content-Length': '1024'})
458 responses.add(responses.HEAD,
459 self.notExistingFolderButlerURI.geturl(),
460 status=404)
461 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(),
462 method="MKCOL",
463 status=201))
464 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(),
465 method="MKCOL",
466 status=403))
468 @responses.activate
469 def testExists(self):
471 self.assertTrue(self.existingFileButlerURI.exists())
472 self.assertFalse(self.notExistingFileButlerURI.exists())
474 @responses.activate
475 def testRemove(self):
477 self.assertIsNone(self.existingFileButlerURI.remove())
478 with self.assertRaises(FileNotFoundError):
479 self.notExistingFileButlerURI.remove()
481 @responses.activate
482 def testMkdir(self):
484 # The mock means that we can't check this now exists
485 self.notExistingFolderButlerURI.mkdir()
487 # This should do nothing
488 self.existingFolderButlerURI.mkdir()
490 with self.assertRaises(ValueError):
491 self.notExistingFileButlerURI.mkdir()
493 @responses.activate
494 def testRead(self):
496 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!")
497 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.")
498 with self.assertRaises(FileNotFoundError):
499 self.notExistingFileButlerURI.read()
501 @responses.activate
502 def testWrite(self):
504 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content.")))
505 with self.assertRaises(FileExistsError):
506 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False)
508 @responses.activate
509 def testTransfer(self):
511 self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
512 src=self.existingFileButlerURI))
513 self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
514 src=self.existingFileButlerURI,
515 transfer="move"))
516 with self.assertRaises(FileExistsError):
517 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI)
518 with self.assertRaises(ValueError):
519 self.notExistingFileButlerURI.transfer_from(
520 src=self.existingFileButlerURI,
521 transfer="unsupported")
523 def testParent(self):
525 self.assertEqual(self.existingFolderButlerURI.geturl(),
526 self.notExistingFileButlerURI.parent().geturl())
527 self.assertEqual(self.baseURL.geturl(),
528 self.baseURL.parent().geturl())
529 self.assertEqual(self.existingFileButlerURI.parent().geturl(),
530 self.existingFileButlerURI.dirname().geturl())
533if __name__ == "__main__": 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true
534 unittest.main()