Coverage for tests/test_uri.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import shutil
24import tempfile
25import unittest
26import urllib.parse
27import responses
29try:
30 import boto3
31 import botocore
32 from moto import mock_s3
33except ImportError:
34 boto3 = None
36 def mock_s3(cls):
37 """A no-op decorator in case moto mock_s3 can not be imported.
38 """
39 return cls
41from lsst.daf.butler import ButlerURI
42from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials,
43 unsetAwsEnvCredentials)
45TESTDIR = os.path.abspath(os.path.dirname(__file__))
48class FileURITestCase(unittest.TestCase):
49 """Concrete tests for local files"""
51 def setUp(self):
52 # Use a local tempdir because on macOS the temp dirs use symlinks
53 # so relsymlink gets quite confused.
54 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR)
56 def tearDown(self):
57 shutil.rmtree(self.tmpdir, ignore_errors=True)
59 def testFile(self):
60 file = os.path.join(self.tmpdir, "test.txt")
61 uri = ButlerURI(file)
62 self.assertFalse(uri.exists(), f"{uri} should not exist")
63 self.assertEqual(uri.ospath, file)
65 content = "abcdefghijklmnopqrstuv\n"
66 uri.write(content.encode())
67 self.assertTrue(os.path.exists(file), "File should exist locally")
68 self.assertTrue(uri.exists(), f"{uri} should now exist")
69 self.assertEqual(uri.read().decode(), content)
71 def testRelative(self):
72 """Check that we can get subpaths back from two URIs"""
73 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
74 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True)
76 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
78 not_child = ButlerURI("/a/b/dir1/file.txt")
79 self.assertFalse(not_child.relative_to(parent))
81 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
82 self.assertFalse(child.relative_to(not_directory))
84 # Relative URIs
85 parent = ButlerURI("a/b/", forceAbsolute=False)
86 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False)
87 self.assertFalse(child.scheme)
88 self.assertEqual(child.relative_to(parent), "c/d.txt")
90 # File URI and schemeless URI
91 parent = ButlerURI("file:/a/b/c/")
92 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
94 # If the child is relative and the parent is absolute we assume
95 # that the child is a child of the parent unless it uses ".."
96 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
98 child = ButlerURI("../e/f/g.txt", forceAbsolute=False)
99 self.assertFalse(child.relative_to(parent))
101 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
102 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
104 def testMkdir(self):
105 tmpdir = ButlerURI(self.tmpdir)
106 newdir = tmpdir.join("newdir/seconddir")
107 newdir.mkdir()
108 self.assertTrue(newdir.exists())
109 newfile = newdir.join("temp.txt")
110 newfile.write("Data".encode())
111 self.assertTrue(newfile.exists())
113 def testTransfer(self):
114 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
115 content = "Content is some content\nwith something to say\n\n"
116 src.write(content.encode())
118 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
119 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
120 dest.transfer_from(src, transfer=mode)
121 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
123 with open(dest.ospath, "r") as fh:
124 new_content = fh.read()
125 self.assertEqual(new_content, content)
127 if mode in ("symlink", "relsymlink"):
128 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
130 with self.assertRaises(FileExistsError):
131 dest.transfer_from(src, transfer=mode)
133 dest.transfer_from(src, transfer=mode, overwrite=True)
135 os.remove(dest.ospath)
137 b = src.read()
138 self.assertEqual(b.decode(), new_content)
140 nbytes = 10
141 subset = src.read(size=nbytes)
142 self.assertEqual(len(subset), nbytes)
143 self.assertEqual(subset.decode(), content[:nbytes])
145 with self.assertRaises(ValueError):
146 src.transfer_from(src, transfer="unknown")
148 def testResource(self):
149 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml")
150 self.assertTrue(u.exists(), f"Check {u} exists")
152 content = u.read().decode()
153 self.assertTrue(content.startswith("datastore:"))
155 truncated = u.read(size=9).decode()
156 self.assertEqual(truncated, "datastore")
158 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True)
159 self.assertTrue(u.exists(), f"Check directory {d} exists")
161 j = d.join("datastore.yaml")
162 self.assertEqual(u, j)
163 self.assertFalse(j.dirLike)
164 self.assertFalse(d.join("not-there.yaml").exists())
166 def testEscapes(self):
167 """Special characters in file paths"""
168 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
169 self.assertFalse(src.scheme)
170 src.write(b"Some content")
171 self.assertTrue(src.exists())
173 # Use the internal API to force to a file
174 file = src._force_to_file()
175 self.assertTrue(file.exists())
176 self.assertIn("???", file.ospath)
177 self.assertNotIn("???", file.path)
179 file.updateFile("tests??.txt")
180 self.assertNotIn("??.txt", file.path)
181 file.write(b"Other content")
182 self.assertEqual(file.read(), b"Other content")
184 src.updateFile("tests??.txt")
185 self.assertIn("??.txt", src.path)
186 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
188 # File URI and schemeless URI
189 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
190 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
191 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
193 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
194 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
196 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
197 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
199 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
201 # Schemeless so should not quote
202 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
203 self.assertIn("???", dir.ospath)
204 self.assertIn("???", dir.path)
205 self.assertFalse(dir.scheme)
207 # dir.join() morphs into a file scheme
208 new = dir.join("test_j.txt")
209 self.assertIn("???", new.ospath, f"Checking {new}")
210 new.write(b"Content")
212 new2name = "###/test??.txt"
213 new2 = dir.join(new2name)
214 self.assertIn("???", new2.ospath)
215 new2.write(b"Content")
216 self.assertTrue(new2.ospath.endswith(new2name))
217 self.assertEqual(new.read(), new2.read())
219 fdir = dir._force_to_file()
220 self.assertNotIn("???", fdir.path)
221 self.assertIn("???", fdir.ospath)
222 self.assertEqual(fdir.scheme, "file")
223 fnew = dir.join("test_jf.txt")
224 fnew.write(b"Content")
226 fnew2 = fdir.join(new2name)
227 fnew2.write(b"Content")
228 self.assertTrue(fnew2.ospath.endswith(new2name))
229 self.assertNotIn("###", fnew2.path)
231 self.assertEqual(fnew.read(), fnew2.read())
233 # Test that children relative to schemeless and file schemes
234 # still return the same unquoted name
235 self.assertEqual(fnew2.relative_to(fdir), new2name)
236 self.assertEqual(fnew2.relative_to(dir), new2name)
237 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
238 self.assertEqual(new2.relative_to(dir), new2name)
240 # Check for double quoting
241 plus_path = "/a/b/c+d/"
242 with self.assertLogs(level="WARNING"):
243 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
244 self.assertEqual(uri.ospath, plus_path)
247@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
248@mock_s3
249class S3URITestCase(unittest.TestCase):
250 """Tests involving S3"""
252 bucketName = "any_bucket"
253 """Bucket name to use in tests"""
255 def setUp(self):
256 # Local test directory
257 self.tmpdir = tempfile.mkdtemp()
259 # set up some fake credentials if they do not exist
260 self.usingDummyCredentials = setAwsEnvCredentials()
262 # MOTO needs to know that we expect Bucket bucketname to exist
263 s3 = boto3.resource("s3")
264 s3.create_bucket(Bucket=self.bucketName)
266 def tearDown(self):
267 s3 = boto3.resource("s3")
268 bucket = s3.Bucket(self.bucketName)
269 try:
270 bucket.objects.all().delete()
271 except botocore.exceptions.ClientError as e:
272 if e.response["Error"]["Code"] == "404":
273 # the key was not reachable - pass
274 pass
275 else:
276 raise
278 bucket = s3.Bucket(self.bucketName)
279 bucket.delete()
281 # unset any potentially set dummy credentials
282 if self.usingDummyCredentials:
283 unsetAwsEnvCredentials()
285 shutil.rmtree(self.tmpdir, ignore_errors=True)
287 def makeS3Uri(self, path):
288 return f"s3://{self.bucketName}/{path}"
290 def testTransfer(self):
291 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
292 content = "Content is some content\nwith something to say\n\n"
293 src.write(content.encode())
295 dest = ButlerURI(self.makeS3Uri("test.txt"))
296 self.assertFalse(dest.exists())
297 dest.transfer_from(src, transfer="copy")
298 self.assertTrue(dest.exists())
300 dest2 = ButlerURI(self.makeS3Uri("copied.txt"))
301 dest2.transfer_from(dest, transfer="copy")
302 self.assertTrue(dest2.exists())
304 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt"))
305 local.transfer_from(dest2, transfer="copy")
306 with open(local.ospath, "r") as fd:
307 new_content = fd.read()
308 self.assertEqual(new_content, content)
310 with self.assertRaises(ValueError):
311 dest2.transfer_from(local, transfer="symlink")
313 b = dest.read()
314 self.assertEqual(b.decode(), new_content)
316 nbytes = 10
317 subset = dest.read(size=nbytes)
318 self.assertEqual(len(subset), nbytes) # Extra byte comes back
319 self.assertEqual(subset.decode(), content[:nbytes])
321 with self.assertRaises(FileExistsError):
322 dest.transfer_from(src, transfer="copy")
324 dest.transfer_from(src, transfer="copy", overwrite=True)
326 def testWrite(self):
327 s3write = ButlerURI(self.makeS3Uri("created.txt"))
328 content = "abcdefghijklmnopqrstuv\n"
329 s3write.write(content.encode())
330 self.assertEqual(s3write.read().decode(), content)
332 def testRelative(self):
333 """Check that we can get subpaths back from two URIs"""
334 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
335 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt"))
337 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
339 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt"))
340 self.assertFalse(not_child.relative_to(parent))
342 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
343 self.assertFalse(child.relative_to(not_s3))
345 def testQuoting(self):
346 """Check that quoting works."""
347 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
348 subpath = "rootdir/dir1+/file?.txt"
349 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath)))
351 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
352 self.assertEqual(child.basename(), "file?.txt")
353 self.assertEqual(child.relativeToPathRoot, subpath)
354 self.assertIn("%", child.path)
355 self.assertEqual(child.unquoted_path, "/" + subpath)
358# Mock required environment variables during tests
359@unittest.mock.patch.dict(os.environ, {"WEBDAV_AUTH_METHOD": "TOKEN",
360 "WEBDAV_BEARER_TOKEN": "XXXXXX"})
361class WebdavURITestCase(unittest.TestCase):
363 def setUp(self):
364 serverRoot = "www.not-exists.orgx"
365 existingFolderName = "existingFolder"
366 existingFileName = "existingFile"
367 notExistingFileName = "notExistingFile"
369 self.baseURL = ButlerURI(
370 f"https://{serverRoot}", forceDirectory=True)
371 self.existingFileButlerURI = ButlerURI(
372 f"https://{serverRoot}/{existingFolderName}/{existingFileName}")
373 self.notExistingFileButlerURI = ButlerURI(
374 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}")
375 self.existingFolderButlerURI = ButlerURI(
376 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True)
377 self.notExistingFolderButlerURI = ButlerURI(
378 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True)
380 # Need to declare the options
381 responses.add(responses.OPTIONS,
382 self.baseURL.geturl(),
383 status=200, headers={"DAV": "1,2,3"})
385 # Used by ButlerHttpURI.exists()
386 responses.add(responses.HEAD,
387 self.existingFileButlerURI.geturl(),
388 status=200, headers={'Content-Length': '1024'})
389 responses.add(responses.HEAD,
390 self.notExistingFileButlerURI.geturl(),
391 status=404)
393 # Used by ButlerHttpURI.read()
394 responses.add(responses.GET,
395 self.existingFileButlerURI.geturl(),
396 status=200,
397 body=str.encode("It works!"))
398 responses.add(responses.GET,
399 self.notExistingFileButlerURI.geturl(),
400 status=404)
402 # Used by ButlerHttpURI.write()
403 responses.add(responses.PUT,
404 self.existingFileButlerURI.geturl(),
405 status=200)
407 # Used by ButlerHttpURI.transfer_from()
408 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
409 method="COPY",
410 headers={"Destination": self.existingFileButlerURI.geturl()},
411 status=200))
412 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
413 method="COPY",
414 headers={"Destination": self.notExistingFileButlerURI.geturl()},
415 status=200))
416 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
417 method="MOVE",
418 headers={"Destination": self.notExistingFileButlerURI.geturl()},
419 status=200))
421 # Used by ButlerHttpURI.remove()
422 responses.add(responses.DELETE,
423 self.existingFileButlerURI.geturl(),
424 status=200)
425 responses.add(responses.DELETE,
426 self.notExistingFileButlerURI.geturl(),
427 status=404)
429 # Used by ButlerHttpURI.mkdir()
430 responses.add(responses.HEAD,
431 self.existingFolderButlerURI.geturl(),
432 status=200, headers={'Content-Length': '1024'})
433 responses.add(responses.HEAD,
434 self.baseURL.geturl(),
435 status=200, headers={'Content-Length': '1024'})
436 responses.add(responses.HEAD,
437 self.notExistingFolderButlerURI.geturl(),
438 status=404)
439 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(),
440 method="MKCOL",
441 status=201))
442 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(),
443 method="MKCOL",
444 status=403))
446 @responses.activate
447 def testExists(self):
449 self.assertTrue(self.existingFileButlerURI.exists())
450 self.assertFalse(self.notExistingFileButlerURI.exists())
452 @responses.activate
453 def testRemove(self):
455 self.assertIsNone(self.existingFileButlerURI.remove())
456 with self.assertRaises(FileNotFoundError):
457 self.notExistingFileButlerURI.remove()
459 @responses.activate
460 def testMkdir(self):
462 # The mock means that we can't check this now exists
463 self.notExistingFolderButlerURI.mkdir()
465 # This should do nothing
466 self.existingFolderButlerURI.mkdir()
468 with self.assertRaises(ValueError):
469 self.notExistingFileButlerURI.mkdir()
471 @responses.activate
472 def testRead(self):
474 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!")
475 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.")
476 with self.assertRaises(FileNotFoundError):
477 self.notExistingFileButlerURI.read()
479 @responses.activate
480 def testWrite(self):
482 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content.")))
483 with self.assertRaises(FileExistsError):
484 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False)
486 @responses.activate
487 def testTransfer(self):
489 self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
490 src=self.existingFileButlerURI))
491 self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
492 src=self.existingFileButlerURI,
493 transfer="move"))
494 with self.assertRaises(FileExistsError):
495 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI)
496 with self.assertRaises(ValueError):
497 self.notExistingFileButlerURI.transfer_from(
498 src=self.existingFileButlerURI,
499 transfer="unsupported")
501 def testParent(self):
503 self.assertEqual(self.existingFolderButlerURI.geturl(),
504 self.notExistingFileButlerURI.parent().geturl())
505 self.assertEqual(self.baseURL.geturl(),
506 self.baseURL.parent().geturl())
507 self.assertEqual(self.existingFileButlerURI.parent().geturl(),
508 self.existingFileButlerURI.dirname().geturl())
511if __name__ == "__main__": 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true
512 unittest.main()