Coverage for tests/test_uri.py: 13%
670 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import glob
23import importlib
24import os
25import shutil
26import stat
27import tempfile
28import unittest
29import urllib.parse
31import requests
32import responses
34try:
35 import boto3
36 import botocore
37 from moto import mock_s3
38except ImportError:
39 boto3 = None
41 def mock_s3(cls):
42 """A no-op decorator in case moto mock_s3 can not be imported."""
43 return cls
46import lsst.daf.butler.core._butlerUri.http as httpUri
47from lsst.daf.butler import ButlerURI
48from lsst.daf.butler.core._butlerUri.s3utils import (
49 setAwsEnvCredentials,
50 unsetAwsEnvCredentials,
51)
52from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
54TESTDIR = os.path.abspath(os.path.dirname(__file__))
57class FileURITestCase(unittest.TestCase):
58 """Concrete tests for local files"""
60 def setUp(self):
61 # Use a local tempdir because on macOS the temp dirs use symlinks
62 # so relsymlink gets quite confused.
63 self.tmpdir = makeTestTempDir(TESTDIR)
65 def tearDown(self):
66 removeTestTempDir(self.tmpdir)
68 def testFile(self):
69 file = os.path.join(self.tmpdir, "test.txt")
70 uri = ButlerURI(file)
71 self.assertFalse(uri.exists(), f"{uri} should not exist")
72 self.assertEqual(uri.ospath, file)
74 content = "abcdefghijklmnopqrstuv\n"
75 uri.write(content.encode())
76 self.assertTrue(os.path.exists(file), "File should exist locally")
77 self.assertTrue(uri.exists(), f"{uri} should now exist")
78 self.assertEqual(uri.read().decode(), content)
79 self.assertEqual(uri.size(), len(content.encode()))
81 with self.assertRaises(FileNotFoundError):
82 ButlerURI("file/not/there.txt").size()
84 # Check that creating a URI from a URI returns the same thing
85 uri2 = ButlerURI(uri)
86 self.assertEqual(uri, uri2)
87 self.assertEqual(id(uri), id(uri2))
89 with self.assertRaises(ValueError):
90 # Scheme-less URIs are not allowed to support non-file roots
91 # at the present time. This may change in the future to become
92 # equivalent to ButlerURI.join()
93 ButlerURI("a/b.txt", root=ButlerURI("s3://bucket/a/b/"))
95 def testExtension(self):
96 file = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
97 self.assertEqual(file.updatedExtension(None), file)
98 self.assertEqual(file.updatedExtension(".txt"), file)
99 self.assertEqual(id(file.updatedExtension(".txt")), id(file))
101 fits = file.updatedExtension(".fits.gz")
102 self.assertEqual(fits.basename(), "test.fits.gz")
103 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
105 def testRelative(self):
106 """Check that we can get subpaths back from two URIs"""
107 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
108 self.assertTrue(parent.isdir())
109 child = ButlerURI(
110 os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True
111 )
113 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
115 not_child = ButlerURI("/a/b/dir1/file.txt")
116 self.assertIsNone(not_child.relative_to(parent))
117 self.assertFalse(not_child.isdir())
119 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
120 self.assertIsNone(child.relative_to(not_directory))
122 # Relative URIs
123 parent = ButlerURI("a/b/", forceAbsolute=False)
124 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False)
125 self.assertFalse(child.scheme)
126 self.assertEqual(child.relative_to(parent), "c/d.txt")
128 # File URI and schemeless URI
129 parent = ButlerURI("file:/a/b/c/")
130 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
132 # If the child is relative and the parent is absolute we assume
133 # that the child is a child of the parent unless it uses ".."
134 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
136 child = ButlerURI("../e/f/g.txt", forceAbsolute=False)
137 self.assertIsNone(child.relative_to(parent))
139 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
140 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
142 # Test non-file root with relative path.
143 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
144 parent = ButlerURI("s3://hello/a/b/c/")
145 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
147 # Test with different netloc
148 child = ButlerURI("http://my.host/a/b/c.txt")
149 parent = ButlerURI("http://other.host/a/")
150 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
152 # Schemeless absolute child.
153 # Schemeless absolute URI is constructed using root= parameter.
154 parent = ButlerURI("file:///a/b/c/")
155 child = ButlerURI("d/e.txt", root=parent)
156 self.assertEqual(
157 child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})"
158 )
160 parent = ButlerURI("c/", root="/a/b/")
161 self.assertEqual(
162 child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})"
163 )
165 # Absolute schemeless child with relative parent will always fail.
166 parent = ButlerURI("d/e.txt", forceAbsolute=False)
167 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
169 def testParents(self):
170 """Test of splitting and parent walking."""
171 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
172 child_file = parent.join("subdir/file.txt")
173 self.assertFalse(child_file.isdir())
174 child_subdir, file = child_file.split()
175 self.assertEqual(file, "file.txt")
176 self.assertTrue(child_subdir.isdir())
177 self.assertEqual(child_file.dirname(), child_subdir)
178 self.assertEqual(child_file.basename(), file)
179 self.assertEqual(child_file.parent(), child_subdir)
180 derived_parent = child_subdir.parent()
181 self.assertEqual(derived_parent, parent)
182 self.assertTrue(derived_parent.isdir())
183 self.assertEqual(child_file.parent().parent(), parent)
185 def testEnvVar(self):
186 """Test that environment variables are expanded."""
188 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}):
189 uri = ButlerURI("${MY_TEST_DIR}/d.txt")
190 self.assertEqual(uri.path, "/a/b/c/d.txt")
191 self.assertEqual(uri.scheme, "file")
193 # This will not expand
194 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False)
195 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt")
196 self.assertFalse(uri.scheme)
198 def testMkdir(self):
199 tmpdir = ButlerURI(self.tmpdir)
200 newdir = tmpdir.join("newdir/seconddir")
201 newdir.mkdir()
202 self.assertTrue(newdir.exists())
203 newfile = newdir.join("temp.txt")
204 newfile.write("Data".encode())
205 self.assertTrue(newfile.exists())
207 def testTransfer(self):
208 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
209 content = "Content is some content\nwith something to say\n\n"
210 src.write(content.encode())
212 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
213 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
214 dest.transfer_from(src, transfer=mode)
215 self.assertTrue(
216 dest.exists(), f"Check that {dest} exists (transfer={mode})"
217 )
219 with open(dest.ospath, "r") as fh:
220 new_content = fh.read()
221 self.assertEqual(new_content, content)
223 if mode in ("symlink", "relsymlink"):
224 self.assertTrue(
225 os.path.islink(dest.ospath), f"Check that {dest} is symlink"
226 )
228 # If the source and destination are hardlinks of each other
229 # the transfer should work even if overwrite=False.
230 if mode in ("link", "hardlink"):
231 dest.transfer_from(src, transfer=mode)
232 else:
233 with self.assertRaises(
234 FileExistsError,
235 msg=f"Overwrite of {dest} should not be allowed ({mode})",
236 ):
237 dest.transfer_from(src, transfer=mode)
239 dest.transfer_from(src, transfer=mode, overwrite=True)
241 os.remove(dest.ospath)
243 b = src.read()
244 self.assertEqual(b.decode(), new_content)
246 nbytes = 10
247 subset = src.read(size=nbytes)
248 self.assertEqual(len(subset), nbytes)
249 self.assertEqual(subset.decode(), content[:nbytes])
251 with self.assertRaises(ValueError):
252 src.transfer_from(src, transfer="unknown")
254 def testTransferIdentical(self):
255 """Test overwrite of identical files."""
256 dir1 = ButlerURI(os.path.join(self.tmpdir, "dir1"), forceDirectory=True)
257 dir1.mkdir()
258 dir2 = os.path.join(self.tmpdir, "dir2")
259 os.symlink(dir1.ospath, dir2)
261 # Write a test file.
262 src_file = dir1.join("test.txt")
263 content = "0123456"
264 src_file.write(content.encode())
266 # Construct URI to destination that should be identical.
267 dest_file = ButlerURI(os.path.join(dir2), forceDirectory=True).join("test.txt")
268 self.assertTrue(dest_file.exists())
269 self.assertNotEqual(src_file, dest_file)
271 # Transfer it over itself.
272 dest_file.transfer_from(src_file, transfer="symlink", overwrite=True)
273 new_content = dest_file.read().decode()
274 self.assertEqual(content, new_content)
276 def testResource(self):
277 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml")
278 self.assertTrue(u.exists(), f"Check {u} exists")
280 content = u.read().decode()
281 self.assertTrue(content.startswith("datastore:"))
283 truncated = u.read(size=9).decode()
284 self.assertEqual(truncated, "datastore")
286 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True)
287 self.assertTrue(u.exists(), f"Check directory {d} exists")
289 j = d.join("datastore.yaml")
290 self.assertEqual(u, j)
291 self.assertFalse(j.dirLike)
292 self.assertFalse(j.isdir())
293 not_there = d.join("not-there.yaml")
294 self.assertFalse(not_there.exists())
296 bad = ButlerURI("resource://bad.module/not.yaml")
297 multi = ButlerURI.mexists([u, bad, not_there])
298 self.assertTrue(multi[u])
299 self.assertFalse(multi[bad])
300 self.assertFalse(multi[not_there])
302 def testEscapes(self):
303 """Special characters in file paths"""
304 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
305 self.assertFalse(src.scheme)
306 src.write(b"Some content")
307 self.assertTrue(src.exists())
309 # abspath always returns a file scheme
310 file = src.abspath()
311 self.assertTrue(file.exists())
312 self.assertIn("???", file.ospath)
313 self.assertNotIn("???", file.path)
315 file = file.updatedFile("tests??.txt")
316 self.assertNotIn("??.txt", file.path)
317 file.write(b"Other content")
318 self.assertEqual(file.read(), b"Other content")
320 src = src.updatedFile("tests??.txt")
321 self.assertIn("??.txt", src.path)
322 self.assertEqual(
323 file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}"
324 )
326 # File URI and schemeless URI
327 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
328 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
329 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
331 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
332 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
334 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
335 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
337 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
339 # Schemeless so should not quote
340 dir = ButlerURI(
341 "bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True
342 )
343 self.assertIn("???", dir.ospath)
344 self.assertIn("???", dir.path)
345 self.assertFalse(dir.scheme)
347 # dir.join() morphs into a file scheme
348 new = dir.join("test_j.txt")
349 self.assertIn("???", new.ospath, f"Checking {new}")
350 new.write(b"Content")
352 new2name = "###/test??.txt"
353 new2 = dir.join(new2name)
354 self.assertIn("???", new2.ospath)
355 new2.write(b"Content")
356 self.assertTrue(new2.ospath.endswith(new2name))
357 self.assertEqual(new.read(), new2.read())
359 fdir = dir.abspath()
360 self.assertNotIn("???", fdir.path)
361 self.assertIn("???", fdir.ospath)
362 self.assertEqual(fdir.scheme, "file")
363 fnew = dir.join("test_jf.txt")
364 fnew.write(b"Content")
366 fnew2 = fdir.join(new2name)
367 fnew2.write(b"Content")
368 self.assertTrue(fnew2.ospath.endswith(new2name))
369 self.assertNotIn("###", fnew2.path)
371 self.assertEqual(fnew.read(), fnew2.read())
373 # Test that children relative to schemeless and file schemes
374 # still return the same unquoted name
375 self.assertEqual(
376 fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})"
377 )
378 self.assertEqual(
379 fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})"
380 )
381 self.assertEqual(
382 new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})"
383 )
384 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
386 # Check for double quoting
387 plus_path = "/a/b/c+d/"
388 with self.assertLogs(level="WARNING"):
389 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
390 self.assertEqual(uri.ospath, plus_path)
392 # Check that # is not escaped for schemeless URIs
393 hash_path = "/a/b#/c&d#xyz"
394 hpos = hash_path.rfind("#")
395 uri = ButlerURI(hash_path)
396 self.assertEqual(uri.ospath, hash_path[:hpos])
397 self.assertEqual(uri.fragment, hash_path[hpos + 1:])
399 def testHash(self):
400 """Test that we can store URIs in sets and as keys."""
401 uri1 = ButlerURI(TESTDIR)
402 uri2 = uri1.join("test/")
403 s = {uri1, uri2}
404 self.assertIn(uri1, s)
406 d = {uri1: "1", uri2: "2"}
407 self.assertEqual(d[uri2], "2")
409 def testWalk(self):
410 """Test ButlerURI.walk()."""
411 test_dir_uri = ButlerURI(TESTDIR)
413 file = test_dir_uri.join("config/basic/butler.yaml")
414 found = list(ButlerURI.findFileResources([file]))
415 self.assertEqual(found[0], file)
417 # Compare against the full local paths
418 expected = set(
419 p
420 for p in glob.glob(os.path.join(TESTDIR, "config", "**"), recursive=True)
421 if os.path.isfile(p)
422 )
423 found = set(
424 u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")])
425 )
426 self.assertEqual(found, expected)
428 # Now solely the YAML files
429 expected_yaml = set(
430 glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"), recursive=True)
431 )
432 found = set(
433 u.ospath
434 for u in ButlerURI.findFileResources(
435 [test_dir_uri.join("config")], file_filter=r".*\.yaml$"
436 )
437 )
438 self.assertEqual(found, expected_yaml)
440 # Now two explicit directories and a file
441 expected = set(
442 glob.glob(
443 os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"), recursive=True
444 )
445 )
446 expected.update(
447 set(
448 glob.glob(
449 os.path.join(TESTDIR, "config", "**", "templates", "*.yaml"),
450 recursive=True,
451 )
452 )
453 )
454 expected.add(file.ospath)
456 found = set(
457 u.ospath
458 for u in ButlerURI.findFileResources(
459 [
460 file,
461 test_dir_uri.join("config/basic"),
462 test_dir_uri.join("config/templates"),
463 ],
464 file_filter=r".*\.yaml$",
465 )
466 )
467 self.assertEqual(found, expected)
469 # Group by directory -- find everything and compare it with what
470 # we expected to be there in total. We expect to find 9 directories
471 # containing yaml files so make sure we only iterate 9 times.
472 found_yaml = set()
473 counter = 0
474 for uris in ButlerURI.findFileResources(
475 [file, test_dir_uri.join("config/")], file_filter=r".*\.yaml$", grouped=True
476 ):
477 found = set(u.ospath for u in uris)
478 if found:
479 counter += 1
481 found_yaml.update(found)
483 self.assertEqual(found_yaml, expected_yaml)
484 self.assertEqual(counter, 9)
486 # Grouping but check that single files are returned in a single group
487 # at the end
488 file2 = test_dir_uri.join("config/templates/templates-bad.yaml")
489 found = list(
490 ButlerURI.findFileResources(
491 [file, file2, test_dir_uri.join("config/dbAuth")], grouped=True
492 )
493 )
494 self.assertEqual(len(found), 2)
495 self.assertEqual(list(found[1]), [file, file2])
497 with self.assertRaises(ValueError):
498 list(file.walk())
500 def testRootURI(self):
501 """Test ButlerURI.root_uri()."""
502 uri = ButlerURI("https://www.notexist.com:8080/file/test")
503 uri2 = ButlerURI("s3://www.notexist.com/file/test")
504 self.assertEqual(uri.root_uri().geturl(), "https://www.notexist.com:8080/")
505 self.assertEqual(uri2.root_uri().geturl(), "s3://www.notexist.com/")
507 def testJoin(self):
508 """Test .join method."""
510 root_str = "s3://bucket/hsc/payload/"
511 root = ButlerURI(root_str)
513 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
514 add_dir = root.join("b/c/d/")
515 self.assertTrue(add_dir.isdir())
516 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
518 quote_example = "b&c.t@x#t"
519 needs_quote = root.join(quote_example)
520 self.assertEqual(needs_quote.unquoted_path, f"/hsc/payload/{quote_example}")
522 other = ButlerURI("file://localhost/test.txt")
523 self.assertEqual(root.join(other), other)
524 self.assertEqual(other.join("b/new.txt").geturl(), "file://localhost/b/new.txt")
526 joined = ButlerURI("s3://bucket/hsc/payload/").join(
527 ButlerURI("test.qgraph", forceAbsolute=False)
528 )
529 self.assertEqual(joined, ButlerURI("s3://bucket/hsc/payload/test.qgraph"))
531 with self.assertRaises(ValueError):
532 ButlerURI("s3://bucket/hsc/payload/").join(ButlerURI("test.qgraph"))
534 def testTemporary(self):
535 with ButlerURI.temporary_uri(suffix=".json") as tmp:
536 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
537 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
538 self.assertFalse(tmp.exists(), f"uri: {tmp}")
539 tmp.write(b"abcd")
540 self.assertTrue(tmp.exists(), f"uri: {tmp}")
541 self.assertTrue(tmp.isTemporary)
542 self.assertFalse(tmp.exists(), f"uri: {tmp}")
544 tmpdir = ButlerURI(self.tmpdir, forceDirectory=True)
545 with ButlerURI.temporary_uri(prefix=tmpdir, suffix=".yaml") as tmp:
546 # Use a specified tmpdir and check it is okay for the file
547 # to not be created.
548 self.assertFalse(tmp.exists(), f"uri: {tmp}")
549 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
552@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
553@mock_s3
554class S3URITestCase(unittest.TestCase):
555 """Tests involving S3"""
557 bucketName = "any_bucket"
558 """Bucket name to use in tests"""
560 def setUp(self):
561 # Local test directory
562 self.tmpdir = makeTestTempDir(TESTDIR)
564 # set up some fake credentials if they do not exist
565 self.usingDummyCredentials = setAwsEnvCredentials()
567 # MOTO needs to know that we expect Bucket bucketname to exist
568 s3 = boto3.resource("s3")
569 s3.create_bucket(Bucket=self.bucketName)
571 def tearDown(self):
572 s3 = boto3.resource("s3")
573 bucket = s3.Bucket(self.bucketName)
574 try:
575 bucket.objects.all().delete()
576 except botocore.exceptions.ClientError as e:
577 if e.response["Error"]["Code"] == "404":
578 # the key was not reachable - pass
579 pass
580 else:
581 raise
583 bucket = s3.Bucket(self.bucketName)
584 bucket.delete()
586 # unset any potentially set dummy credentials
587 if self.usingDummyCredentials:
588 unsetAwsEnvCredentials()
590 shutil.rmtree(self.tmpdir, ignore_errors=True)
592 def makeS3Uri(self, path):
593 return f"s3://{self.bucketName}/{path}"
595 def testTransfer(self):
596 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
597 content = "Content is some content\nwith something to say\n\n"
598 src.write(content.encode())
599 self.assertTrue(src.exists())
600 self.assertEqual(src.size(), len(content.encode()))
602 dest = ButlerURI(self.makeS3Uri("test.txt"))
603 self.assertFalse(dest.exists())
605 with self.assertRaises(FileNotFoundError):
606 dest.size()
608 dest.transfer_from(src, transfer="copy")
609 self.assertTrue(dest.exists())
611 dest2 = ButlerURI(self.makeS3Uri("copied.txt"))
612 dest2.transfer_from(dest, transfer="copy")
613 self.assertTrue(dest2.exists())
615 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt"))
616 local.transfer_from(dest2, transfer="copy")
617 with open(local.ospath, "r") as fd:
618 new_content = fd.read()
619 self.assertEqual(new_content, content)
621 with self.assertRaises(ValueError):
622 dest2.transfer_from(local, transfer="symlink")
624 b = dest.read()
625 self.assertEqual(b.decode(), new_content)
627 nbytes = 10
628 subset = dest.read(size=nbytes)
629 self.assertEqual(len(subset), nbytes) # Extra byte comes back
630 self.assertEqual(subset.decode(), content[:nbytes])
632 with self.assertRaises(FileExistsError):
633 dest.transfer_from(src, transfer="copy")
635 dest.transfer_from(src, transfer="copy", overwrite=True)
637 def testWalk(self):
638 """Test that we can list an S3 bucket"""
639 # Files we want to create
640 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json")
641 expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected]
642 for uri in expected_uris:
643 # Doesn't matter what we write
644 uri.write("123".encode())
646 # Find all the files in the a/ tree
647 found = set(
648 uri.path
649 for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))])
650 )
651 self.assertEqual(found, {uri.path for uri in expected_uris})
653 # Find all the files in the a/ tree but group by folder
654 found = ButlerURI.findFileResources(
655 [ButlerURI(self.makeS3Uri("a/"))], grouped=True
656 )
657 expected = (
658 ("/a/x.txt", "/a/y.txt", "/a/z.json"),
659 ("/a/b/w.txt",),
660 ("/a/b/c/d/v.json",),
661 )
663 for got, expect in zip(found, expected):
664 self.assertEqual(tuple(u.path for u in got), expect)
666 # Find only JSON files
667 found = set(
668 uri.path
669 for uri in ButlerURI.findFileResources(
670 [ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$"
671 )
672 )
673 self.assertEqual(
674 found, {uri.path for uri in expected_uris if uri.path.endswith(".json")}
675 )
677 # JSON files grouped by directory
678 found = ButlerURI.findFileResources(
679 [ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$", grouped=True
680 )
681 expected = (("/a/z.json",), ("/a/b/c/d/v.json",))
683 for got, expect in zip(found, expected):
684 self.assertEqual(tuple(u.path for u in got), expect)
686 # Check pagination works with large numbers of files. S3 API limits
687 # us to 1000 response per list_objects call so create lots of files
688 created = set()
689 counter = 1
690 n_dir1 = 1100
691 while counter <= n_dir1:
692 new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt"))
693 new.write(f"{counter}".encode())
694 created.add(str(new))
695 counter += 1
696 counter = 1
697 # Put some in a subdirectory to make sure we are looking in a
698 # hierarchy.
699 n_dir2 = 100
700 while counter <= n_dir2:
701 new = ButlerURI(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt"))
702 new.write(f"{counter}".encode())
703 created.add(str(new))
704 counter += 1
706 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))])
707 self.assertEqual({str(u) for u in found}, created)
709 # Again with grouping.
710 found = list(
711 ButlerURI.findFileResources(
712 [ButlerURI(self.makeS3Uri("test/"))], grouped=True
713 )
714 )
715 self.assertEqual(len(found), 2)
716 dir_1 = list(found[0])
717 dir_2 = list(found[1])
718 self.assertEqual(len(dir_1), n_dir1)
719 self.assertEqual(len(dir_2), n_dir2)
721 def testWrite(self):
722 s3write = ButlerURI(self.makeS3Uri("created.txt"))
723 content = "abcdefghijklmnopqrstuv\n"
724 s3write.write(content.encode())
725 self.assertEqual(s3write.read().decode(), content)
727 def testTemporary(self):
728 s3root = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
729 with ButlerURI.temporary_uri(prefix=s3root, suffix=".json") as tmp:
730 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
731 self.assertEqual(tmp.scheme, "s3", f"uri: {tmp}")
732 self.assertEqual(tmp.parent(), s3root)
733 basename = tmp.basename()
734 content = "abcd"
735 tmp.write(content.encode())
736 self.assertTrue(tmp.exists(), f"uri: {tmp}")
737 self.assertFalse(tmp.exists())
739 # Again without writing anything, to check that there is no complaint
740 # on exit of context manager.
741 with ButlerURI.temporary_uri(prefix=s3root, suffix=".json") as tmp:
742 self.assertFalse(tmp.exists())
743 # Check that the file has a different name than before.
744 self.assertNotEqual(tmp.basename(), basename, f"uri: {tmp}")
745 self.assertFalse(tmp.exists())
747 def testRelative(self):
748 """Check that we can get subpaths back from two URIs"""
749 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
750 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt"))
752 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
754 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt"))
755 self.assertFalse(not_child.relative_to(parent))
757 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
758 self.assertFalse(child.relative_to(not_s3))
760 def testQuoting(self):
761 """Check that quoting works."""
762 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
763 subpath = "rootdir/dir1+/file?.txt"
764 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath)))
766 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
767 self.assertEqual(child.basename(), "file?.txt")
768 self.assertEqual(child.relativeToPathRoot, subpath)
769 self.assertIn("%", child.path)
770 self.assertEqual(child.unquoted_path, "/" + subpath)
773# Mock required environment variables during tests
774class WebdavURITestCase(unittest.TestCase):
775 def setUp(self):
776 serverRoot = "www.not-exists.orgx"
777 existingFolderName = "existingFolder"
778 existingFileName = "existingFile"
779 notExistingFileName = "notExistingFile"
781 self.baseURL = ButlerURI(f"https://{serverRoot}", forceDirectory=True)
782 self.existingFileButlerURI = ButlerURI(
783 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
784 )
785 self.notExistingFileButlerURI = ButlerURI(
786 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
787 )
788 self.existingFolderButlerURI = ButlerURI(
789 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
790 )
791 self.notExistingFolderButlerURI = ButlerURI(
792 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
793 )
795 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR))
797 # Need to declare the options
798 responses.add(
799 responses.OPTIONS,
800 self.baseURL.geturl(),
801 status=200,
802 headers={"DAV": "1,2,3"},
803 )
805 # Used by ButlerhttpUri.exists()
806 responses.add(
807 responses.HEAD,
808 self.existingFileButlerURI.geturl(),
809 status=200,
810 headers={"Content-Length": "1024"},
811 )
812 responses.add(
813 responses.HEAD, self.notExistingFileButlerURI.geturl(), status=404
814 )
816 # Used by ButlerhttpUri.read()
817 responses.add(
818 responses.GET,
819 self.existingFileButlerURI.geturl(),
820 status=200,
821 body=str.encode("It works!"),
822 )
823 responses.add(responses.GET, self.notExistingFileButlerURI.geturl(), status=404)
825 # Used by ButlerhttpUri.write()
826 responses.add(responses.PUT, self.existingFileButlerURI.geturl(), status=201)
828 # Used by ButlerhttpUri.transfer_from()
829 responses.add(
830 responses.Response(
831 url=self.existingFileButlerURI.geturl(),
832 method="COPY",
833 headers={"Destination": self.existingFileButlerURI.geturl()},
834 status=201,
835 )
836 )
837 responses.add(
838 responses.Response(
839 url=self.existingFileButlerURI.geturl(),
840 method="COPY",
841 headers={"Destination": self.notExistingFileButlerURI.geturl()},
842 status=201,
843 )
844 )
845 responses.add(
846 responses.Response(
847 url=self.existingFileButlerURI.geturl(),
848 method="MOVE",
849 headers={"Destination": self.notExistingFileButlerURI.geturl()},
850 status=201,
851 )
852 )
854 # Used by ButlerhttpUri.remove()
855 responses.add(responses.DELETE, self.existingFileButlerURI.geturl(), status=200)
856 responses.add(
857 responses.DELETE, self.notExistingFileButlerURI.geturl(), status=404
858 )
860 # Used by ButlerhttpUri.mkdir()
861 responses.add(
862 responses.HEAD,
863 self.existingFolderButlerURI.geturl(),
864 status=200,
865 headers={"Content-Length": "1024"},
866 )
867 responses.add(
868 responses.HEAD,
869 self.baseURL.geturl(),
870 status=200,
871 headers={"Content-Length": "1024"},
872 )
873 responses.add(
874 responses.HEAD, self.notExistingFolderButlerURI.geturl(), status=404
875 )
876 responses.add(
877 responses.Response(
878 url=self.notExistingFolderButlerURI.geturl(), method="MKCOL", status=201
879 )
880 )
881 responses.add(
882 responses.Response(
883 url=self.existingFolderButlerURI.geturl(), method="MKCOL", status=403
884 )
885 )
887 # Used by ButlerHttpURI._do_put()
888 self.redirectPathNoExpect = ButlerURI(
889 f"https://{serverRoot}/redirect-no-expect/file"
890 )
891 self.redirectPathExpect = ButlerURI(
892 f"https://{serverRoot}/redirect-expect/file"
893 )
894 redirected_url = f"https://{serverRoot}/redirect/location"
895 responses.add(
896 responses.PUT,
897 self.redirectPathNoExpect.geturl(),
898 headers={"Location": redirected_url},
899 status=307,
900 )
901 responses.add(
902 responses.PUT,
903 self.redirectPathExpect.geturl(),
904 headers={"Location": redirected_url},
905 status=307,
906 match=[
907 responses.matchers.header_matcher(
908 {"Content-Length": "0", "Expect": "100-continue"}
909 )
910 ],
911 )
912 responses.add(responses.PUT, redirected_url, status=202)
914 def tearDown(self):
915 if self.tmpdir and self.tmpdir.isLocal:
916 removeTestTempDir(self.tmpdir.ospath)
918 @responses.activate
919 def testExists(self):
921 self.assertTrue(self.existingFileButlerURI.exists())
922 self.assertFalse(self.notExistingFileButlerURI.exists())
924 self.assertEqual(self.existingFileButlerURI.size(), 1024)
925 with self.assertRaises(FileNotFoundError):
926 self.notExistingFileButlerURI.size()
928 @responses.activate
929 def testRemove(self):
931 self.assertIsNone(self.existingFileButlerURI.remove())
932 with self.assertRaises(FileNotFoundError):
933 self.notExistingFileButlerURI.remove()
935 @responses.activate
936 def testMkdir(self):
938 # The mock means that we can't check this now exists
939 self.notExistingFolderButlerURI.mkdir()
941 # This should do nothing
942 self.existingFolderButlerURI.mkdir()
944 with self.assertRaises(ValueError):
945 self.notExistingFileButlerURI.mkdir()
947 @responses.activate
948 def testRead(self):
950 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!")
951 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.")
952 with self.assertRaises(FileNotFoundError):
953 self.notExistingFileButlerURI.read()
955 # Run this twice to ensure use of cache in code coverage
956 for _ in (1, 2):
957 with self.existingFileButlerURI.as_local() as local_uri:
958 self.assertTrue(local_uri.isLocal)
959 content = local_uri.read().decode()
960 self.assertEqual(content, "It works!")
962 # Ensure the LSST_RESOURCES_TMPDIR environment variable is used if set
963 # (requires to reset the cache in _TMPDIR)
964 httpUri._TMPDIR = None
965 tmpdir = makeTestTempDir(TESTDIR)
966 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": tmpdir}):
967 with self.existingFileButlerURI.as_local() as local_uri:
968 self.assertTrue(local_uri.isLocal)
969 content = local_uri.read().decode()
970 self.assertEqual(content, "It works!")
971 self.assertIsNotNone(local_uri.relative_to(ButlerURI(tmpdir)))
973 @responses.activate
974 def testWrite(self):
976 self.assertIsNone(
977 self.existingFileButlerURI.write(data=str.encode("Some content."))
978 )
979 with self.assertRaises(FileExistsError):
980 self.existingFileButlerURI.write(
981 data=str.encode("Some content."), overwrite=False
982 )
984 @responses.activate
985 def test_do_put_with_redirection(self):
987 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER.
988 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
989 importlib.reload(httpUri)
990 body = str.encode("any contents")
991 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body))
993 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER.
994 with unittest.mock.patch.dict(
995 os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True
996 ):
997 importlib.reload(httpUri)
998 self.assertIsNone(self.redirectPathExpect._do_put(data=body))
1000 @responses.activate
1001 def testTransfer(self):
1003 self.assertIsNone(
1004 self.notExistingFileButlerURI.transfer_from(src=self.existingFileButlerURI)
1005 )
1006 self.assertIsNone(
1007 self.notExistingFileButlerURI.transfer_from(
1008 src=self.existingFileButlerURI, transfer="move"
1009 )
1010 )
1011 with self.assertRaises(FileExistsError):
1012 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI)
1013 with self.assertRaises(ValueError):
1014 self.notExistingFileButlerURI.transfer_from(
1015 src=self.existingFileButlerURI, transfer="unsupported"
1016 )
1018 def testParent(self):
1020 self.assertEqual(
1021 self.existingFolderButlerURI.geturl(),
1022 self.notExistingFileButlerURI.parent().geturl(),
1023 )
1024 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
1025 self.assertEqual(
1026 self.existingFileButlerURI.parent().geturl(),
1027 self.existingFileButlerURI.dirname().geturl(),
1028 )
1030 def test_send_expect_header(self):
1032 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from
1033 # the environment.
1034 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
1035 importlib.reload(httpUri)
1036 self.assertFalse(httpUri._SEND_EXPECT_HEADER_ON_PUT)
1038 with unittest.mock.patch.dict(
1039 os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True
1040 ):
1041 importlib.reload(httpUri)
1042 self.assertTrue(httpUri._SEND_EXPECT_HEADER_ON_PUT)
1044 def test_timeout(self):
1046 connect_timeout = 100
1047 read_timeout = 200
1048 with unittest.mock.patch.dict(
1049 os.environ,
1050 {
1051 "LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout),
1052 "LSST_HTTP_TIMEOUT_READ": str(read_timeout),
1053 },
1054 clear=True,
1055 ):
1056 # Force module reload to initialize TIMEOUT.
1057 importlib.reload(httpUri)
1058 self.assertEqual(
1059 httpUri.TIMEOUT,
1060 (connect_timeout, read_timeout),
1061 )
1063 def test_is_protected(self):
1065 _is_protected = httpUri._is_protected
1066 self.assertFalse(_is_protected("/this-file-does-not-exist"))
1068 with tempfile.NamedTemporaryFile(
1069 mode="wt", dir=self.tmpdir.ospath, delete=False
1070 ) as f:
1071 f.write("XXXX")
1072 file_path = f.name
1074 os.chmod(file_path, stat.S_IRUSR)
1075 self.assertTrue(_is_protected(file_path))
1077 for mode in (
1078 stat.S_IRGRP,
1079 stat.S_IWGRP,
1080 stat.S_IXGRP,
1081 stat.S_IROTH,
1082 stat.S_IWOTH,
1083 stat.S_IXOTH,
1084 ):
1085 os.chmod(file_path, stat.S_IRUSR | mode)
1086 self.assertFalse(_is_protected(file_path))
1089class WebdavUtilsTestCase(unittest.TestCase):
1090 """Test for the Webdav related utilities."""
1092 serverRoot = "www.lsstwithwebdav.orgx"
1093 wrongRoot = "www.lsstwithoutwebdav.org"
1095 def setUp(self):
1096 responses.add(
1097 responses.OPTIONS,
1098 f"https://{self.serverRoot}",
1099 status=200,
1100 headers={"DAV": "1,2,3"},
1101 )
1102 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200)
1104 @responses.activate
1105 def test_is_webdav_endpoint(self):
1107 _is_webdav_endpoint = httpUri._is_webdav_endpoint
1108 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}"))
1109 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}"))
1112class BearerTokenAuthTestCase(unittest.TestCase):
1113 """Test for the BearerTokenAuth class."""
1115 def setUp(self):
1116 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR))
1117 self.token = "ABCDE1234"
1119 def tearDown(self):
1120 if self.tmpdir and self.tmpdir.isLocal:
1121 removeTestTempDir(self.tmpdir.ospath)
1123 def test_empty_token(self):
1124 """Ensure that when no token is provided the request is not
1125 modified.
1126 """
1127 auth = httpUri.BearerTokenAuth(None)
1128 auth._refresh()
1129 self.assertIsNone(auth._token)
1130 self.assertIsNone(auth._path)
1131 req = requests.Request("GET", "https://example.org")
1132 self.assertEqual(auth(req), req)
1134 def test_token_value(self):
1135 """Ensure that when a token value is provided, the 'Authorization'
1136 header is added to the requests.
1137 """
1138 auth = httpUri.BearerTokenAuth(self.token)
1139 req = auth(requests.Request("GET", "https://example.org").prepare())
1140 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
1142 def test_token_file(self):
1143 """Ensure when the provided token is a file path, its contents is
1144 correctly used in the the 'Authorization' header of the requests.
1145 """
1146 with tempfile.NamedTemporaryFile(
1147 mode="wt", dir=self.tmpdir.ospath, delete=False
1148 ) as f:
1149 f.write(self.token)
1150 token_file_path = f.name
1152 # Ensure the request's "Authorization" header is set with the right
1153 # token value
1154 os.chmod(token_file_path, stat.S_IRUSR)
1155 auth = httpUri.BearerTokenAuth(token_file_path)
1156 req = auth(requests.Request("GET", "https://example.org").prepare())
1157 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
1159 # Ensure an exception is raised if either group or other can read the
1160 # token file
1161 for mode in (
1162 stat.S_IRGRP,
1163 stat.S_IWGRP,
1164 stat.S_IXGRP,
1165 stat.S_IROTH,
1166 stat.S_IWOTH,
1167 stat.S_IXOTH,
1168 ):
1169 os.chmod(token_file_path, stat.S_IRUSR | mode)
1170 with self.assertRaises(PermissionError):
1171 httpUri.BearerTokenAuth(token_file_path)
1174class SessionStoreTestCase(unittest.TestCase):
1175 """Test for the SessionStore class."""
1177 def setUp(self):
1178 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR))
1179 self.rpath = ButlerURI("https://example.org")
1181 def tearDown(self):
1182 if self.tmpdir and self.tmpdir.isLocal:
1183 removeTestTempDir(self.tmpdir.ospath)
1185 def test_ca_cert_bundle(self):
1186 """Ensure a certificate authorities bundle is used to authentify
1187 the remote server.
1188 """
1189 with tempfile.NamedTemporaryFile(
1190 mode="wt", dir=self.tmpdir.ospath, delete=False
1191 ) as f:
1192 f.write("CERT BUNDLE")
1193 cert_bundle = f.name
1195 with unittest.mock.patch.dict(
1196 os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True
1197 ):
1198 session = httpUri.SessionStore().get(self.rpath)
1199 self.assertEqual(session.verify, cert_bundle)
1201 def test_user_cert(self):
1202 """Ensure if user certificate and private key are provided, they are
1203 used for authenticating the client.
1204 """
1206 # Create mock certificate and private key files.
1207 with tempfile.NamedTemporaryFile(
1208 mode="wt", dir=self.tmpdir.ospath, delete=False
1209 ) as f:
1210 f.write("CERT")
1211 client_cert = f.name
1213 with tempfile.NamedTemporaryFile(
1214 mode="wt", dir=self.tmpdir.ospath, delete=False
1215 ) as f:
1216 f.write("KEY")
1217 client_key = f.name
1219 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY
1220 # must be initialized.
1221 with unittest.mock.patch.dict(
1222 os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True
1223 ):
1224 with self.assertRaises(ValueError):
1225 httpUri.SessionStore().get(self.rpath)
1227 with unittest.mock.patch.dict(
1228 os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True
1229 ):
1230 with self.assertRaises(ValueError):
1231 httpUri.SessionStore().get(self.rpath)
1233 # Check private key file must be accessible only by its owner.
1234 with unittest.mock.patch.dict(
1235 os.environ,
1236 {
1237 "LSST_HTTP_AUTH_CLIENT_CERT": client_cert,
1238 "LSST_HTTP_AUTH_CLIENT_KEY": client_key,
1239 },
1240 clear=True,
1241 ):
1242 # Ensure the session client certificate is initialized when
1243 # only the owner can read the private key file.
1244 os.chmod(client_key, stat.S_IRUSR)
1245 session = httpUri.SessionStore().get(self.rpath)
1246 self.assertEqual(session.cert[0], client_cert)
1247 self.assertEqual(session.cert[1], client_key)
1249 # Ensure an exception is raised if either group or other can access
1250 # the private key file.
1251 for mode in (
1252 stat.S_IRGRP,
1253 stat.S_IWGRP,
1254 stat.S_IXGRP,
1255 stat.S_IROTH,
1256 stat.S_IWOTH,
1257 stat.S_IXOTH,
1258 ):
1259 os.chmod(client_key, stat.S_IRUSR | mode)
1260 with self.assertRaises(PermissionError):
1261 httpUri.SessionStore().get(self.rpath)
1263 def test_token_env(self):
1264 """Ensure when the token is provided via an environment variable
1265 the sessions are equipped with a BearerTokenAuth.
1266 """
1267 token = "ABCDE"
1268 with unittest.mock.patch.dict(
1269 os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True
1270 ):
1271 session = httpUri.SessionStore().get(self.rpath)
1272 self.assertEqual(type(session.auth), httpUri.BearerTokenAuth)
1273 self.assertEqual(session.auth._token, token)
1274 self.assertIsNone(session.auth._path)
1276 def test_sessions(self):
1277 """Ensure the session caching mechanism works."""
1279 # Ensure the store provides a session for a given URL
1280 root_url = "https://example.org"
1281 store = httpUri.SessionStore()
1282 session = store.get(ButlerURI(root_url))
1283 self.assertIsNotNone(session)
1285 # Ensure the sessions retrieved from a single store with the same
1286 # root URIs are equal
1287 for u in (f"{root_url}", f"{root_url}/path/to/file"):
1288 self.assertEqual(session, store.get(ButlerURI(u)))
1290 # Ensure sessions retrieved for different root URIs are different
1291 another_url = "https://another.example.org"
1292 self.assertNotEqual(session, store.get(ButlerURI(another_url)))
1294 # Ensure the sessions retrieved from a single store for URLs with
1295 # different port numbers are different
1296 root_url_with_port = f"{another_url}:12345"
1297 session = store.get(ButlerURI(root_url_with_port))
1298 self.assertNotEqual(session, store.get(ButlerURI(another_url)))
1300 # Ensure the sessions retrieved from a single store with the same
1301 # root URIs (including port numbers) are equal
1302 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"):
1303 self.assertEqual(session, store.get(ButlerURI(u)))
1306if __name__ == "__main__": 1306 ↛ 1307line 1306 didn't jump to line 1307, because the condition on line 1306 was never true
1307 unittest.main()