Coverage for python/lsst/resources/_resourcePath.py: 21%
554 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-25 00:30 -0800
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = ("ResourcePath", "ResourcePathExpression")
16import concurrent.futures
17import contextlib
18import copy
19import io
20import locale
21import logging
22import os
23import posixpath
24import re
25import sys
26import urllib.parse
27from collections import defaultdict
28from pathlib import Path, PurePath, PurePosixPath
29from random import Random
30from typing import TypeAlias
32try:
33 import fsspec
34 from fsspec.spec import AbstractFileSystem
35except ImportError:
36 fsspec = None
37 AbstractFileSystem = type
39from collections.abc import Iterable, Iterator
40from typing import TYPE_CHECKING, Any, Literal, NamedTuple, overload
42from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
43from .utils import _get_num_workers, get_tempdir
45if TYPE_CHECKING:
46 from .utils import TransactionProtocol
49log = logging.getLogger(__name__)
51# Regex for looking for URI escapes
52ESCAPES_RE = re.compile(r"%[A-F0-9]{2}")
54# Precomputed escaped hash
55ESCAPED_HASH = urllib.parse.quote("#")
58class MBulkResult(NamedTuple):
59 """Report on a bulk operation."""
61 success: bool
62 exception: Exception | None
65_EXECUTOR_TYPE: TypeAlias = type[
66 concurrent.futures.ThreadPoolExecutor | concurrent.futures.ProcessPoolExecutor
67]
69# Cache value for executor class so as not to issue warning multiple
70# times but still allow tests to override the value.
71_POOL_EXECUTOR_CLASS: _EXECUTOR_TYPE | None = None
74def _get_executor_class() -> _EXECUTOR_TYPE:
75 """Return the executor class used for parallelized execution.
77 Returns
78 -------
79 cls : `concurrent.futures.Executor`
80 The ``Executor`` class. Default is
81 `concurrent.futures.ThreadPoolExecutor`. Can be set explicitly by
82 setting the ``$LSST_RESOURCES_EXECUTOR`` environment variable to
83 "thread" or "process". Returns "thread" pool if the value of the
84 variable is not recognized.
85 """
86 global _POOL_EXECUTOR_CLASS
88 if _POOL_EXECUTOR_CLASS is not None:
89 return _POOL_EXECUTOR_CLASS
91 pool_executor_classes = {
92 "threads": concurrent.futures.ThreadPoolExecutor,
93 "process": concurrent.futures.ProcessPoolExecutor,
94 }
95 default_executor = "threads"
96 external = os.getenv("LSST_RESOURCES_EXECUTOR", default_executor)
97 if not external:
98 external = default_executor
99 if external not in pool_executor_classes:
100 log.warning(
101 "Unrecognized value of '%s' for LSST_RESOURCES_EXECUTOR env var. Using '%s'",
102 external,
103 default_executor,
104 )
105 external = default_executor
106 _POOL_EXECUTOR_CLASS = pool_executor_classes[external]
107 return _POOL_EXECUTOR_CLASS
110@contextlib.contextmanager
111def _patch_environ(new_values: dict[str, str]) -> Iterator[None]:
112 """Patch os.environ temporarily using the supplied values.
114 Parameters
115 ----------
116 new_values : `dict` [ `str`, `str` ]
117 New values to be stored in the environment.
118 """
119 old_values: dict[str, str] = {}
120 for k, v in new_values.items():
121 if k in os.environ:
122 old_values[k] = os.environ[k]
123 os.environ[k] = v
125 try:
126 yield
127 finally:
128 for k in new_values:
129 del os.environ[k]
130 if k in old_values:
131 os.environ[k] = old_values[k]
134class ResourcePath: # numpydoc ignore=PR02
135 """Convenience wrapper around URI parsers.
137 Provides access to URI components and can convert file
138 paths into absolute path URIs. Scheme-less URIs are treated as if
139 they are local file system paths and are converted to absolute URIs.
141 A specialist subclass is created for each supported URI scheme.
143 Parameters
144 ----------
145 uri : `str`, `pathlib.Path`, `urllib.parse.ParseResult`, or `ResourcePath`
146 URI in string form. Can be scheme-less if referring to a relative
147 path or an absolute path on the local file system.
148 root : `str` or `ResourcePath`, optional
149 When fixing up a relative path in a ``file`` scheme or if scheme-less,
150 use this as the root. Must be absolute. If `None` the current
151 working directory will be used. Can be any supported URI scheme.
152 Not used if ``forceAbsolute`` is `False`.
153 forceAbsolute : `bool`, optional
154 If `True`, scheme-less relative URI will be converted to an absolute
155 path using a ``file`` scheme. If `False` scheme-less URI will remain
156 scheme-less and will not be updated to ``file`` or absolute path unless
157 it is already an absolute path, in which case it will be updated to
158 a ``file`` scheme.
159 forceDirectory : `bool` or `None`, optional
160 If `True` forces the URI to end with a separator. If `False` the URI
161 is interpreted as a file-like entity. Default, `None`, is that the
162 given URI is interpreted as a directory if there is a trailing ``/`` or
163 for some schemes the system will check to see if it is a file or a
164 directory.
165 isTemporary : `bool`, optional
166 If `True` indicates that this URI points to a temporary resource.
167 The default is `False`, unless ``uri`` is already a `ResourcePath`
168 instance and ``uri.isTemporary is True``.
170 Notes
171 -----
172 A non-standard URI of the form ``file:dir/file.txt`` is always converted
173 to an absolute ``file`` URI.
174 """
176 _pathLib: type[PurePath] = PurePosixPath
177 """Path library to use for this scheme."""
179 _pathModule = posixpath
180 """Path module to use for this scheme."""
182 transferModes: tuple[str, ...] = ("copy", "auto", "move")
183 """Transfer modes supported by this implementation.
185 Move is special in that it is generally a copy followed by an unlink.
186 Whether that unlink works depends critically on whether the source URI
187 implements unlink. If it does not the move will be reported as a failure.
188 """
190 transferDefault: str = "copy"
191 """Default mode to use for transferring if ``auto`` is specified."""
193 quotePaths = True
194 """True if path-like elements modifying a URI should be quoted.
196 All non-schemeless URIs have to internally use quoted paths. Therefore
197 if a new file name is given (e.g. to updatedFile or join) a decision must
198 be made whether to quote it to be consistent.
199 """
201 isLocal = False
202 """If `True` this URI refers to a local file."""
204 # This is not an ABC with abstract methods because the __new__ being
205 # a factory confuses mypy such that it assumes that every constructor
206 # returns a ResourcePath and then determines that all the abstract methods
207 # are still abstract. If they are not marked abstract but just raise
208 # mypy is fine with it.
210 # mypy is confused without these
211 _uri: urllib.parse.ParseResult
212 isTemporary: bool
213 dirLike: bool | None
214 """Whether the resource looks like a directory resource. `None` means that
215 the status is uncertain."""
217 def __new__(
218 cls,
219 uri: ResourcePathExpression,
220 root: str | ResourcePath | None = None,
221 forceAbsolute: bool = True,
222 forceDirectory: bool | None = None,
223 isTemporary: bool | None = None,
224 ) -> ResourcePath:
225 """Create and return new specialist ResourcePath subclass."""
226 parsed: urllib.parse.ParseResult
227 dirLike: bool | None = forceDirectory
228 subclass: type[ResourcePath] | None = None
230 # Force root to be a ResourcePath -- this simplifies downstream
231 # code.
232 if root is None:
233 root_uri = None
234 elif isinstance(root, str):
235 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True)
236 else:
237 root_uri = root
239 if isinstance(uri, os.PathLike):
240 uri = str(uri)
242 # Record if we need to post process the URI components
243 # or if the instance is already fully configured
244 if isinstance(uri, str):
245 # Since local file names can have special characters in them
246 # we need to quote them for the parser but we can unquote
247 # later. Assume that all other URI schemes are quoted.
248 # Since sometimes people write file:/a/b and not file:///a/b
249 # we should not quote in the explicit case of file:
250 if "://" not in uri and not uri.startswith("file:"):
251 if ESCAPES_RE.search(uri):
252 log.warning("Possible double encoding of %s", uri)
253 else:
254 # Fragments are generally not encoded so we must search
255 # for the fragment boundary ourselves. This is making
256 # an assumption that the filename does not include a "#"
257 # and also that there is no "/" in the fragment itself.
258 to_encode = uri
259 fragment = ""
260 if "#" in uri:
261 dirpos = uri.rfind("/")
262 trailing = uri[dirpos + 1 :]
263 hashpos = trailing.rfind("#")
264 if hashpos != -1:
265 fragment = trailing[hashpos:]
266 to_encode = uri[: dirpos + hashpos + 1]
268 uri = urllib.parse.quote(to_encode) + fragment
270 parsed = urllib.parse.urlparse(uri)
271 elif isinstance(uri, urllib.parse.ParseResult):
272 parsed = copy.copy(uri)
273 # If we are being instantiated with a subclass, rather than
274 # ResourcePath, ensure that that subclass is used directly.
275 # This could lead to inconsistencies if this constructor
276 # is used externally outside of the ResourcePath.replace() method.
277 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt"))
278 # will be a problem.
279 # This is needed to prevent a schemeless absolute URI become
280 # a file URI unexpectedly when calling updatedFile or
281 # updatedExtension
282 if cls is not ResourcePath:
283 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory)
284 subclass = cls
286 elif isinstance(uri, ResourcePath):
287 # Since ResourcePath is immutable we can return the argument
288 # unchanged if it already agrees with forceDirectory, isTemporary,
289 # and forceAbsolute.
290 # We invoke __new__ again with str(self) to add a scheme for
291 # forceAbsolute, but for the others that seems more likely to paper
292 # over logic errors than do something useful, so we just raise.
293 if forceDirectory is not None and uri.dirLike is not None and forceDirectory is not uri.dirLike:
294 # Can not force a file-like URI to become a dir-like one or
295 # vice versa.
296 raise RuntimeError(
297 f"{uri} can not be forced to change directory vs file state when previously declared."
298 )
299 if isTemporary is not None and isTemporary is not uri.isTemporary:
300 raise RuntimeError(
301 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} "
302 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}."
303 )
305 if forceAbsolute and not uri.scheme:
306 # Create new absolute from relative.
307 return ResourcePath(
308 str(uri),
309 root=root,
310 forceAbsolute=forceAbsolute,
311 forceDirectory=forceDirectory or uri.dirLike,
312 isTemporary=uri.isTemporary,
313 )
314 elif forceDirectory is not None and uri.dirLike is None:
315 # Clone but with a new dirLike status.
316 return uri.replace(forceDirectory=forceDirectory)
317 return uri
318 else:
319 raise ValueError(
320 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'"
321 )
323 if subclass is None:
324 # Work out the subclass from the URI scheme
325 if not parsed.scheme:
326 # Root may be specified as a ResourcePath that overrides
327 # the schemeless determination.
328 if (
329 root_uri is not None
330 and root_uri.scheme != "file" # file scheme has different code path
331 and not parsed.path.startswith("/") # Not already absolute path
332 ):
333 if root_uri.dirLike is False:
334 raise ValueError(
335 f"Root URI ({root}) was not a directory so can not be joined with"
336 f" path {parsed.path!r}"
337 )
338 # If root is temporary or this schemeless is temporary we
339 # assume this URI is temporary.
340 isTemporary = isTemporary or root_uri.isTemporary
341 joined = root_uri.join(
342 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary
343 )
345 # Rather than returning this new ResourcePath directly we
346 # instead extract the path and the scheme and adjust the
347 # URI we were given -- we need to do this to preserve
348 # fragments since join() will drop them.
349 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc)
350 subclass = type(joined)
352 # Clear the root parameter to indicate that it has
353 # been applied already.
354 root_uri = None
355 else:
356 from .schemeless import SchemelessResourcePath
358 subclass = SchemelessResourcePath
359 elif parsed.scheme == "file":
360 from .file import FileResourcePath
362 subclass = FileResourcePath
363 elif parsed.scheme == "s3":
364 from .s3 import S3ResourcePath
366 subclass = S3ResourcePath
367 elif parsed.scheme.startswith("http"):
368 from .http import HttpResourcePath
370 subclass = HttpResourcePath
371 elif parsed.scheme in {"dav", "davs"}:
372 from .dav import DavResourcePath
374 subclass = DavResourcePath
375 elif parsed.scheme == "gs":
376 from .gs import GSResourcePath
378 subclass = GSResourcePath
379 elif parsed.scheme == "resource":
380 # Rules for scheme names disallow pkg_resource
381 from .packageresource import PackageResourcePath
383 subclass = PackageResourcePath
384 elif parsed.scheme == "mem":
385 # in-memory datastore object
386 from .mem import InMemoryResourcePath
388 subclass = InMemoryResourcePath
389 else:
390 raise NotImplementedError(
391 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}"
392 )
394 parsed, dirLike = subclass._fixupPathUri(
395 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory
396 )
398 # It is possible for the class to change from schemeless
399 # to file so handle that
400 if parsed.scheme == "file":
401 from .file import FileResourcePath
403 subclass = FileResourcePath
405 # Now create an instance of the correct subclass and set the
406 # attributes directly
407 self = object.__new__(subclass)
408 self._uri = parsed
409 self.dirLike = dirLike
410 if isTemporary is None:
411 isTemporary = False
412 self.isTemporary = isTemporary
413 return self
415 @property
416 def scheme(self) -> str:
417 """Return the URI scheme.
419 Notes
420 -----
421 (``://`` is not part of the scheme).
422 """
423 return self._uri.scheme
425 @property
426 def netloc(self) -> str:
427 """Return the URI network location."""
428 return self._uri.netloc
430 @property
431 def path(self) -> str:
432 """Return the path component of the URI."""
433 return self._uri.path
435 @property
436 def unquoted_path(self) -> str:
437 """Return path component of the URI with any URI quoting reversed."""
438 return urllib.parse.unquote(self._uri.path)
440 @property
441 def ospath(self) -> str:
442 """Return the path component of the URI localized to current OS."""
443 raise AttributeError(f"Non-file URI ({self}) has no local OS path.")
445 @property
446 def relativeToPathRoot(self) -> str:
447 """Return path relative to network location.
449 This is the path property with posix separator stripped
450 from the left hand side of the path.
452 Always unquotes.
453 """
454 relToRoot = self.path.lstrip("/")
455 if relToRoot == "":
456 return "./"
457 return urllib.parse.unquote(relToRoot)
459 @property
460 def is_root(self) -> bool:
461 """Return whether this URI points to the root of the network location.
463 This means that the path components refers to the top level.
464 """
465 relpath = self.relativeToPathRoot
466 if relpath == "./":
467 return True
468 return False
470 @property
471 def fragment(self) -> str:
472 """Return the fragment component of the URI. May be quoted."""
473 return self._uri.fragment
475 @property
476 def unquoted_fragment(self) -> str:
477 """Return unquoted fragment."""
478 return urllib.parse.unquote(self.fragment)
480 @property
481 def params(self) -> str:
482 """Return any parameters included in the URI."""
483 return self._uri.params
485 @property
486 def query(self) -> str:
487 """Return any query strings included in the URI."""
488 return self._uri.query
490 def geturl(self) -> str:
491 """Return the URI in string form.
493 Returns
494 -------
495 url : `str`
496 String form of URI.
497 """
498 return self._uri.geturl()
500 def to_fsspec(self) -> tuple[AbstractFileSystem, str]:
501 """Return an abstract file system and path that can be used by fsspec.
503 Returns
504 -------
505 fs : `fsspec.spec.AbstractFileSystem`
506 A file system object suitable for use with the returned path.
507 path : `str`
508 A path that can be opened by the file system object.
509 """
510 if fsspec is None:
511 raise ImportError("fsspec is not available")
512 # By default give the URL to fsspec and hope.
513 return fsspec.url_to_fs(self.geturl())
515 def root_uri(self) -> ResourcePath:
516 """Return the base root URI.
518 Returns
519 -------
520 uri : `ResourcePath`
521 Root URI.
522 """
523 return self.replace(path="", query="", fragment="", params="", forceDirectory=True)
525 def split(self) -> tuple[ResourcePath, str]:
526 """Split URI into head and tail.
528 Returns
529 -------
530 head: `ResourcePath`
531 Everything leading up to tail, expanded and normalized as per
532 ResourcePath rules.
533 tail : `str`
534 Last path component. Tail will be empty if path ends on a
535 separator or if the URI is known to be associated with a directory.
536 Tail will never contain separators. It will be unquoted.
538 Notes
539 -----
540 Equivalent to `os.path.split` where head preserves the URI
541 components. In some cases this method can result in a file system
542 check to verify whether the URI is a directory or not (only if
543 ``forceDirectory`` was `None` during construction). For a scheme-less
544 URI this can mean that the result might change depending on current
545 working directory.
546 """
547 if self.isdir():
548 # This is known to be a directory so must return itself and
549 # the empty string.
550 return self, ""
552 head, tail = self._pathModule.split(self.path)
553 headuri = self._uri._replace(path=head, fragment="", query="", params="")
555 # The file part should never include quoted metacharacters
556 tail = urllib.parse.unquote(tail)
558 # Schemeless is special in that it can be a relative path.
559 # We need to ensure that it stays that way. All other URIs will
560 # be absolute already.
561 forceAbsolute = self.isabs()
562 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail
564 def basename(self) -> str:
565 """Return the base name, last element of path, of the URI.
567 Returns
568 -------
569 tail : `str`
570 Last part of the path attribute. Trail will be empty if path ends
571 on a separator.
573 Notes
574 -----
575 If URI ends on a slash returns an empty string. This is the second
576 element returned by `split()`.
578 Equivalent of `os.path.basename`.
579 """
580 return self.split()[1]
582 def dirname(self) -> ResourcePath:
583 """Return the directory component of the path as a new `ResourcePath`.
585 Returns
586 -------
587 head : `ResourcePath`
588 Everything except the tail of path attribute, expanded and
589 normalized as per ResourcePath rules.
591 Notes
592 -----
593 Equivalent of `os.path.dirname`. If this is a directory URI it will
594 be returned unchanged. If the parent directory is always required
595 use `parent`.
596 """
597 return self.split()[0]
599 def parent(self) -> ResourcePath:
600 """Return a `ResourcePath` of the parent directory.
602 Returns
603 -------
604 head : `ResourcePath`
605 Everything except the tail of path attribute, expanded and
606 normalized as per `ResourcePath` rules.
608 Notes
609 -----
610 For a file-like URI this will be the same as calling `dirname`.
611 For a directory-like URI this will always return the parent directory
612 whereas `dirname()` will return the original URI. This is consistent
613 with `os.path.dirname` compared to the `pathlib.Path` property
614 ``parent``.
615 """
616 if self.dirLike is False:
617 # os.path.split() is slightly faster than calling Path().parent.
618 return self.dirname()
619 # When self is dir-like, returns its parent directory,
620 # regardless of the presence of a trailing separator
621 originalPath = self._pathLib(self.path)
622 parentPath = originalPath.parent
623 return self.replace(path=str(parentPath), forceDirectory=True, fragment="", query="", params="")
625 def replace(
626 self, forceDirectory: bool | None = None, isTemporary: bool = False, **kwargs: Any
627 ) -> ResourcePath:
628 """Return new `ResourcePath` with specified components replaced.
630 Parameters
631 ----------
632 forceDirectory : `bool` or `None`, optional
633 Parameter passed to ResourcePath constructor to force this
634 new URI to be dir-like or file-like.
635 isTemporary : `bool`, optional
636 Indicate that the resulting URI is temporary resource.
637 **kwargs
638 Components of a `urllib.parse.ParseResult` that should be
639 modified for the newly-created `ResourcePath`.
641 Returns
642 -------
643 new : `ResourcePath`
644 New `ResourcePath` object with updated values.
646 Notes
647 -----
648 Does not, for now, allow a change in URI scheme.
649 """
650 # Disallow a change in scheme
651 if "scheme" in kwargs:
652 raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
653 result = self.__class__(
654 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
655 )
656 result._copy_extra_attributes(self)
657 return result
659 def updatedFile(self, newfile: str) -> ResourcePath:
660 """Return new URI with an updated final component of the path.
662 Parameters
663 ----------
664 newfile : `str`
665 File name with no path component.
667 Returns
668 -------
669 updated : `ResourcePath`
670 Updated `ResourcePath` with new updated final component.
672 Notes
673 -----
674 Forces the ``ResourcePath.dirLike`` attribute to be false. The new file
675 path will be quoted if necessary. If the current URI is known to
676 refer to a directory, the new file will be joined to the current file.
677 It is recommended that this behavior no longer be used and a call
678 to `isdir` by the caller should be used to decide whether to join or
679 replace. In the future this method may be modified to always replace
680 the final element of the path.
681 """
682 if self.dirLike:
683 return self.join(newfile, forceDirectory=False)
684 return self.parent().join(newfile, forceDirectory=False)
686 def updatedExtension(self, ext: str | None) -> ResourcePath:
687 """Return a new `ResourcePath` with updated file extension.
689 All file extensions are replaced.
691 Parameters
692 ----------
693 ext : `str` or `None`
694 New extension. If an empty string is given any extension will
695 be removed. If `None` is given there will be no change.
697 Returns
698 -------
699 updated : `ResourcePath`
700 URI with the specified extension. Can return itself if
701 no extension was specified.
702 """
703 if ext is None:
704 return self
706 # Get the extension
707 current = self.getExtension()
709 # Nothing to do if the extension already matches
710 if current == ext:
711 return self
713 # Remove the current extension from the path
714 # .fits.gz counts as one extension do not use os.path.splitext
715 path = self.path
716 if current:
717 path = path.removesuffix(current)
719 # Ensure that we have a leading "." on file extension (and we do not
720 # try to modify the empty string)
721 if ext and not ext.startswith("."):
722 ext = "." + ext
724 return self.replace(path=path + ext, forceDirectory=False)
726 def getExtension(self) -> str:
727 """Return the extension(s) associated with this URI path.
729 Returns
730 -------
731 ext : `str`
732 The file extension (including the ``.``). Can be empty string
733 if there is no file extension. Usually returns only the last
734 file extension unless there is a special extension modifier
735 indicating file compression, in which case the combined
736 extension (e.g. ``.fits.gz``) will be returned.
738 Notes
739 -----
740 Does not distinguish between file and directory URIs when determining
741 a suffix. An extension is only determined from the final component
742 of the path.
743 """
744 special = {".gz", ".bz2", ".xz", ".fz"}
746 # path lib will ignore any "." in directories.
747 # path lib works well:
748 # extensions = self._pathLib(self.path).suffixes
749 # But the constructor is slow. Therefore write our own implementation.
750 # Strip trailing separator if present, do not care if this is a
751 # directory or not.
752 parts = self.path.rstrip("/").rsplit(self._pathModule.sep, 1)
753 _, *extensions = parts[-1].split(".")
755 if not extensions:
756 return ""
757 extensions = ["." + x for x in extensions]
759 ext = extensions.pop()
761 # Multiple extensions, decide whether to include the final two
762 if extensions and ext in special:
763 ext = f"{extensions[-1]}{ext}"
765 return ext
767 def join(
768 self, path: str | ResourcePath, isTemporary: bool | None = None, forceDirectory: bool | None = None
769 ) -> ResourcePath:
770 """Return new `ResourcePath` with additional path components.
772 Parameters
773 ----------
774 path : `str`, `ResourcePath`
775 Additional file components to append to the current URI. Will be
776 quoted depending on the associated URI scheme. If the path looks
777 like a URI referring to an absolute location, it will be returned
778 directly (matching the behavior of `os.path.join`). It can
779 also be a `ResourcePath`. Fragments are propagated.
780 isTemporary : `bool`, optional
781 Indicate that the resulting URI represents a temporary resource.
782 Default is ``self.isTemporary``.
783 forceDirectory : `bool` or `None`, optional
784 If `True` forces the URI to end with a separator. If `False` the
785 resultant URI is declared to refer to a file. `None` indicates
786 that the file directory status is unknown.
788 Returns
789 -------
790 new : `ResourcePath`
791 New URI with the path appended.
793 Notes
794 -----
795 Schemeless URIs assume local path separator but all other URIs assume
796 POSIX separator if the supplied path has directory structure. It
797 may be this never becomes a problem but datastore templates assume
798 POSIX separator is being used.
800 If an absolute `ResourcePath` is given for ``path`` is is assumed that
801 this should be returned directly. Giving a ``path`` of an absolute
802 scheme-less URI is not allowed for safety reasons as it may indicate
803 a mistake in the calling code.
805 It is an error to attempt to join to something that is known to
806 refer to a file. Use `updatedFile` if the file is to be
807 replaced.
809 If an unquoted ``#`` is included in the path it is assumed to be
810 referring to a fragment and not part of the file name.
812 Raises
813 ------
814 ValueError
815 Raised if the given path object refers to a directory but the
816 ``forceDirectory`` parameter insists the outcome should be a file,
817 and vice versa. Also raised if the URI being joined with is known
818 to refer to a file.
819 RuntimeError
820 Raised if this attempts to join a temporary URI to a non-temporary
821 URI.
822 """
823 if self.dirLike is False:
824 raise ValueError("Can not join a new path component to a file.")
825 if isTemporary is None:
826 isTemporary = self.isTemporary
827 elif not isTemporary and self.isTemporary:
828 raise RuntimeError("Cannot join temporary URI to non-temporary URI.")
829 # If we have a full URI in path we will use it directly
830 # but without forcing to absolute so that we can trap the
831 # expected option of relative path.
832 path_uri = ResourcePath(
833 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary
834 )
835 if forceDirectory is not None and path_uri.dirLike is not forceDirectory:
836 raise ValueError(
837 "The supplied path URI to join has inconsistent directory state "
838 f"with forceDirectory parameter: {path_uri.dirLike} vs {forceDirectory}"
839 )
840 forceDirectory = path_uri.dirLike
842 if path_uri.isabs():
843 # Absolute URI so return it directly.
844 return path_uri
846 # We want to propagate fragments to the joined path and we rely on
847 # the ResourcePath parser to find these fragments for us even in plain
848 # strings. Must assume there are no `#` characters in filenames.
849 if not isinstance(path, str) or path_uri.fragment:
850 path = path_uri.unquoted_path
852 # Might need to quote the path.
853 if self.quotePaths:
854 path = urllib.parse.quote(path)
856 newpath = self._pathModule.normpath(self._pathModule.join(self.path, path))
858 # normpath can strip trailing / so we force directory if the supplied
859 # path ended with a /
860 has_dir_sep = path.endswith(self._pathModule.sep)
861 if forceDirectory is None and has_dir_sep:
862 forceDirectory = True
863 elif forceDirectory is False and has_dir_sep:
864 raise ValueError("Path to join has trailing / but is being forced to be a file.")
865 return self.replace(
866 path=newpath,
867 forceDirectory=forceDirectory,
868 isTemporary=isTemporary,
869 fragment=path_uri.fragment,
870 query=path_uri.query,
871 params=path_uri.params,
872 )
874 def relative_to(self, other: ResourcePath, walk_up: bool = False) -> str | None:
875 """Return the relative path from this URI to the other URI.
877 Parameters
878 ----------
879 other : `ResourcePath`
880 URI to use to calculate the relative path. Must be a parent
881 of this URI.
882 walk_up : `bool`, optional
883 Control whether "``..``" can be used to resolve a relative path.
884 Default is `False`. Can not be `True` on Python version 3.11.
886 Returns
887 -------
888 subpath : `str`
889 The sub path of this URI relative to the supplied other URI.
890 Returns `None` if there is no parent child relationship.
891 Scheme and netloc must match.
892 """
893 # Scheme-less self is handled elsewhere.
894 if self.scheme != other.scheme:
895 return None
896 if self.netloc != other.netloc:
897 # Special case for localhost vs empty string.
898 # There can be many variants of localhost.
899 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"}
900 if not {self.netloc, other.netloc}.issubset(local_netlocs):
901 return None
903 # Rather than trying to guess a failure reason from the TypeError
904 # explicitly check for python 3.11. Doing this will simplify the
905 # rediscovery of a useless python version check when we set a new
906 # minimum version.
907 kwargs = {}
908 if walk_up:
909 if sys.version_info < (3, 12, 0):
910 raise TypeError("walk_up parameter can not be true in python 3.11 and older")
912 kwargs["walk_up"] = True
914 enclosed_path = self._pathLib(self.relativeToPathRoot)
915 parent_path = other.relativeToPathRoot
916 subpath: str | None
917 try:
918 subpath = str(enclosed_path.relative_to(parent_path, **kwargs))
919 except ValueError:
920 subpath = None
921 else:
922 subpath = urllib.parse.unquote(subpath)
923 return subpath
925 def exists(self) -> bool:
926 """Indicate that the resource is available.
928 Returns
929 -------
930 exists : `bool`
931 `True` if the resource exists.
932 """
933 raise NotImplementedError()
935 @classmethod
936 def _group_uris(cls, uris: Iterable[ResourcePath]) -> dict[type[ResourcePath], list[ResourcePath]]:
937 """Group URIs by class/scheme."""
938 grouped: dict[type, list[ResourcePath]] = defaultdict(list)
939 for uri in uris:
940 grouped[uri.__class__].append(uri)
941 return grouped
943 @classmethod
944 def mexists(
945 cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
946 ) -> dict[ResourcePath, bool]:
947 """Check for existence of multiple URIs at once.
949 Parameters
950 ----------
951 uris : iterable of `ResourcePath`
952 The URIs to test.
953 num_workers : `int` or `None`, optional
954 The number of parallel workers to use when checking for existence
955 If `None`, the default value will be taken from the environment.
956 If this number is higher than the default and a thread pool is
957 used, there may not be enough cached connections available.
959 Returns
960 -------
961 existence : `dict` of [`ResourcePath`, `bool`]
962 Mapping of original URI to boolean indicating existence.
963 """
964 existence: dict[ResourcePath, bool] = {}
965 for uri_class, group in cls._group_uris(uris).items():
966 existence.update(uri_class._mexists(group, num_workers=num_workers))
968 return existence
970 @classmethod
971 def _mexists(
972 cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
973 ) -> dict[ResourcePath, bool]:
974 """Check for existence of multiple URIs at once.
976 Implementation helper method for `mexists`.
979 Parameters
980 ----------
981 uris : iterable of `ResourcePath`
982 The URIs to test.
983 num_workers : `int` or `None`, optional
984 The number of parallel workers to use when checking for existence
985 If `None`, the default value will be taken from the environment.
987 Returns
988 -------
989 existence : `dict` of [`ResourcePath`, `bool`]
990 Mapping of original URI to boolean indicating existence.
991 """
992 pool_executor_class = _get_executor_class()
993 if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
994 # Patch the environment to make it think there is only one worker
995 # for each subprocess.
996 with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
997 return cls._mexists_pool(pool_executor_class, uris)
998 else:
999 return cls._mexists_pool(pool_executor_class, uris, num_workers=num_workers)
1001 @classmethod
1002 def _mexists_pool(
1003 cls,
1004 pool_executor_class: _EXECUTOR_TYPE,
1005 uris: Iterable[ResourcePath],
1006 *,
1007 num_workers: int | None = None,
1008 ) -> dict[ResourcePath, bool]:
1009 """Check for existence of multiple URIs at once using specified pool
1010 executor.
1012 Implementation helper method for `_mexists`.
1014 Parameters
1015 ----------
1016 pool_executor_class : `type` [ `concurrent.futures.Executor` ]
1017 Type of executor pool to use.
1018 uris : iterable of `ResourcePath`
1019 The URIs to test.
1020 num_workers : `int` or `None`, optional
1021 The number of parallel workers to use when checking for existence
1022 If `None`, the default value will be taken from the environment.
1024 Returns
1025 -------
1026 existence : `dict` of [`ResourcePath`, `bool`]
1027 Mapping of original URI to boolean indicating existence.
1028 """
1029 max_workers = num_workers if num_workers is not None else _get_num_workers()
1030 with pool_executor_class(max_workers=max_workers) as exists_executor:
1031 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris}
1033 results: dict[ResourcePath, bool] = {}
1034 for future in concurrent.futures.as_completed(future_exists):
1035 uri = future_exists[future]
1036 try:
1037 exists = future.result()
1038 except Exception:
1039 exists = False
1040 results[uri] = exists
1041 return results
1043 @classmethod
1044 def mtransfer(
1045 cls,
1046 transfer: str,
1047 from_to: Iterable[tuple[ResourcePath, ResourcePath]],
1048 overwrite: bool = False,
1049 transaction: TransactionProtocol | None = None,
1050 do_raise: bool = True,
1051 ) -> dict[ResourcePath, MBulkResult]:
1052 """Transfer many files in bulk.
1054 Parameters
1055 ----------
1056 transfer : `str`
1057 Mode to use for transferring the resource. Generically there are
1058 many standard options: copy, link, symlink, hardlink, relsymlink.
1059 Not all URIs support all modes.
1060 from_to : `list` [ `tuple` [ `ResourcePath`, `ResourcePath` ] ]
1061 A sequence of the source URIs and the target URIs.
1062 overwrite : `bool`, optional
1063 Allow an existing file to be overwritten. Defaults to `False`.
1064 transaction : `~lsst.resources.utils.TransactionProtocol`, optional
1065 A transaction object that can (depending on implementation)
1066 rollback transfers on error. Not guaranteed to be implemented.
1067 The transaction object must be thread safe.
1068 do_raise : `bool`, optional
1069 If `True` an `ExceptionGroup` will be raised containing any
1070 exceptions raised by the individual transfers. If `False`, or if
1071 there were no exceptions, a dict reporting the status of each
1072 `ResourcePath` will be returned.
1074 Returns
1075 -------
1076 copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
1077 A dict of all the transfer attempts with a value indicating
1078 whether the transfer succeeded for the target URI. If ``do_raise``
1079 is `True`, this will only be returned if there are no errors.
1080 """
1081 pool_executor_class = _get_executor_class()
1082 if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
1083 # Patch the environment to make it think there is only one worker
1084 # for each subprocess.
1085 with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
1086 return cls._mtransfer(
1087 pool_executor_class,
1088 transfer,
1089 from_to,
1090 overwrite=overwrite,
1091 transaction=transaction,
1092 do_raise=do_raise,
1093 )
1094 return cls._mtransfer(
1095 pool_executor_class,
1096 transfer,
1097 from_to,
1098 overwrite=overwrite,
1099 transaction=transaction,
1100 do_raise=do_raise,
1101 )
1103 @classmethod
1104 def _mtransfer(
1105 cls,
1106 pool_executor_class: _EXECUTOR_TYPE,
1107 transfer: str,
1108 from_to: Iterable[tuple[ResourcePath, ResourcePath]],
1109 overwrite: bool = False,
1110 transaction: TransactionProtocol | None = None,
1111 do_raise: bool = True,
1112 ) -> dict[ResourcePath, MBulkResult]:
1113 """Transfer many files in bulk.
1115 Parameters
1116 ----------
1117 transfer : `str`
1118 Mode to use for transferring the resource. Generically there are
1119 many standard options: copy, link, symlink, hardlink, relsymlink.
1120 Not all URIs support all modes.
1121 from_to : `list` [ `tuple` [ `ResourcePath`, `ResourcePath` ] ]
1122 A sequence of the source URIs and the target URIs.
1123 overwrite : `bool`, optional
1124 Allow an existing file to be overwritten. Defaults to `False`.
1125 transaction : `~lsst.resources.utils.TransactionProtocol`, optional
1126 A transaction object that can (depending on implementation)
1127 rollback transfers on error. Not guaranteed to be implemented.
1128 The transaction object must be thread safe.
1129 do_raise : `bool`, optional
1130 If `True` an `ExceptionGroup` will be raised containing any
1131 exceptions raised by the individual transfers. Else a dict
1132 reporting the status of each `ResourcePath` will be returned.
1134 Returns
1135 -------
1136 copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
1137 A dict of all the transfer attempts with a value indicating
1138 whether the transfer succeeded for the target URI.
1139 """
1140 with pool_executor_class(max_workers=_get_num_workers()) as transfer_executor:
1141 future_transfers = {
1142 transfer_executor.submit(
1143 to_uri.transfer_from,
1144 from_uri,
1145 transfer=transfer,
1146 overwrite=overwrite,
1147 transaction=transaction,
1148 multithreaded=False,
1149 ): to_uri
1150 for from_uri, to_uri in from_to
1151 }
1152 results: dict[ResourcePath, MBulkResult] = {}
1153 failed = False
1154 for future in concurrent.futures.as_completed(future_transfers):
1155 to_uri = future_transfers[future]
1156 try:
1157 future.result()
1158 except Exception as e:
1159 transferred = MBulkResult(False, e)
1160 failed = True
1161 else:
1162 transferred = MBulkResult(True, None)
1163 results[to_uri] = transferred
1165 if do_raise and failed:
1166 raise ExceptionGroup(
1167 f"Errors transferring {len(results)} artifacts",
1168 tuple(res.exception for res in results.values() if res.exception is not None),
1169 )
1171 return results
1173 def remove(self) -> None:
1174 """Remove the resource."""
1175 raise NotImplementedError()
1177 @classmethod
1178 def mremove(
1179 cls, uris: Iterable[ResourcePath], *, do_raise: bool = True
1180 ) -> dict[ResourcePath, MBulkResult]:
1181 """Remove multiple URIs at once.
1183 Parameters
1184 ----------
1185 uris : iterable of `ResourcePath`
1186 URIs to remove.
1187 do_raise : `bool`, optional
1188 If `True` an `ExceptionGroup` will be raised containing any
1189 exceptions raised by the individual transfers. If `False`, or if
1190 there were no exceptions, a dict reporting the status of each
1191 `ResourcePath` will be returned.
1193 Returns
1194 -------
1195 results : `dict` [ `ResourcePath`, `MBulkResult` ]
1196 Dictionary mapping each URI to a result object indicating whether
1197 the removal succeeded or resulted in an exception. If ``do_raise``
1198 is `True` this will only be returned if everything succeeded.
1199 """
1200 # Group URIs by scheme since some URI schemes support native bulk
1201 # APIs.
1202 results: dict[ResourcePath, MBulkResult] = {}
1203 for uri_class, group in cls._group_uris(uris).items():
1204 results.update(uri_class._mremove(group))
1205 if do_raise:
1206 failed = any(not r.success for r in results.values())
1207 if failed:
1208 s = "s" if len(results) != 1 else ""
1209 raise ExceptionGroup(
1210 f"Error{s} removing {len(results)} artifact{s}",
1211 tuple(res.exception for res in results.values() if res.exception is not None),
1212 )
1214 return results
1216 @classmethod
1217 def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
1218 """Remove multiple URIs using futures."""
1219 pool_executor_class = _get_executor_class()
1220 if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
1221 # Patch the environment to make it think there is only one worker
1222 # for each subprocess.
1223 with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
1224 return cls._mremove_pool(pool_executor_class, uris)
1225 else:
1226 return cls._mremove_pool(pool_executor_class, uris)
1228 @classmethod
1229 def _mremove_pool(
1230 cls,
1231 pool_executor_class: _EXECUTOR_TYPE,
1232 uris: Iterable[ResourcePath],
1233 *,
1234 num_workers: int | None = None,
1235 ) -> dict[ResourcePath, MBulkResult]:
1236 """Remove URIs using a futures pool."""
1237 max_workers = num_workers if num_workers is not None else _get_num_workers()
1238 results: dict[ResourcePath, MBulkResult] = {}
1239 with pool_executor_class(max_workers=max_workers) as remove_executor:
1240 future_remove = {remove_executor.submit(uri.remove): uri for uri in uris}
1241 for future in concurrent.futures.as_completed(future_remove):
1242 try:
1243 future.result()
1244 except Exception as e:
1245 removed = MBulkResult(False, e)
1246 else:
1247 removed = MBulkResult(True, None)
1248 uri = future_remove[future]
1249 results[uri] = removed
1250 return results
1252 def isabs(self) -> bool:
1253 """Indicate that the resource is fully specified.
1255 For non-schemeless URIs this is always true.
1257 Returns
1258 -------
1259 isabs : `bool`
1260 `True` in all cases except schemeless URI.
1261 """
1262 return True
1264 def abspath(self) -> ResourcePath:
1265 """Return URI using an absolute path.
1267 Returns
1268 -------
1269 abs : `ResourcePath`
1270 Absolute URI. For non-schemeless URIs this always returns itself.
1271 Schemeless URIs are upgraded to file URIs.
1272 """
1273 return self
1275 @contextlib.contextmanager
1276 def _as_local(
1277 self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
1278 ) -> Iterator[ResourcePath]:
1279 """Return the location of the (possibly remote) resource as local file.
1281 This is a helper function for `as_local` context manager.
1283 Parameters
1284 ----------
1285 multithreaded : `bool`, optional
1286 If `True` the transfer will be allowed to attempt to improve
1287 throughput by using parallel download streams. This may of no
1288 effect if the URI scheme does not support parallel streams or
1289 if a global override has been applied. If `False` parallel
1290 streams will be disabled.
1291 tmpdir : `ResourcePath` or `None`, optional
1292 Explicit override of the temporary directory to use for remote
1293 downloads.
1295 Returns
1296 -------
1297 local_uri : `ResourcePath`
1298 A URI to a local POSIX file. This can either be the same resource
1299 or a local downloaded copy of the resource.
1300 """
1301 raise NotImplementedError()
1303 @contextlib.contextmanager
1304 def as_local(
1305 self, multithreaded: bool = True, tmpdir: ResourcePathExpression | None = None
1306 ) -> Iterator[ResourcePath]:
1307 """Return the location of the (possibly remote) resource as local file.
1309 Parameters
1310 ----------
1311 multithreaded : `bool`, optional
1312 If `True` the transfer will be allowed to attempt to improve
1313 throughput by using parallel download streams. This may of no
1314 effect if the URI scheme does not support parallel streams or
1315 if a global override has been applied. If `False` parallel
1316 streams will be disabled.
1317 tmpdir : `lsst.resources.ResourcePathExpression` or `None`, optional
1318 Explicit override of the temporary directory to use for remote
1319 downloads. This directory must be a local POSIX directory and
1320 must exist.
1322 Yields
1323 ------
1324 local : `ResourcePath`
1325 If this is a remote resource, it will be a copy of the resource
1326 on the local file system, probably in a temporary directory.
1327 For a local resource this should be the actual path to the
1328 resource.
1330 Notes
1331 -----
1332 The context manager will automatically delete any local temporary
1333 file.
1335 Examples
1336 --------
1337 Should be used as a context manager:
1339 .. code-block:: py
1341 with uri.as_local() as local:
1342 ospath = local.ospath
1343 """
1344 if self.isdir():
1345 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.")
1346 temp_dir = ResourcePath(tmpdir, forceDirectory=True) if tmpdir is not None else None
1347 if temp_dir is not None and not temp_dir.isLocal:
1348 raise ValueError(f"Temporary directory for as_local must be local resource not {temp_dir}")
1349 with self._as_local(multithreaded=multithreaded, tmpdir=temp_dir) as local_uri:
1350 yield local_uri
1352 @classmethod
1353 @contextlib.contextmanager
1354 def temporary_uri(
1355 cls,
1356 prefix: ResourcePath | None = None,
1357 suffix: str | None = None,
1358 delete: bool = True,
1359 ) -> Iterator[ResourcePath]:
1360 """Create a temporary file-like URI.
1362 Parameters
1363 ----------
1364 prefix : `ResourcePath`, optional
1365 Temporary directory to use (can be any scheme). Without this the
1366 path will be formed as a local file URI in a temporary directory
1367 obtained from `lsst.resources.utils.get_tempdir`. Ensuring that the
1368 prefix location exists is the responsibility of the caller.
1369 suffix : `str`, optional
1370 A file suffix to be used. The ``.`` should be included in this
1371 suffix.
1372 delete : `bool`, optional
1373 By default the resource will be deleted when the context manager
1374 is exited. Setting this flag to `False` will leave the resource
1375 alone.
1377 Yields
1378 ------
1379 uri : `ResourcePath`
1380 The temporary URI. Will be removed when the context is completed.
1381 """
1382 if prefix is None:
1383 prefix = ResourcePath(get_tempdir(), forceDirectory=True)
1385 # Need to create a randomized file name. For consistency do not
1386 # use mkstemp for local and something else for remote. Additionally
1387 # this method does not create the file to prevent name clashes.
1388 characters = "abcdefghijklmnopqrstuvwxyz0123456789_"
1389 rng = Random()
1390 tempname = "".join(rng.choice(characters) for _ in range(16))
1391 if suffix:
1392 tempname += suffix
1393 temporary_uri = prefix.join(tempname, isTemporary=True)
1394 if temporary_uri.isdir():
1395 # If we had a safe way to clean up a remote temporary directory, we
1396 # could support this.
1397 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.")
1398 try:
1399 yield temporary_uri
1400 finally:
1401 if delete:
1402 with contextlib.suppress(FileNotFoundError):
1403 # It's okay if this does not work because the user
1404 # removed the file.
1405 temporary_uri.remove()
1407 def read(self, size: int = -1) -> bytes:
1408 """Open the resource and return the contents in bytes.
1410 Parameters
1411 ----------
1412 size : `int`, optional
1413 The number of bytes to read. Negative or omitted indicates
1414 that all data should be read.
1415 """
1416 raise NotImplementedError()
1418 def write(self, data: bytes, overwrite: bool = True) -> None:
1419 """Write the supplied bytes to the new resource.
1421 Parameters
1422 ----------
1423 data : `bytes`
1424 The bytes to write to the resource. The entire contents of the
1425 resource will be replaced.
1426 overwrite : `bool`, optional
1427 If `True` the resource will be overwritten if it exists. Otherwise
1428 the write will fail.
1429 """
1430 raise NotImplementedError()
1432 def mkdir(self) -> None:
1433 """For a dir-like URI, create the directory resource if needed."""
1434 raise NotImplementedError()
1436 def isdir(self) -> bool:
1437 """Return True if this URI looks like a directory, else False."""
1438 return bool(self.dirLike)
1440 def size(self) -> int:
1441 """For non-dir-like URI, return the size of the resource.
1443 Returns
1444 -------
1445 sz : `int`
1446 The size in bytes of the resource associated with this URI.
1447 Returns 0 if dir-like.
1448 """
1449 raise NotImplementedError()
1451 def __str__(self) -> str:
1452 """Convert the URI to its native string form."""
1453 return self.geturl()
1455 def __repr__(self) -> str:
1456 """Return string representation suitable for evaluation."""
1457 return f'ResourcePath("{self.geturl()}")'
1459 def __eq__(self, other: Any) -> bool:
1460 """Compare supplied object with this `ResourcePath`."""
1461 if not isinstance(other, ResourcePath):
1462 return NotImplemented
1463 return self.geturl() == other.geturl()
1465 def __hash__(self) -> int:
1466 """Return hash of this object."""
1467 return hash(str(self))
1469 def __lt__(self, other: ResourcePath) -> bool:
1470 return self.geturl() < other.geturl()
1472 def __le__(self, other: ResourcePath) -> bool:
1473 return self.geturl() <= other.geturl()
1475 def __gt__(self, other: ResourcePath) -> bool:
1476 return self.geturl() > other.geturl()
1478 def __ge__(self, other: ResourcePath) -> bool:
1479 return self.geturl() >= other.geturl()
1481 def __copy__(self) -> ResourcePath:
1482 """Copy constructor.
1484 Object is immutable so copy can return itself.
1485 """
1486 # Implement here because the __new__ method confuses things
1487 return self
1489 def __deepcopy__(self, memo: Any) -> ResourcePath:
1490 """Deepcopy the object.
1492 Object is immutable so copy can return itself.
1493 """
1494 # Implement here because the __new__ method confuses things
1495 return self
1497 def __getnewargs__(self) -> tuple:
1498 """Support pickling."""
1499 return (str(self),)
1501 @classmethod
1502 def _fixDirectorySep(
1503 cls, parsed: urllib.parse.ParseResult, forceDirectory: bool | None = None
1504 ) -> tuple[urllib.parse.ParseResult, bool | None]:
1505 """Ensure that a path separator is present on directory paths.
1507 Parameters
1508 ----------
1509 parsed : `~urllib.parse.ParseResult`
1510 The result from parsing a URI using `urllib.parse`.
1511 forceDirectory : `bool` or `None`, optional
1512 If `True` forces the URI to end with a separator, otherwise given
1513 URI is interpreted as is. Specifying that the URI is conceptually
1514 equivalent to a directory can break some ambiguities when
1515 interpreting the last element of a path.
1517 Returns
1518 -------
1519 modified : `~urllib.parse.ParseResult`
1520 Update result if a URI is being handled.
1521 dirLike : `bool` or `None`
1522 `True` if given parsed URI has a trailing separator or
1523 ``forceDirectory`` is `True`. Otherwise returns the given value of
1524 ``forceDirectory``.
1525 """
1526 # Assume the forceDirectory flag can give us a clue.
1527 dirLike = forceDirectory
1529 # Directory separator
1530 sep = cls._pathModule.sep
1532 # URI is dir-like if explicitly stated or if it ends on a separator
1533 endsOnSep = parsed.path.endswith(sep)
1535 if forceDirectory is False and endsOnSep:
1536 raise ValueError(
1537 f"URI {parsed.geturl()} ends with {sep} but "
1538 "forceDirectory parameter declares it to be a file."
1539 )
1541 if forceDirectory or endsOnSep:
1542 dirLike = True
1543 # only add the separator if it's not already there
1544 if not endsOnSep:
1545 parsed = parsed._replace(path=parsed.path + sep)
1547 return parsed, dirLike
1549 @classmethod
1550 def _fixupPathUri(
1551 cls,
1552 parsed: urllib.parse.ParseResult,
1553 root: ResourcePath | None = None,
1554 forceAbsolute: bool = False,
1555 forceDirectory: bool | None = None,
1556 ) -> tuple[urllib.parse.ParseResult, bool | None]:
1557 """Correct any issues with the supplied URI.
1559 Parameters
1560 ----------
1561 parsed : `~urllib.parse.ParseResult`
1562 The result from parsing a URI using `urllib.parse`.
1563 root : `ResourcePath`, ignored
1564 Not used by the this implementation since all URIs are
1565 absolute except for those representing the local file system.
1566 forceAbsolute : `bool`, ignored.
1567 Not used by this implementation. URIs are generally always
1568 absolute.
1569 forceDirectory : `bool` or `None`, optional
1570 If `True` forces the URI to end with a separator, otherwise given
1571 URI is interpreted as is. Specifying that the URI is conceptually
1572 equivalent to a directory can break some ambiguities when
1573 interpreting the last element of a path.
1575 Returns
1576 -------
1577 modified : `~urllib.parse.ParseResult`
1578 Update result if a URI is being handled.
1579 dirLike : `bool`
1580 `True` if given parsed URI has a trailing separator or
1581 ``forceDirectory`` is `True`. Otherwise returns the given value
1582 of ``forceDirectory``.
1584 Notes
1585 -----
1586 Relative paths are explicitly not supported by RFC8089 but `urllib`
1587 does accept URIs of the form ``file:relative/path.ext``. They need
1588 to be turned into absolute paths before they can be used. This is
1589 always done regardless of the ``forceAbsolute`` parameter.
1591 AWS S3 differentiates between keys with trailing POSIX separators (i.e
1592 ``/dir`` and ``/dir/``) whereas POSIX does not necessarily.
1594 Scheme-less paths are normalized.
1595 """
1596 return cls._fixDirectorySep(parsed, forceDirectory)
1598 def transfer_from(
1599 self,
1600 src: ResourcePath,
1601 transfer: str,
1602 overwrite: bool = False,
1603 transaction: TransactionProtocol | None = None,
1604 multithreaded: bool = True,
1605 ) -> None:
1606 """Transfer to this URI from another.
1608 Parameters
1609 ----------
1610 src : `ResourcePath`
1611 Source URI.
1612 transfer : `str`
1613 Mode to use for transferring the resource. Generically there are
1614 many standard options: copy, link, symlink, hardlink, relsymlink.
1615 Not all URIs support all modes.
1616 overwrite : `bool`, optional
1617 Allow an existing file to be overwritten. Defaults to `False`.
1618 transaction : `~lsst.resources.utils.TransactionProtocol`, optional
1619 A transaction object that can (depending on implementation)
1620 rollback transfers on error. Not guaranteed to be implemented.
1621 multithreaded : `bool`, optional
1622 If `True` the transfer will be allowed to attempt to improve
1623 throughput by using parallel download streams. This may of no
1624 effect if the URI scheme does not support parallel streams or
1625 if a global override has been applied. If `False` parallel
1626 streams will be disabled.
1628 Notes
1629 -----
1630 Conceptually this is hard to scale as the number of URI schemes
1631 grow. The destination URI is more important than the source URI
1632 since that is where all the transfer modes are relevant (with the
1633 complication that "move" deletes the source).
1635 Local file to local file is the fundamental use case but every
1636 other scheme has to support "copy" to local file (with implicit
1637 support for "move") and copy from local file.
1638 All the "link" options tend to be specific to local file systems.
1640 "move" is a "copy" where the remote resource is deleted at the end.
1641 Whether this works depends on the source URI rather than the
1642 destination URI. Reverting a move on transaction rollback is
1643 expected to be problematic if a remote resource was involved.
1644 """
1645 raise NotImplementedError(f"No transfer modes supported by URI scheme {self.scheme}")
1647 def walk(
1648 self, file_filter: str | re.Pattern | None = None
1649 ) -> Iterator[list | tuple[ResourcePath, list[str], list[str]]]:
1650 """Walk the directory tree returning matching files and directories.
1652 Parameters
1653 ----------
1654 file_filter : `str` or `re.Pattern`, optional
1655 Regex to filter out files from the list before it is returned.
1657 Yields
1658 ------
1659 dirpath : `ResourcePath`
1660 Current directory being examined.
1661 dirnames : `list` of `str`
1662 Names of subdirectories within dirpath.
1663 filenames : `list` of `str`
1664 Names of all the files within dirpath.
1665 """
1666 raise NotImplementedError()
1668 @overload
1669 @classmethod
1670 def findFileResources( 1670 ↛ exitline 1670 didn't return from function 'findFileResources' because
1671 cls,
1672 candidates: Iterable[ResourcePathExpression],
1673 file_filter: str | re.Pattern | None,
1674 grouped: Literal[True],
1675 ) -> Iterator[Iterator[ResourcePath]]: ...
1677 @overload
1678 @classmethod
1679 def findFileResources( 1679 ↛ exitline 1679 didn't return from function 'findFileResources' because
1680 cls,
1681 candidates: Iterable[ResourcePathExpression],
1682 *,
1683 grouped: Literal[True],
1684 ) -> Iterator[Iterator[ResourcePath]]: ...
1686 @overload
1687 @classmethod
1688 def findFileResources( 1688 ↛ exitline 1688 didn't return from function 'findFileResources' because
1689 cls,
1690 candidates: Iterable[ResourcePathExpression],
1691 file_filter: str | re.Pattern | None = None,
1692 grouped: Literal[False] = False,
1693 ) -> Iterator[ResourcePath]: ...
1695 @classmethod
1696 def findFileResources(
1697 cls,
1698 candidates: Iterable[ResourcePathExpression],
1699 file_filter: str | re.Pattern | None = None,
1700 grouped: bool = False,
1701 ) -> Iterator[ResourcePath | Iterator[ResourcePath]]:
1702 """Get all the files from a list of values.
1704 Parameters
1705 ----------
1706 candidates : iterable [`str` or `ResourcePath`]
1707 The files to return and directories in which to look for files to
1708 return.
1709 file_filter : `str` or `re.Pattern`, optional
1710 The regex to use when searching for files within directories.
1711 By default returns all the found files.
1712 grouped : `bool`, optional
1713 If `True` the results will be grouped by directory and each
1714 yielded value will be an iterator over URIs. If `False` each
1715 URI will be returned separately.
1717 Yields
1718 ------
1719 found_file: `ResourcePath`
1720 The passed-in URIs and URIs found in passed-in directories.
1721 If grouping is enabled, each of the yielded values will be an
1722 iterator yielding members of the group. Files given explicitly
1723 will be returned as a single group at the end.
1725 Notes
1726 -----
1727 If a value is a file it is yielded immediately without checking that it
1728 exists. If a value is a directory, all the files in the directory
1729 (recursively) that match the regex will be yielded in turn.
1730 """
1731 fileRegex = None if file_filter is None else re.compile(file_filter)
1733 singles = []
1735 # Find all the files of interest
1736 for location in candidates:
1737 uri = ResourcePath(location)
1738 if uri.isdir():
1739 for found in uri.walk(fileRegex):
1740 if not found:
1741 # This means the uri does not exist and by
1742 # convention we ignore it
1743 continue
1744 root, dirs, files = found
1745 if not files:
1746 continue
1747 if grouped:
1748 yield (root.join(name) for name in files)
1749 else:
1750 for name in files:
1751 yield root.join(name)
1752 else:
1753 if grouped:
1754 singles.append(uri)
1755 else:
1756 yield uri
1758 # Finally, return any explicitly given files in one group
1759 if grouped and singles:
1760 yield iter(singles)
1762 @contextlib.contextmanager
1763 def open(
1764 self,
1765 mode: str = "r",
1766 *,
1767 encoding: str | None = None,
1768 prefer_file_temporary: bool = False,
1769 ) -> Iterator[ResourceHandleProtocol]:
1770 """Return a context manager that wraps an object that behaves like an
1771 open file at the location of the URI.
1773 Parameters
1774 ----------
1775 mode : `str`
1776 String indicating the mode in which to open the file. Values are
1777 the same as those accepted by `open`, though intrinsically
1778 read-only URI types may only support read modes, and
1779 `io.IOBase.seekable` is not guaranteed to be `True` on the returned
1780 object.
1781 encoding : `str`, optional
1782 Unicode encoding for text IO; ignored for binary IO. Defaults to
1783 ``locale.getpreferredencoding(False)``, just as `open`
1784 does.
1785 prefer_file_temporary : `bool`, optional
1786 If `True`, for implementations that require transfers from a remote
1787 system to temporary local storage and/or back, use a temporary file
1788 instead of an in-memory buffer; this is generally slower, but it
1789 may be necessary to avoid excessive memory usage by large files.
1790 Ignored by implementations that do not require a temporary.
1792 Yields
1793 ------
1794 cm : `~contextlib.AbstractContextManager`
1795 A context manager that wraps a `ResourceHandleProtocol` file-like
1796 object.
1798 Notes
1799 -----
1800 The default implementation of this method uses a local temporary buffer
1801 (in-memory or file, depending on ``prefer_file_temporary``) with calls
1802 to `read`, `write`, `as_local`, and `transfer_from` as necessary to
1803 read and write from/to remote systems. Remote writes thus occur only
1804 when the context manager is exited. `ResourcePath` implementations
1805 that can return a more efficient native buffer should do so whenever
1806 possible (as is guaranteed for local files). `ResourcePath`
1807 implementations for which `as_local` does not return a temporary are
1808 required to reimplement `open`, though they may delegate to `super`
1809 when ``prefer_file_temporary`` is `False`.
1810 """
1811 if self.isdir():
1812 raise IsADirectoryError(f"Directory-like URI {self} cannot be opened.")
1813 if "x" in mode and self.exists():
1814 raise FileExistsError(f"File at {self} already exists.")
1815 if prefer_file_temporary:
1816 if "r" in mode or "a" in mode:
1817 local_cm = self.as_local()
1818 else:
1819 local_cm = self.temporary_uri(suffix=self.getExtension())
1820 with local_cm as local_uri:
1821 assert local_uri.isTemporary, (
1822 "ResourcePath implementations for which as_local is not "
1823 "a temporary must reimplement `open`."
1824 )
1825 with open(local_uri.ospath, mode=mode, encoding=encoding) as file_buffer:
1826 if "a" in mode:
1827 file_buffer.seek(0, io.SEEK_END)
1828 yield file_buffer
1829 if "r" not in mode or "+" in mode:
1830 self.transfer_from(local_uri, transfer="copy", overwrite=("x" not in mode))
1831 else:
1832 with self._openImpl(mode, encoding=encoding) as handle:
1833 yield handle
1835 @contextlib.contextmanager
1836 def _openImpl(self, mode: str = "r", *, encoding: str | None = None) -> Iterator[ResourceHandleProtocol]:
1837 """Implement opening of a resource handle.
1839 This private method may be overridden by specific `ResourcePath`
1840 implementations to provide a customized handle like interface.
1842 Parameters
1843 ----------
1844 mode : `str`
1845 The mode the handle should be opened with
1846 encoding : `str`, optional
1847 The byte encoding of any binary text
1849 Yields
1850 ------
1851 handle : `~._resourceHandles.BaseResourceHandle`
1852 A handle that conforms to the
1853 `~._resourceHandles.BaseResourceHandle` interface
1855 Notes
1856 -----
1857 The base implementation of a file handle reads in a files entire
1858 contents into a buffer for manipulation, and then writes it back out
1859 upon close. Subclasses of this class may offer more fine grained
1860 control.
1861 """
1862 in_bytes = self.read() if "r" in mode or "a" in mode else b""
1863 if "b" in mode:
1864 bytes_buffer = io.BytesIO(in_bytes)
1865 bytes_buffer.name = str(self)
1866 if "a" in mode:
1867 bytes_buffer.seek(0, io.SEEK_END)
1868 yield bytes_buffer
1869 out_bytes = bytes_buffer.getvalue()
1870 else:
1871 if encoding is None:
1872 encoding = locale.getpreferredencoding(False)
1873 str_buffer = io.StringIO(in_bytes.decode(encoding))
1874 str_buffer.name = str(self)
1875 if "a" in mode:
1876 str_buffer.seek(0, io.SEEK_END)
1877 yield str_buffer
1878 out_bytes = str_buffer.getvalue().encode(encoding)
1879 if "r" not in mode or "+" in mode:
1880 self.write(out_bytes, overwrite=("x" not in mode))
1882 def generate_presigned_get_url(self, *, expiration_time_seconds: int) -> str:
1883 """Return a pre-signed URL that can be used to retrieve this resource
1884 using an HTTP GET without supplying any access credentials.
1886 Parameters
1887 ----------
1888 expiration_time_seconds : `int`
1889 Number of seconds until the generated URL is no longer valid.
1891 Returns
1892 -------
1893 url : `str`
1894 HTTP URL signed for GET.
1895 """
1896 raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
1898 def generate_presigned_put_url(self, *, expiration_time_seconds: int) -> str:
1899 """Return a pre-signed URL that can be used to upload a file to this
1900 path using an HTTP PUT without supplying any access credentials.
1902 Parameters
1903 ----------
1904 expiration_time_seconds : `int`
1905 Number of seconds until the generated URL is no longer valid.
1907 Returns
1908 -------
1909 url : `str`
1910 HTTP URL signed for PUT.
1911 """
1912 raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
1914 def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
1915 # May be overridden by subclasses to transfer attributes when a
1916 # ResourcePath is constructed using the "clone" version of the
1917 # ResourcePath constructor by passing in a ResourcePath object.
1918 pass
1921ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
1922"""Type-annotation alias for objects that can be coerced to ResourcePath.
1923"""