Coverage for python/lsst/resources/_resourcePath.py: 29%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14import concurrent.futures 

15import contextlib 

16import copy 

17import io 

18import locale 

19import logging 

20import os 

21import posixpath 

22import re 

23import shutil 

24import tempfile 

25import urllib.parse 

26from pathlib import Path, PurePath, PurePosixPath 

27from random import Random 

28 

29__all__ = ("ResourcePath", "ResourcePathExpression") 

30 

31from collections.abc import Iterable, Iterator 

32from typing import TYPE_CHECKING, Any, Literal, overload 

33 

34from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol 

35 

36if TYPE_CHECKING: 

37 from .utils import TransactionProtocol 

38 

39 

40log = logging.getLogger(__name__) 

41 

42# Regex for looking for URI escapes 

43ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

44 

45# Precomputed escaped hash 

46ESCAPED_HASH = urllib.parse.quote("#") 

47 

48# Maximum number of worker threads for parallelized operations. 

49# If greater than 10, be aware that this number has to be consistent 

50# with connection pool sizing (for example in urllib3). 

51MAX_WORKERS = 10 

52 

53 

54class ResourcePath: 

55 """Convenience wrapper around URI parsers. 

56 

57 Provides access to URI components and can convert file 

58 paths into absolute path URIs. Scheme-less URIs are treated as if 

59 they are local file system paths and are converted to absolute URIs. 

60 

61 A specialist subclass is created for each supported URI scheme. 

62 

63 Parameters 

64 ---------- 

65 uri : `str`, `pathlib.Path`, `urllib.parse.ParseResult`, or `ResourcePath`. 

66 URI in string form. Can be scheme-less if referring to a relative 

67 path or an absolute path on the local file system. 

68 root : `str` or `ResourcePath`, optional 

69 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

70 use this as the root. Must be absolute. If `None` the current 

71 working directory will be used. Can be any supported URI scheme. 

72 Not used if ``forceAbsolute`` is `False`. 

73 forceAbsolute : `bool`, optional 

74 If `True`, scheme-less relative URI will be converted to an absolute 

75 path using a ``file`` scheme. If `False` scheme-less URI will remain 

76 scheme-less and will not be updated to ``file`` or absolute path unless 

77 it is already an absolute path, in which case it will be updated to 

78 a ``file`` scheme. 

79 forceDirectory: `bool`, optional 

80 If `True` forces the URI to end with a separator, otherwise given URI 

81 is interpreted as is. 

82 isTemporary : `bool`, optional 

83 If `True` indicates that this URI points to a temporary resource. 

84 The default is `False`, unless ``uri`` is already a `ResourcePath` 

85 instance and ``uri.isTemporary is True``. 

86 

87 Notes 

88 ----- 

89 A non-standard URI of the form ``file:dir/file.txt`` is always converted 

90 to an absolute ``file`` URI. 

91 """ 

92 

93 _pathLib: type[PurePath] = PurePosixPath 

94 """Path library to use for this scheme.""" 

95 

96 _pathModule = posixpath 

97 """Path module to use for this scheme.""" 

98 

99 transferModes: tuple[str, ...] = ("copy", "auto", "move") 

100 """Transfer modes supported by this implementation. 

101 

102 Move is special in that it is generally a copy followed by an unlink. 

103 Whether that unlink works depends critically on whether the source URI 

104 implements unlink. If it does not the move will be reported as a failure. 

105 """ 

106 

107 transferDefault: str = "copy" 

108 """Default mode to use for transferring if ``auto`` is specified.""" 

109 

110 quotePaths = True 

111 """True if path-like elements modifying a URI should be quoted. 

112 

113 All non-schemeless URIs have to internally use quoted paths. Therefore 

114 if a new file name is given (e.g. to updatedFile or join) a decision must 

115 be made whether to quote it to be consistent. 

116 """ 

117 

118 isLocal = False 

119 """If `True` this URI refers to a local file.""" 

120 

121 # This is not an ABC with abstract methods because the __new__ being 

122 # a factory confuses mypy such that it assumes that every constructor 

123 # returns a ResourcePath and then determines that all the abstract methods 

124 # are still abstract. If they are not marked abstract but just raise 

125 # mypy is fine with it. 

126 

127 # mypy is confused without these 

128 _uri: urllib.parse.ParseResult 

129 isTemporary: bool 

130 dirLike: bool 

131 

132 def __new__( 

133 cls, 

134 uri: ResourcePathExpression, 

135 root: str | ResourcePath | None = None, 

136 forceAbsolute: bool = True, 

137 forceDirectory: bool = False, 

138 isTemporary: bool | None = None, 

139 ) -> ResourcePath: 

140 """Create and return new specialist ResourcePath subclass.""" 

141 parsed: urllib.parse.ParseResult 

142 dirLike: bool = False 

143 subclass: type[ResourcePath] | None = None 

144 

145 # Force root to be a ResourcePath -- this simplifies downstream 

146 # code. 

147 if root is None: 

148 root_uri = None 

149 elif isinstance(root, str): 

150 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True) 

151 else: 

152 root_uri = root 

153 

154 if isinstance(uri, os.PathLike): 

155 uri = str(uri) 

156 

157 # Record if we need to post process the URI components 

158 # or if the instance is already fully configured 

159 if isinstance(uri, str): 

160 # Since local file names can have special characters in them 

161 # we need to quote them for the parser but we can unquote 

162 # later. Assume that all other URI schemes are quoted. 

163 # Since sometimes people write file:/a/b and not file:///a/b 

164 # we should not quote in the explicit case of file: 

165 if "://" not in uri and not uri.startswith("file:"): 

166 if ESCAPES_RE.search(uri): 

167 log.warning("Possible double encoding of %s", uri) 

168 else: 

169 uri = urllib.parse.quote(uri) 

170 # Special case hash since we must support fragments 

171 # even in schemeless URIs -- although try to only replace 

172 # them in file part and not directory part 

173 if ESCAPED_HASH in uri: 

174 dirpos = uri.rfind("/") 

175 # Do replacement after this / 

176 uri = uri[: dirpos + 1] + uri[dirpos + 1 :].replace(ESCAPED_HASH, "#") 

177 

178 parsed = urllib.parse.urlparse(uri) 

179 elif isinstance(uri, urllib.parse.ParseResult): 

180 parsed = copy.copy(uri) 

181 # If we are being instantiated with a subclass, rather than 

182 # ResourcePath, ensure that that subclass is used directly. 

183 # This could lead to inconsistencies if this constructor 

184 # is used externally outside of the ResourcePath.replace() method. 

185 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt")) 

186 # will be a problem. 

187 # This is needed to prevent a schemeless absolute URI become 

188 # a file URI unexpectedly when calling updatedFile or 

189 # updatedExtension 

190 if cls is not ResourcePath: 

191 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory) 

192 subclass = cls 

193 

194 elif isinstance(uri, ResourcePath): 

195 # Since ResourcePath is immutable we can return the argument 

196 # unchanged if it already agrees with forceDirectory, isTemporary, 

197 # and forceAbsolute. 

198 # We invoke __new__ again with str(self) to add a scheme for 

199 # forceAbsolute, but for the others that seems more likely to paper 

200 # over logic errors than do something useful, so we just raise. 

201 if forceDirectory and not uri.dirLike: 

202 raise RuntimeError( 

203 f"{uri} is already a file-like ResourcePath; cannot force it to directory." 

204 ) 

205 if isTemporary is not None and isTemporary is not uri.isTemporary: 

206 raise RuntimeError( 

207 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} " 

208 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}." 

209 ) 

210 if forceAbsolute and not uri.scheme: 

211 return ResourcePath( 

212 str(uri), 

213 root=root, 

214 forceAbsolute=True, 

215 forceDirectory=uri.dirLike, 

216 isTemporary=uri.isTemporary, 

217 ) 

218 return uri 

219 else: 

220 raise ValueError( 

221 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'" 

222 ) 

223 

224 if subclass is None: 

225 # Work out the subclass from the URI scheme 

226 if not parsed.scheme: 

227 # Root may be specified as a ResourcePath that overrides 

228 # the schemeless determination. 

229 if ( 

230 root_uri is not None 

231 and root_uri.scheme != "file" # file scheme has different code path 

232 and not parsed.path.startswith("/") # Not already absolute path 

233 ): 

234 if not root_uri.dirLike: 

235 raise ValueError( 

236 f"Root URI ({root}) was not a directory so can not be joined with" 

237 f" path {parsed.path!r}" 

238 ) 

239 # If root is temporary or this schemeless is temporary we 

240 # assume this URI is temporary. 

241 isTemporary = isTemporary or root_uri.isTemporary 

242 joined = root_uri.join( 

243 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary 

244 ) 

245 

246 # Rather than returning this new ResourcePath directly we 

247 # instead extract the path and the scheme and adjust the 

248 # URI we were given -- we need to do this to preserve 

249 # fragments since join() will drop them. 

250 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc) 

251 subclass = type(joined) 

252 

253 # Clear the root parameter to indicate that it has 

254 # been applied already. 

255 root_uri = None 

256 else: 

257 from .schemeless import SchemelessResourcePath 

258 

259 subclass = SchemelessResourcePath 

260 elif parsed.scheme == "file": 

261 from .file import FileResourcePath 

262 

263 subclass = FileResourcePath 

264 elif parsed.scheme == "s3": 

265 from .s3 import S3ResourcePath 

266 

267 subclass = S3ResourcePath 

268 elif parsed.scheme.startswith("http"): 

269 from .http import HttpResourcePath 

270 

271 subclass = HttpResourcePath 

272 elif parsed.scheme == "gs": 

273 from .gs import GSResourcePath 

274 

275 subclass = GSResourcePath 

276 elif parsed.scheme == "resource": 

277 # Rules for scheme names disallow pkg_resource 

278 from .packageresource import PackageResourcePath 

279 

280 subclass = PackageResourcePath 

281 elif parsed.scheme == "mem": 

282 # in-memory datastore object 

283 from .mem import InMemoryResourcePath 

284 

285 subclass = InMemoryResourcePath 

286 else: 

287 raise NotImplementedError( 

288 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" 

289 ) 

290 

291 parsed, dirLike = subclass._fixupPathUri( 

292 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory 

293 ) 

294 

295 # It is possible for the class to change from schemeless 

296 # to file so handle that 

297 if parsed.scheme == "file": 

298 from .file import FileResourcePath 

299 

300 subclass = FileResourcePath 

301 

302 # Now create an instance of the correct subclass and set the 

303 # attributes directly 

304 self = object.__new__(subclass) 

305 self._uri = parsed 

306 self.dirLike = dirLike 

307 if isTemporary is None: 

308 isTemporary = False 

309 self.isTemporary = isTemporary 

310 return self 

311 

312 @property 

313 def scheme(self) -> str: 

314 """Return the URI scheme. 

315 

316 Notes 

317 ----- 

318 (``://`` is not part of the scheme). 

319 """ 

320 return self._uri.scheme 

321 

322 @property 

323 def netloc(self) -> str: 

324 """Return the URI network location.""" 

325 return self._uri.netloc 

326 

327 @property 

328 def path(self) -> str: 

329 """Return the path component of the URI.""" 

330 return self._uri.path 

331 

332 @property 

333 def unquoted_path(self) -> str: 

334 """Return path component of the URI with any URI quoting reversed.""" 

335 return urllib.parse.unquote(self._uri.path) 

336 

337 @property 

338 def ospath(self) -> str: 

339 """Return the path component of the URI localized to current OS.""" 

340 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

341 

342 @property 

343 def relativeToPathRoot(self) -> str: 

344 """Return path relative to network location. 

345 

346 Effectively, this is the path property with posix separator stripped 

347 from the left hand side of the path. 

348 

349 Always unquotes. 

350 """ 

351 p = self._pathLib(self.path) 

352 relToRoot = str(p.relative_to(p.root)) 

353 if self.dirLike and not relToRoot.endswith("/"): 

354 relToRoot += "/" 

355 return urllib.parse.unquote(relToRoot) 

356 

357 @property 

358 def is_root(self) -> bool: 

359 """Return whether this URI points to the root of the network location. 

360 

361 This means that the path components refers to the top level. 

362 """ 

363 relpath = self.relativeToPathRoot 

364 if relpath == "./": 

365 return True 

366 return False 

367 

368 @property 

369 def fragment(self) -> str: 

370 """Return the fragment component of the URI.""" 

371 return self._uri.fragment 

372 

373 @property 

374 def params(self) -> str: 

375 """Return any parameters included in the URI.""" 

376 return self._uri.params 

377 

378 @property 

379 def query(self) -> str: 

380 """Return any query strings included in the URI.""" 

381 return self._uri.query 

382 

383 def geturl(self) -> str: 

384 """Return the URI in string form. 

385 

386 Returns 

387 ------- 

388 url : `str` 

389 String form of URI. 

390 """ 

391 return self._uri.geturl() 

392 

393 def root_uri(self) -> ResourcePath: 

394 """Return the base root URI. 

395 

396 Returns 

397 ------- 

398 uri : `ResourcePath` 

399 root URI. 

400 """ 

401 return self.replace(path="", forceDirectory=True) 

402 

403 def split(self) -> tuple[ResourcePath, str]: 

404 """Split URI into head and tail. 

405 

406 Returns 

407 ------- 

408 head: `ResourcePath` 

409 Everything leading up to tail, expanded and normalized as per 

410 ResourcePath rules. 

411 tail : `str` 

412 Last path component. Tail will be empty if path ends on a 

413 separator. Tail will never contain separators. It will be 

414 unquoted. 

415 

416 Notes 

417 ----- 

418 Equivalent to `os.path.split` where head preserves the URI 

419 components. 

420 """ 

421 head, tail = self._pathModule.split(self.path) 

422 headuri = self._uri._replace(path=head) 

423 

424 # The file part should never include quoted metacharacters 

425 tail = urllib.parse.unquote(tail) 

426 

427 # Schemeless is special in that it can be a relative path 

428 # We need to ensure that it stays that way. All other URIs will 

429 # be absolute already. 

430 forceAbsolute = self._pathModule.isabs(self.path) 

431 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

432 

433 def basename(self) -> str: 

434 """Return the base name, last element of path, of the URI. 

435 

436 Returns 

437 ------- 

438 tail : `str` 

439 Last part of the path attribute. Trail will be empty if path ends 

440 on a separator. 

441 

442 Notes 

443 ----- 

444 If URI ends on a slash returns an empty string. This is the second 

445 element returned by `split()`. 

446 

447 Equivalent of `os.path.basename`. 

448 """ 

449 return self.split()[1] 

450 

451 def dirname(self) -> ResourcePath: 

452 """Return the directory component of the path as a new `ResourcePath`. 

453 

454 Returns 

455 ------- 

456 head : `ResourcePath` 

457 Everything except the tail of path attribute, expanded and 

458 normalized as per ResourcePath rules. 

459 

460 Notes 

461 ----- 

462 Equivalent of `os.path.dirname`. 

463 """ 

464 return self.split()[0] 

465 

466 def parent(self) -> ResourcePath: 

467 """Return a `ResourcePath` of the parent directory. 

468 

469 Returns 

470 ------- 

471 head : `ResourcePath` 

472 Everything except the tail of path attribute, expanded and 

473 normalized as per `ResourcePath` rules. 

474 

475 Notes 

476 ----- 

477 For a file-like URI this will be the same as calling `dirname()`. 

478 """ 

479 # When self is file-like, return self.dirname() 

480 if not self.dirLike: 

481 return self.dirname() 

482 # When self is dir-like, return its parent directory, 

483 # regardless of the presence of a trailing separator 

484 originalPath = self._pathLib(self.path) 

485 parentPath = originalPath.parent 

486 return self.replace(path=str(parentPath), forceDirectory=True) 

487 

488 def replace(self, forceDirectory: bool = False, isTemporary: bool = False, **kwargs: Any) -> ResourcePath: 

489 """Return new `ResourcePath` with specified components replaced. 

490 

491 Parameters 

492 ---------- 

493 forceDirectory : `bool`, optional 

494 Parameter passed to ResourcePath constructor to force this 

495 new URI to be dir-like. 

496 isTemporary : `bool`, optional 

497 Indicate that the resulting URI is temporary resource. 

498 **kwargs 

499 Components of a `urllib.parse.ParseResult` that should be 

500 modified for the newly-created `ResourcePath`. 

501 

502 Returns 

503 ------- 

504 new : `ResourcePath` 

505 New `ResourcePath` object with updated values. 

506 

507 Notes 

508 ----- 

509 Does not, for now, allow a change in URI scheme. 

510 """ 

511 # Disallow a change in scheme 

512 if "scheme" in kwargs: 

513 raise ValueError(f"Can not use replace() method to change URI scheme for {self}") 

514 return self.__class__( 

515 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary 

516 ) 

517 

518 def updatedFile(self, newfile: str) -> ResourcePath: 

519 """Return new URI with an updated final component of the path. 

520 

521 Parameters 

522 ---------- 

523 newfile : `str` 

524 File name with no path component. 

525 

526 Returns 

527 ------- 

528 updated : `ResourcePath` 

529 

530 Notes 

531 ----- 

532 Forces the ResourcePath.dirLike attribute to be false. The new file 

533 path will be quoted if necessary. 

534 """ 

535 if self.quotePaths: 

536 newfile = urllib.parse.quote(newfile) 

537 dir, _ = self._pathModule.split(self.path) 

538 newpath = self._pathModule.join(dir, newfile) 

539 

540 updated = self.replace(path=newpath) 

541 updated.dirLike = False 

542 return updated 

543 

544 def updatedExtension(self, ext: str | None) -> ResourcePath: 

545 """Return a new `ResourcePath` with updated file extension. 

546 

547 All file extensions are replaced. 

548 

549 Parameters 

550 ---------- 

551 ext : `str` or `None` 

552 New extension. If an empty string is given any extension will 

553 be removed. If `None` is given there will be no change. 

554 

555 Returns 

556 ------- 

557 updated : `ResourcePath` 

558 URI with the specified extension. Can return itself if 

559 no extension was specified. 

560 """ 

561 if ext is None: 

562 return self 

563 

564 # Get the extension 

565 current = self.getExtension() 

566 

567 # Nothing to do if the extension already matches 

568 if current == ext: 

569 return self 

570 

571 # Remove the current extension from the path 

572 # .fits.gz counts as one extension do not use os.path.splitext 

573 path = self.path 

574 if current: 

575 path = path[: -len(current)] 

576 

577 # Ensure that we have a leading "." on file extension (and we do not 

578 # try to modify the empty string) 

579 if ext and not ext.startswith("."): 

580 ext = "." + ext 

581 

582 return self.replace(path=path + ext) 

583 

584 def getExtension(self) -> str: 

585 """Return the file extension(s) associated with this URI path. 

586 

587 Returns 

588 ------- 

589 ext : `str` 

590 The file extension (including the ``.``). Can be empty string 

591 if there is no file extension. Usually returns only the last 

592 file extension unless there is a special extension modifier 

593 indicating file compression, in which case the combined 

594 extension (e.g. ``.fits.gz``) will be returned. 

595 """ 

596 special = {".gz", ".bz2", ".xz", ".fz"} 

597 

598 # Get the file part of the path so as not to be confused by 

599 # "." in directory names. 

600 basename = self.basename() 

601 extensions = self._pathLib(basename).suffixes 

602 

603 if not extensions: 

604 return "" 

605 

606 ext = extensions.pop() 

607 

608 # Multiple extensions, decide whether to include the final two 

609 if extensions and ext in special: 

610 ext = f"{extensions[-1]}{ext}" 

611 

612 return ext 

613 

614 def join( 

615 self, path: str | ResourcePath, isTemporary: bool | None = None, forceDirectory: bool = False 

616 ) -> ResourcePath: 

617 """Return new `ResourcePath` with additional path components. 

618 

619 Parameters 

620 ---------- 

621 path : `str`, `ResourcePath` 

622 Additional file components to append to the current URI. Assumed 

623 to include a file at the end. Will be quoted depending on the 

624 associated URI scheme. If the path looks like a URI with a scheme 

625 referring to an absolute location, it will be returned 

626 directly (matching the behavior of `os.path.join`). It can 

627 also be a `ResourcePath`. 

628 isTemporary : `bool`, optional 

629 Indicate that the resulting URI represents a temporary resource. 

630 Default is ``self.isTemporary``. 

631 forceDirectory : `bool`, optional 

632 If `True` forces the URI to end with a separator, otherwise given 

633 URI is interpreted as is. 

634 

635 Returns 

636 ------- 

637 new : `ResourcePath` 

638 New URI with any file at the end replaced with the new path 

639 components. 

640 

641 Notes 

642 ----- 

643 Schemeless URIs assume local path separator but all other URIs assume 

644 POSIX separator if the supplied path has directory structure. It 

645 may be this never becomes a problem but datastore templates assume 

646 POSIX separator is being used. 

647 

648 If an absolute `ResourcePath` is given for ``path`` is is assumed that 

649 this should be returned directly. Giving a ``path`` of an absolute 

650 scheme-less URI is not allowed for safety reasons as it may indicate 

651 a mistake in the calling code. 

652 

653 Raises 

654 ------ 

655 ValueError 

656 Raised if the ``path`` is an absolute scheme-less URI. In that 

657 situation it is unclear whether the intent is to return a 

658 ``file`` URI or it was a mistake and a relative scheme-less URI 

659 was meant. 

660 RuntimeError 

661 Raised if this attempts to join a temporary URI to a non-temporary 

662 URI. 

663 """ 

664 if isTemporary is None: 

665 isTemporary = self.isTemporary 

666 elif not isTemporary and self.isTemporary: 

667 raise RuntimeError("Cannot join temporary URI to non-temporary URI.") 

668 # If we have a full URI in path we will use it directly 

669 # but without forcing to absolute so that we can trap the 

670 # expected option of relative path. 

671 path_uri = ResourcePath( 

672 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary 

673 ) 

674 if path_uri.scheme: 

675 # Check for scheme so can distinguish explicit URIs from 

676 # absolute scheme-less URIs. 

677 return path_uri 

678 

679 if path_uri.isabs(): 

680 # Absolute scheme-less path. 

681 raise ValueError(f"Can not join absolute scheme-less {path_uri!r} to another URI.") 

682 

683 # If this was originally a ResourcePath extract the unquoted path from 

684 # it. Otherwise we use the string we were given to allow "#" to appear 

685 # in the filename if given as a plain string. 

686 if not isinstance(path, str): 

687 path = path_uri.unquoted_path 

688 

689 new = self.dirname() # By definition a directory URI 

690 

691 # new should be asked about quoting, not self, since dirname can 

692 # change the URI scheme for schemeless -> file 

693 if new.quotePaths: 

694 path = urllib.parse.quote(path) 

695 

696 newpath = self._pathModule.normpath(self._pathModule.join(new.path, path)) 

697 

698 # normpath can strip trailing / so we force directory if the supplied 

699 # path ended with a / 

700 return new.replace( 

701 path=newpath, 

702 forceDirectory=(forceDirectory or path.endswith(self._pathModule.sep)), 

703 isTemporary=isTemporary, 

704 ) 

705 

706 def relative_to(self, other: ResourcePath) -> str | None: 

707 """Return the relative path from this URI to the other URI. 

708 

709 Parameters 

710 ---------- 

711 other : `ResourcePath` 

712 URI to use to calculate the relative path. Must be a parent 

713 of this URI. 

714 

715 Returns 

716 ------- 

717 subpath : `str` 

718 The sub path of this URI relative to the supplied other URI. 

719 Returns `None` if there is no parent child relationship. 

720 Scheme and netloc must match. 

721 """ 

722 # Scheme-less absolute other is treated as if it's a file scheme. 

723 # Scheme-less relative other can only return non-None if self 

724 # is also scheme-less relative and that is handled specifically 

725 # in a subclass. 

726 if not other.scheme and other.isabs(): 

727 other = other.abspath() 

728 

729 # Scheme-less self is handled elsewhere. 

730 if self.scheme != other.scheme: 

731 return None 

732 if self.netloc != other.netloc: 

733 # Special case for localhost vs empty string. 

734 # There can be many variants of localhost. 

735 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"} 

736 if not {self.netloc, other.netloc}.issubset(local_netlocs): 

737 return None 

738 

739 enclosed_path = self._pathLib(self.relativeToPathRoot) 

740 parent_path = other.relativeToPathRoot 

741 subpath: str | None 

742 try: 

743 subpath = str(enclosed_path.relative_to(parent_path)) 

744 except ValueError: 

745 subpath = None 

746 else: 

747 subpath = urllib.parse.unquote(subpath) 

748 return subpath 

749 

750 def exists(self) -> bool: 

751 """Indicate that the resource is available. 

752 

753 Returns 

754 ------- 

755 exists : `bool` 

756 `True` if the resource exists. 

757 """ 

758 raise NotImplementedError() 

759 

760 @classmethod 

761 def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

762 """Check for existence of multiple URIs at once. 

763 

764 Parameters 

765 ---------- 

766 uris : iterable of `ResourcePath` 

767 The URIs to test. 

768 

769 Returns 

770 ------- 

771 existence : `dict` of [`ResourcePath`, `bool`] 

772 Mapping of original URI to boolean indicating existence. 

773 """ 

774 exists_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 

775 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris} 

776 

777 results: dict[ResourcePath, bool] = {} 

778 for future in concurrent.futures.as_completed(future_exists): 

779 uri = future_exists[future] 

780 try: 

781 exists = future.result() 

782 except Exception: 

783 exists = False 

784 results[uri] = exists 

785 return results 

786 

787 def remove(self) -> None: 

788 """Remove the resource.""" 

789 raise NotImplementedError() 

790 

791 def isabs(self) -> bool: 

792 """Indicate that the resource is fully specified. 

793 

794 For non-schemeless URIs this is always true. 

795 

796 Returns 

797 ------- 

798 isabs : `bool` 

799 `True` in all cases except schemeless URI. 

800 """ 

801 return True 

802 

803 def abspath(self) -> ResourcePath: 

804 """Return URI using an absolute path. 

805 

806 Returns 

807 ------- 

808 abs : `ResourcePath` 

809 Absolute URI. For non-schemeless URIs this always returns itself. 

810 Schemeless URIs are upgraded to file URIs. 

811 """ 

812 return self 

813 

814 def _as_local(self) -> tuple[str, bool]: 

815 """Return the location of the (possibly remote) resource as local file. 

816 

817 This is a helper function for `as_local` context manager. 

818 

819 Returns 

820 ------- 

821 path : `str` 

822 If this is a remote resource, it will be a copy of the resource 

823 on the local file system, probably in a temporary directory. 

824 For a local resource this should be the actual path to the 

825 resource. 

826 is_temporary : `bool` 

827 Indicates if the local path is a temporary file or not. 

828 """ 

829 raise NotImplementedError() 

830 

831 @contextlib.contextmanager 

832 def as_local(self) -> Iterator[ResourcePath]: 

833 """Return the location of the (possibly remote) resource as local file. 

834 

835 Yields 

836 ------ 

837 local : `ResourcePath` 

838 If this is a remote resource, it will be a copy of the resource 

839 on the local file system, probably in a temporary directory. 

840 For a local resource this should be the actual path to the 

841 resource. 

842 

843 Notes 

844 ----- 

845 The context manager will automatically delete any local temporary 

846 file. 

847 

848 Examples 

849 -------- 

850 Should be used as a context manager: 

851 

852 .. code-block:: py 

853 

854 with uri.as_local() as local: 

855 ospath = local.ospath 

856 """ 

857 if self.dirLike: 

858 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.") 

859 local_src, is_temporary = self._as_local() 

860 local_uri = ResourcePath(local_src, isTemporary=is_temporary) 

861 

862 try: 

863 yield local_uri 

864 finally: 

865 # The caller might have relocated the temporary file. 

866 # Do not ever delete if the temporary matches self 

867 # (since it may have been that a temporary file was made local 

868 # but already was local). 

869 if self != local_uri and is_temporary and local_uri.exists(): 

870 local_uri.remove() 

871 

872 @classmethod 

873 @contextlib.contextmanager 

874 def temporary_uri( 

875 cls, prefix: ResourcePath | None = None, suffix: str | None = None 

876 ) -> Iterator[ResourcePath]: 

877 """Create a temporary file-like URI. 

878 

879 Parameters 

880 ---------- 

881 prefix : `ResourcePath`, optional 

882 Prefix to use. Without this the path will be formed as a local 

883 file URI in a temporary directory. Ensuring that the prefix 

884 location exists is the responsibility of the caller. 

885 suffix : `str`, optional 

886 A file suffix to be used. The ``.`` should be included in this 

887 suffix. 

888 

889 Yields 

890 ------ 

891 uri : `ResourcePath` 

892 The temporary URI. Will be removed when the context is completed. 

893 """ 

894 use_tempdir = False 

895 if prefix is None: 

896 prefix = ResourcePath(tempfile.mkdtemp(), forceDirectory=True, isTemporary=True) 

897 # Record that we need to delete this directory. Can not rely 

898 # on isTemporary flag since an external prefix may have that 

899 # set as well. 

900 use_tempdir = True 

901 

902 # Need to create a randomized file name. For consistency do not 

903 # use mkstemp for local and something else for remote. Additionally 

904 # this method does not create the file to prevent name clashes. 

905 characters = "abcdefghijklmnopqrstuvwxyz0123456789_" 

906 rng = Random() 

907 tempname = "".join(rng.choice(characters) for _ in range(16)) 

908 if suffix: 

909 tempname += suffix 

910 temporary_uri = prefix.join(tempname, isTemporary=True) 

911 if temporary_uri.dirLike: 

912 # If we had a safe way to clean up a remote temporary directory, we 

913 # could support this. 

914 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.") 

915 try: 

916 yield temporary_uri 

917 finally: 

918 if use_tempdir: 

919 shutil.rmtree(prefix.ospath, ignore_errors=True) 

920 else: 

921 with contextlib.suppress(FileNotFoundError): 

922 # It's okay if this does not work because the user removed 

923 # the file. 

924 temporary_uri.remove() 

925 

926 def read(self, size: int = -1) -> bytes: 

927 """Open the resource and return the contents in bytes. 

928 

929 Parameters 

930 ---------- 

931 size : `int`, optional 

932 The number of bytes to read. Negative or omitted indicates 

933 that all data should be read. 

934 """ 

935 raise NotImplementedError() 

936 

937 def write(self, data: bytes, overwrite: bool = True) -> None: 

938 """Write the supplied bytes to the new resource. 

939 

940 Parameters 

941 ---------- 

942 data : `bytes` 

943 The bytes to write to the resource. The entire contents of the 

944 resource will be replaced. 

945 overwrite : `bool`, optional 

946 If `True` the resource will be overwritten if it exists. Otherwise 

947 the write will fail. 

948 """ 

949 raise NotImplementedError() 

950 

951 def mkdir(self) -> None: 

952 """For a dir-like URI, create the directory resource if needed.""" 

953 raise NotImplementedError() 

954 

955 def isdir(self) -> bool: 

956 """Return True if this URI looks like a directory, else False.""" 

957 return self.dirLike 

958 

959 def size(self) -> int: 

960 """For non-dir-like URI, return the size of the resource. 

961 

962 Returns 

963 ------- 

964 sz : `int` 

965 The size in bytes of the resource associated with this URI. 

966 Returns 0 if dir-like. 

967 """ 

968 raise NotImplementedError() 

969 

970 def __str__(self) -> str: 

971 """Convert the URI to its native string form.""" 

972 return self.geturl() 

973 

974 def __repr__(self) -> str: 

975 """Return string representation suitable for evaluation.""" 

976 return f'ResourcePath("{self.geturl()}")' 

977 

978 def __eq__(self, other: Any) -> bool: 

979 """Compare supplied object with this `ResourcePath`.""" 

980 if not isinstance(other, ResourcePath): 

981 return NotImplemented 

982 return self.geturl() == other.geturl() 

983 

984 def __hash__(self) -> int: 

985 """Return hash of this object.""" 

986 return hash(str(self)) 

987 

988 def __lt__(self, other: ResourcePath) -> bool: 

989 return self.geturl() < other.geturl() 

990 

991 def __le__(self, other: ResourcePath) -> bool: 

992 return self.geturl() <= other.geturl() 

993 

994 def __gt__(self, other: ResourcePath) -> bool: 

995 return self.geturl() > other.geturl() 

996 

997 def __ge__(self, other: ResourcePath) -> bool: 

998 return self.geturl() >= other.geturl() 

999 

1000 def __copy__(self) -> ResourcePath: 

1001 """Copy constructor. 

1002 

1003 Object is immutable so copy can return itself. 

1004 """ 

1005 # Implement here because the __new__ method confuses things 

1006 return self 

1007 

1008 def __deepcopy__(self, memo: Any) -> ResourcePath: 

1009 """Deepcopy the object. 

1010 

1011 Object is immutable so copy can return itself. 

1012 """ 

1013 # Implement here because the __new__ method confuses things 

1014 return self 

1015 

1016 def __getnewargs__(self) -> tuple: