Coverage for python/lsst/resources/_resourcePath.py: 21%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14import concurrent.futures 

15import contextlib 

16import copy 

17import io 

18import locale 

19import logging 

20import os 

21import posixpath 

22import re 

23import shutil 

24import tempfile 

25import urllib.parse 

26from pathlib import Path, PurePath, PurePosixPath 

27from random import Random 

28 

29__all__ = ("ResourcePath", "ResourcePathExpression") 

30 

31from typing import ( 

32 TYPE_CHECKING, 

33 Any, 

34 Dict, 

35 Iterable, 

36 Iterator, 

37 List, 

38 Literal, 

39 Optional, 

40 Tuple, 

41 Type, 

42 Union, 

43 overload, 

44) 

45 

46from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol 

47 

48if TYPE_CHECKING: 

49 from .utils import TransactionProtocol 

50 

51 

52log = logging.getLogger(__name__) 

53 

54# Regex for looking for URI escapes 

55ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

56 

57# Precomputed escaped hash 

58ESCAPED_HASH = urllib.parse.quote("#") 

59 

60# Maximum number of worker threads for parallelized operations. 

61# If greater than 10, be aware that this number has to be consistent 

62# with connection pool sizing (for example in urllib3). 

63MAX_WORKERS = 10 

64 

65 

66ResourcePathExpression = Union[str, urllib.parse.ParseResult, "ResourcePath", Path] 

67"""Type-annotation alias for objects that can be coerced to ResourcePath. 

68""" 

69 

70 

71class ResourcePath: 

72 """Convenience wrapper around URI parsers. 

73 

74 Provides access to URI components and can convert file 

75 paths into absolute path URIs. Scheme-less URIs are treated as if 

76 they are local file system paths and are converted to absolute URIs. 

77 

78 A specialist subclass is created for each supported URI scheme. 

79 

80 Parameters 

81 ---------- 

82 uri : `str`, `Path`, `urllib.parse.ParseResult`, or `ResourcePath`. 

83 URI in string form. Can be scheme-less if referring to a relative 

84 path or an absolute path on the local file system. 

85 root : `str` or `ResourcePath`, optional 

86 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

87 use this as the root. Must be absolute. If `None` the current 

88 working directory will be used. Can be any supported URI scheme. 

89 Not used if ``forceAbsolute`` is `False`. 

90 forceAbsolute : `bool`, optional 

91 If `True`, scheme-less relative URI will be converted to an absolute 

92 path using a ``file`` scheme. If `False` scheme-less URI will remain 

93 scheme-less and will not be updated to ``file`` or absolute path unless 

94 it is already an absolute path, in which case it will be updated to 

95 a ``file`` scheme. 

96 forceDirectory: `bool`, optional 

97 If `True` forces the URI to end with a separator, otherwise given URI 

98 is interpreted as is. 

99 isTemporary : `bool`, optional 

100 If `True` indicates that this URI points to a temporary resource. 

101 The default is `False`, unless ``uri`` is already a `ResourcePath` 

102 instance and ``uri.isTemporary is True``. 

103 

104 Notes 

105 ----- 

106 A non-standard URI of the form ``file:dir/file.txt`` is always converted 

107 to an absolute ``file`` URI. 

108 """ 

109 

110 _pathLib: Type[PurePath] = PurePosixPath 

111 """Path library to use for this scheme.""" 

112 

113 _pathModule = posixpath 

114 """Path module to use for this scheme.""" 

115 

116 transferModes: Tuple[str, ...] = ("copy", "auto", "move") 

117 """Transfer modes supported by this implementation. 

118 

119 Move is special in that it is generally a copy followed by an unlink. 

120 Whether that unlink works depends critically on whether the source URI 

121 implements unlink. If it does not the move will be reported as a failure. 

122 """ 

123 

124 transferDefault: str = "copy" 

125 """Default mode to use for transferring if ``auto`` is specified.""" 

126 

127 quotePaths = True 

128 """True if path-like elements modifying a URI should be quoted. 

129 

130 All non-schemeless URIs have to internally use quoted paths. Therefore 

131 if a new file name is given (e.g. to updatedFile or join) a decision must 

132 be made whether to quote it to be consistent. 

133 """ 

134 

135 isLocal = False 

136 """If `True` this URI refers to a local file.""" 

137 

138 # This is not an ABC with abstract methods because the __new__ being 

139 # a factory confuses mypy such that it assumes that every constructor 

140 # returns a ResourcePath and then determines that all the abstract methods 

141 # are still abstract. If they are not marked abstract but just raise 

142 # mypy is fine with it. 

143 

144 # mypy is confused without these 

145 _uri: urllib.parse.ParseResult 

146 isTemporary: bool 

147 dirLike: bool 

148 

149 def __new__( 

150 cls, 

151 uri: ResourcePathExpression, 

152 root: Optional[Union[str, ResourcePath]] = None, 

153 forceAbsolute: bool = True, 

154 forceDirectory: bool = False, 

155 isTemporary: Optional[bool] = None, 

156 ) -> ResourcePath: 

157 """Create and return new specialist ResourcePath subclass.""" 

158 parsed: urllib.parse.ParseResult 

159 dirLike: bool = False 

160 subclass: Optional[Type[ResourcePath]] = None 

161 

162 # Force root to be a ResourcePath -- this simplifies downstream 

163 # code. 

164 if root is None: 

165 root_uri = None 

166 elif isinstance(root, str): 

167 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True) 

168 else: 

169 root_uri = root 

170 

171 if isinstance(uri, os.PathLike): 

172 uri = str(uri) 

173 

174 # Record if we need to post process the URI components 

175 # or if the instance is already fully configured 

176 if isinstance(uri, str): 

177 # Since local file names can have special characters in them 

178 # we need to quote them for the parser but we can unquote 

179 # later. Assume that all other URI schemes are quoted. 

180 # Since sometimes people write file:/a/b and not file:///a/b 

181 # we should not quote in the explicit case of file: 

182 if "://" not in uri and not uri.startswith("file:"): 

183 if ESCAPES_RE.search(uri): 

184 log.warning("Possible double encoding of %s", uri) 

185 else: 

186 uri = urllib.parse.quote(uri) 

187 # Special case hash since we must support fragments 

188 # even in schemeless URIs -- although try to only replace 

189 # them in file part and not directory part 

190 if ESCAPED_HASH in uri: 

191 dirpos = uri.rfind("/") 

192 # Do replacement after this / 

193 uri = uri[: dirpos + 1] + uri[dirpos + 1 :].replace(ESCAPED_HASH, "#") 

194 

195 parsed = urllib.parse.urlparse(uri) 

196 elif isinstance(uri, urllib.parse.ParseResult): 

197 parsed = copy.copy(uri) 

198 # If we are being instantiated with a subclass, rather than 

199 # ResourcePath, ensure that that subclass is used directly. 

200 # This could lead to inconsistencies if this constructor 

201 # is used externally outside of the ResourcePath.replace() method. 

202 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt")) 

203 # will be a problem. 

204 # This is needed to prevent a schemeless absolute URI become 

205 # a file URI unexpectedly when calling updatedFile or 

206 # updatedExtension 

207 if cls is not ResourcePath: 

208 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory) 

209 subclass = cls 

210 

211 elif isinstance(uri, ResourcePath): 

212 # Since ResourcePath is immutable we can return the argument 

213 # unchanged if it already agrees with forceDirectory, isTemporary, 

214 # and forceAbsolute. 

215 # We invoke __new__ again with str(self) to add a scheme for 

216 # forceAbsolute, but for the others that seems more likely to paper 

217 # over logic errors than do something useful, so we just raise. 

218 if forceDirectory and not uri.dirLike: 

219 raise RuntimeError( 

220 f"{uri} is already a file-like ResourcePath; cannot force it to directory." 

221 ) 

222 if isTemporary is not None and isTemporary is not uri.isTemporary: 

223 raise RuntimeError( 

224 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} " 

225 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}." 

226 ) 

227 if forceAbsolute and not uri.scheme: 

228 return ResourcePath( 

229 str(uri), 

230 root=root, 

231 forceAbsolute=True, 

232 forceDirectory=uri.dirLike, 

233 isTemporary=uri.isTemporary, 

234 ) 

235 return uri 

236 else: 

237 raise ValueError( 

238 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'" 

239 ) 

240 

241 if subclass is None: 

242 # Work out the subclass from the URI scheme 

243 if not parsed.scheme: 

244 # Root may be specified as a ResourcePath that overrides 

245 # the schemeless determination. 

246 if ( 

247 root_uri is not None 

248 and root_uri.scheme != "file" # file scheme has different code path 

249 and not parsed.path.startswith("/") # Not already absolute path 

250 ): 

251 if not root_uri.dirLike: 

252 raise ValueError( 

253 f"Root URI ({root}) was not a directory so can not be joined with" 

254 f" path {parsed.path!r}" 

255 ) 

256 # If root is temporary or this schemeless is temporary we 

257 # assume this URI is temporary. 

258 isTemporary = isTemporary or root_uri.isTemporary 

259 joined = root_uri.join( 

260 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary 

261 ) 

262 

263 # Rather than returning this new ResourcePath directly we 

264 # instead extract the path and the scheme and adjust the 

265 # URI we were given -- we need to do this to preserve 

266 # fragments since join() will drop them. 

267 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc) 

268 subclass = type(joined) 

269 

270 # Clear the root parameter to indicate that it has 

271 # been applied already. 

272 root_uri = None 

273 else: 

274 from .schemeless import SchemelessResourcePath 

275 

276 subclass = SchemelessResourcePath 

277 elif parsed.scheme == "file": 

278 from .file import FileResourcePath 

279 

280 subclass = FileResourcePath 

281 elif parsed.scheme == "s3": 

282 from .s3 import S3ResourcePath 

283 

284 subclass = S3ResourcePath 

285 elif parsed.scheme.startswith("http"): 

286 from .http import HttpResourcePath 

287 

288 subclass = HttpResourcePath 

289 elif parsed.scheme == "gs": 

290 from .gs import GSResourcePath 

291 

292 subclass = GSResourcePath 

293 elif parsed.scheme == "resource": 

294 # Rules for scheme names disallow pkg_resource 

295 from .packageresource import PackageResourcePath 

296 

297 subclass = PackageResourcePath 

298 elif parsed.scheme == "mem": 

299 # in-memory datastore object 

300 from .mem import InMemoryResourcePath 

301 

302 subclass = InMemoryResourcePath 

303 else: 

304 raise NotImplementedError( 

305 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" 

306 ) 

307 

308 parsed, dirLike = subclass._fixupPathUri( 

309 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory 

310 ) 

311 

312 # It is possible for the class to change from schemeless 

313 # to file so handle that 

314 if parsed.scheme == "file": 

315 from .file import FileResourcePath 

316 

317 subclass = FileResourcePath 

318 

319 # Now create an instance of the correct subclass and set the 

320 # attributes directly 

321 self = object.__new__(subclass) 

322 self._uri = parsed 

323 self.dirLike = dirLike 

324 if isTemporary is None: 

325 isTemporary = False 

326 self.isTemporary = isTemporary 

327 return self 

328 

329 @property 

330 def scheme(self) -> str: 

331 """Return the URI scheme. 

332 

333 Notes 

334 ----- 

335 (``://`` is not part of the scheme). 

336 """ 

337 return self._uri.scheme 

338 

339 @property 

340 def netloc(self) -> str: 

341 """Return the URI network location.""" 

342 return self._uri.netloc 

343 

344 @property 

345 def path(self) -> str: 

346 """Return the path component of the URI.""" 

347 return self._uri.path 

348 

349 @property 

350 def unquoted_path(self) -> str: 

351 """Return path component of the URI with any URI quoting reversed.""" 

352 return urllib.parse.unquote(self._uri.path) 

353 

354 @property 

355 def ospath(self) -> str: 

356 """Return the path component of the URI localized to current OS.""" 

357 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

358 

359 @property 

360 def relativeToPathRoot(self) -> str: 

361 """Return path relative to network location. 

362 

363 Effectively, this is the path property with posix separator stripped 

364 from the left hand side of the path. 

365 

366 Always unquotes. 

367 """ 

368 p = self._pathLib(self.path) 

369 relToRoot = str(p.relative_to(p.root)) 

370 if self.dirLike and not relToRoot.endswith("/"): 

371 relToRoot += "/" 

372 return urllib.parse.unquote(relToRoot) 

373 

374 @property 

375 def is_root(self) -> bool: 

376 """Return whether this URI points to the root of the network location. 

377 

378 This means that the path components refers to the top level. 

379 """ 

380 relpath = self.relativeToPathRoot 

381 if relpath == "./": 

382 return True 

383 return False 

384 

385 @property 

386 def fragment(self) -> str: 

387 """Return the fragment component of the URI.""" 

388 return self._uri.fragment 

389 

390 @property 

391 def params(self) -> str: 

392 """Return any parameters included in the URI.""" 

393 return self._uri.params 

394 

395 @property 

396 def query(self) -> str: 

397 """Return any query strings included in the URI.""" 

398 return self._uri.query 

399 

400 def geturl(self) -> str: 

401 """Return the URI in string form. 

402 

403 Returns 

404 ------- 

405 url : `str` 

406 String form of URI. 

407 """ 

408 return self._uri.geturl() 

409 

410 def root_uri(self) -> ResourcePath: 

411 """Return the base root URI. 

412 

413 Returns 

414 ------- 

415 uri : `ResourcePath` 

416 root URI. 

417 """ 

418 return self.replace(path="", forceDirectory=True) 

419 

420 def split(self) -> Tuple[ResourcePath, str]: 

421 """Split URI into head and tail. 

422 

423 Returns 

424 ------- 

425 head: `ResourcePath` 

426 Everything leading up to tail, expanded and normalized as per 

427 ResourcePath rules. 

428 tail : `str` 

429 Last `self.path` component. Tail will be empty if path ends on a 

430 separator. Tail will never contain separators. It will be 

431 unquoted. 

432 

433 Notes 

434 ----- 

435 Equivalent to `os.path.split()` where head preserves the URI 

436 components. 

437 """ 

438 head, tail = self._pathModule.split(self.path) 

439 headuri = self._uri._replace(path=head) 

440 

441 # The file part should never include quoted metacharacters 

442 tail = urllib.parse.unquote(tail) 

443 

444 # Schemeless is special in that it can be a relative path 

445 # We need to ensure that it stays that way. All other URIs will 

446 # be absolute already. 

447 forceAbsolute = self._pathModule.isabs(self.path) 

448 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

449 

450 def basename(self) -> str: 

451 """Return the base name, last element of path, of the URI. 

452 

453 Returns 

454 ------- 

455 tail : `str` 

456 Last part of the path attribute. Trail will be empty if path ends 

457 on a separator. 

458 

459 Notes 

460 ----- 

461 If URI ends on a slash returns an empty string. This is the second 

462 element returned by `split()`. 

463 

464 Equivalent of `os.path.basename()``. 

465 """ 

466 return self.split()[1] 

467 

468 def dirname(self) -> ResourcePath: 

469 """Return the directory component of the path as a new `ResourcePath`. 

470 

471 Returns 

472 ------- 

473 head : `ResourcePath` 

474 Everything except the tail of path attribute, expanded and 

475 normalized as per ResourcePath rules. 

476 

477 Notes 

478 ----- 

479 Equivalent of `os.path.dirname()`. 

480 """ 

481 return self.split()[0] 

482 

483 def parent(self) -> ResourcePath: 

484 """Return a `ResourcePath` of the parent directory. 

485 

486 Returns 

487 ------- 

488 head : `ResourcePath` 

489 Everything except the tail of path attribute, expanded and 

490 normalized as per `ResourcePath` rules. 

491 

492 Notes 

493 ----- 

494 For a file-like URI this will be the same as calling `dirname()`. 

495 """ 

496 # When self is file-like, return self.dirname() 

497 if not self.dirLike: 

498 return self.dirname() 

499 # When self is dir-like, return its parent directory, 

500 # regardless of the presence of a trailing separator 

501 originalPath = self._pathLib(self.path) 

502 parentPath = originalPath.parent 

503 return self.replace(path=str(parentPath), forceDirectory=True) 

504 

505 def replace(self, forceDirectory: bool = False, isTemporary: bool = False, **kwargs: Any) -> ResourcePath: 

506 """Return new `ResourcePath` with specified components replaced. 

507 

508 Parameters 

509 ---------- 

510 forceDirectory : `bool`, optional 

511 Parameter passed to ResourcePath constructor to force this 

512 new URI to be dir-like. 

513 isTemporary : `bool`, optional 

514 Indicate that the resulting URI is temporary resource. 

515 **kwargs 

516 Components of a `urllib.parse.ParseResult` that should be 

517 modified for the newly-created `ResourcePath`. 

518 

519 Returns 

520 ------- 

521 new : `ResourcePath` 

522 New `ResourcePath` object with updated values. 

523 

524 Notes 

525 ----- 

526 Does not, for now, allow a change in URI scheme. 

527 """ 

528 # Disallow a change in scheme 

529 if "scheme" in kwargs: 

530 raise ValueError(f"Can not use replace() method to change URI scheme for {self}") 

531 return self.__class__( 

532 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary 

533 ) 

534 

535 def updatedFile(self, newfile: str) -> ResourcePath: 

536 """Return new URI with an updated final component of the path. 

537 

538 Parameters 

539 ---------- 

540 newfile : `str` 

541 File name with no path component. 

542 

543 Returns 

544 ------- 

545 updated : `ResourcePath` 

546 

547 Notes 

548 ----- 

549 Forces the ResourcePath.dirLike attribute to be false. The new file 

550 path will be quoted if necessary. 

551 """ 

552 if self.quotePaths: 

553 newfile = urllib.parse.quote(newfile) 

554 dir, _ = self._pathModule.split(self.path) 

555 newpath = self._pathModule.join(dir, newfile) 

556 

557 updated = self.replace(path=newpath) 

558 updated.dirLike = False 

559 return updated 

560 

561 def updatedExtension(self, ext: Optional[str]) -> ResourcePath: 

562 """Return a new `ResourcePath` with updated file extension. 

563 

564 All file extensions are replaced. 

565 

566 Parameters 

567 ---------- 

568 ext : `str` or `None` 

569 New extension. If an empty string is given any extension will 

570 be removed. If `None` is given there will be no change. 

571 

572 Returns 

573 ------- 

574 updated : `ResourcePath` 

575 URI with the specified extension. Can return itself if 

576 no extension was specified. 

577 """ 

578 if ext is None: 

579 return self 

580 

581 # Get the extension 

582 current = self.getExtension() 

583 

584 # Nothing to do if the extension already matches 

585 if current == ext: 

586 return self 

587 

588 # Remove the current extension from the path 

589 # .fits.gz counts as one extension do not use os.path.splitext 

590 path = self.path 

591 if current: 

592 path = path[: -len(current)] 

593 

594 # Ensure that we have a leading "." on file extension (and we do not 

595 # try to modify the empty string) 

596 if ext and not ext.startswith("."): 

597 ext = "." + ext 

598 

599 return self.replace(path=path + ext) 

600 

601 def getExtension(self) -> str: 

602 """Return the file extension(s) associated with this URI path. 

603 

604 Returns 

605 ------- 

606 ext : `str` 

607 The file extension (including the ``.``). Can be empty string 

608 if there is no file extension. Usually returns only the last 

609 file extension unless there is a special extension modifier 

610 indicating file compression, in which case the combined 

611 extension (e.g. ``.fits.gz``) will be returned. 

612 """ 

613 special = {".gz", ".bz2", ".xz", ".fz"} 

614 

615 # Get the file part of the path so as not to be confused by 

616 # "." in directory names. 

617 basename = self.basename() 

618 extensions = self._pathLib(basename).suffixes 

619 

620 if not extensions: 

621 return "" 

622 

623 ext = extensions.pop() 

624 

625 # Multiple extensions, decide whether to include the final two 

626 if extensions and ext in special: 

627 ext = f"{extensions[-1]}{ext}" 

628 

629 return ext 

630 

631 def join( 

632 self, path: Union[str, ResourcePath], isTemporary: Optional[bool] = None, forceDirectory: bool = False 

633 ) -> ResourcePath: 

634 """Return new `ResourcePath` with additional path components. 

635 

636 Parameters 

637 ---------- 

638 path : `str`, `ResourcePath` 

639 Additional file components to append to the current URI. Assumed 

640 to include a file at the end. Will be quoted depending on the 

641 associated URI scheme. If the path looks like a URI with a scheme 

642 referring to an absolute location, it will be returned 

643 directly (matching the behavior of `os.path.join()`). It can 

644 also be a `ResourcePath`. 

645 isTemporary : `bool`, optional 

646 Indicate that the resulting URI represents a temporary resource. 

647 Default is ``self.isTemporary``. 

648 forceDirectory : `bool`, optional 

649 If `True` forces the URI to end with a separator, otherwise given 

650 URI is interpreted as is. 

651 

652 Returns 

653 ------- 

654 new : `ResourcePath` 

655 New URI with any file at the end replaced with the new path 

656 components. 

657 

658 Notes 

659 ----- 

660 Schemeless URIs assume local path separator but all other URIs assume 

661 POSIX separator if the supplied path has directory structure. It 

662 may be this never becomes a problem but datastore templates assume 

663 POSIX separator is being used. 

664 

665 If an absolute `ResourcePath` is given for ``path`` is is assumed that 

666 this should be returned directly. Giving a ``path`` of an absolute 

667 scheme-less URI is not allowed for safety reasons as it may indicate 

668 a mistake in the calling code. 

669 

670 Raises 

671 ------ 

672 ValueError 

673 Raised if the ``path`` is an absolute scheme-less URI. In that 

674 situation it is unclear whether the intent is to return a 

675 ``file`` URI or it was a mistake and a relative scheme-less URI 

676 was meant. 

677 RuntimeError 

678 Raised if this attempts to join a temporary URI to a non-temporary 

679 URI. 

680 """ 

681 if isTemporary is None: 

682 isTemporary = self.isTemporary 

683 elif not isTemporary and self.isTemporary: 

684 raise RuntimeError("Cannot join temporary URI to non-temporary URI.") 

685 # If we have a full URI in path we will use it directly 

686 # but without forcing to absolute so that we can trap the 

687 # expected option of relative path. 

688 path_uri = ResourcePath( 

689 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary 

690 ) 

691 if path_uri.scheme: 

692 # Check for scheme so can distinguish explicit URIs from 

693 # absolute scheme-less URIs. 

694 return path_uri 

695 

696 if path_uri.isabs(): 

697 # Absolute scheme-less path. 

698 raise ValueError(f"Can not join absolute scheme-less {path_uri!r} to another URI.") 

699 

700 # If this was originally a ResourcePath extract the unquoted path from 

701 # it. Otherwise we use the string we were given to allow "#" to appear 

702 # in the filename if given as a plain string. 

703 if not isinstance(path, str): 

704 path = path_uri.unquoted_path 

705 

706 new = self.dirname() # By definition a directory URI 

707 

708 # new should be asked about quoting, not self, since dirname can 

709 # change the URI scheme for schemeless -> file 

710 if new.quotePaths: 

711 path = urllib.parse.quote(path) 

712 

713 newpath = self._pathModule.normpath(self._pathModule.join(new.path, path)) 

714 

715 # normpath can strip trailing / so we force directory if the supplied 

716 # path ended with a / 

717 return new.replace( 

718 path=newpath, 

719 forceDirectory=(forceDirectory or path.endswith(self._pathModule.sep)), 

720 isTemporary=isTemporary, 

721 ) 

722 

723 def relative_to(self, other: ResourcePath) -> Optional[str]: 

724 """Return the relative path from this URI to the other URI. 

725 

726 Parameters 

727 ---------- 

728 other : `ResourcePath` 

729 URI to use to calculate the relative path. Must be a parent 

730 of this URI. 

731 

732 Returns 

733 ------- 

734 subpath : `str` 

735 The sub path of this URI relative to the supplied other URI. 

736 Returns `None` if there is no parent child relationship. 

737 Scheme and netloc must match. 

738 """ 

739 # Scheme-less absolute other is treated as if it's a file scheme. 

740 # Scheme-less relative other can only return non-None if self 

741 # is also scheme-less relative and that is handled specifically 

742 # in a subclass. 

743 if not other.scheme and other.isabs(): 

744 other = other.abspath() 

745 

746 # Scheme-less self is handled elsewhere. 

747 if self.scheme != other.scheme: 

748 return None 

749 if self.netloc != other.netloc: 

750 # Special case for localhost vs empty string. 

751 # There can be many variants of localhost. 

752 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"} 

753 if not {self.netloc, other.netloc}.issubset(local_netlocs): 

754 return None 

755 

756 enclosed_path = self._pathLib(self.relativeToPathRoot) 

757 parent_path = other.relativeToPathRoot 

758 subpath: Optional[str] 

759 try: 

760 subpath = str(enclosed_path.relative_to(parent_path)) 

761 except ValueError: 

762 subpath = None 

763 else: 

764 subpath = urllib.parse.unquote(subpath) 

765 return subpath 

766 

767 def exists(self) -> bool: 

768 """Indicate that the resource is available. 

769 

770 Returns 

771 ------- 

772 exists : `bool` 

773 `True` if the resource exists. 

774 """ 

775 raise NotImplementedError() 

776 

777 @classmethod 

778 def mexists(cls, uris: Iterable[ResourcePath]) -> Dict[ResourcePath, bool]: 

779 """Check for existence of multiple URIs at once. 

780 

781 Parameters 

782 ---------- 

783 uris : iterable of `ResourcePath` 

784 The URIs to test. 

785 

786 Returns 

787 ------- 

788 existence : `dict` of [`ResourcePath`, `bool`] 

789 Mapping of original URI to boolean indicating existence. 

790 """ 

791 exists_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 

792 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris} 

793 

794 results: Dict[ResourcePath, bool] = {} 

795 for future in concurrent.futures.as_completed(future_exists): 

796 uri = future_exists[future] 

797 try: 

798 exists = future.result() 

799 except Exception: 

800 exists = False 

801 results[uri] = exists 

802 return results 

803 

804 def remove(self) -> None: 

805 """Remove the resource.""" 

806 raise NotImplementedError() 

807 

808 def isabs(self) -> bool: 

809 """Indicate that the resource is fully specified. 

810 

811 For non-schemeless URIs this is always true. 

812 

813 Returns 

814 ------- 

815 isabs : `bool` 

816 `True` in all cases except schemeless URI. 

817 """ 

818 return True 

819 

820 def abspath(self) -> ResourcePath: 

821 """Return URI using an absolute path. 

822 

823 Returns 

824 ------- 

825 abs : `ResourcePath` 

826 Absolute URI. For non-schemeless URIs this always returns itself. 

827 Schemeless URIs are upgraded to file URIs. 

828 """ 

829 return self 

830 

831 def _as_local(self) -> Tuple[str, bool]: 

832 """Return the location of the (possibly remote) resource as local file. 

833 

834 This is a helper function for `as_local` context manager. 

835 

836 Returns 

837 ------- 

838 path : `str` 

839 If this is a remote resource, it will be a copy of the resource 

840 on the local file system, probably in a temporary directory. 

841 For a local resource this should be the actual path to the 

842 resource. 

843 is_temporary : `bool` 

844 Indicates if the local path is a temporary file or not. 

845 """ 

846 raise NotImplementedError() 

847 

848 @contextlib.contextmanager 

849 def as_local(self) -> Iterator[ResourcePath]: 

850 """Return the location of the (possibly remote) resource as local file. 

851 

852 Yields 

853 ------ 

854 local : `ResourcePath` 

855 If this is a remote resource, it will be a copy of the resource 

856 on the local file system, probably in a temporary directory. 

857 For a local resource this should be the actual path to the 

858 resource. 

859 

860 Notes 

861 ----- 

862 The context manager will automatically delete any local temporary 

863 file. 

864 

865 Examples 

866 -------- 

867 Should be used as a context manager: 

868 

869 .. code-block:: py 

870 

871 with uri.as_local() as local: 

872 ospath = local.ospath 

873 """ 

874 if self.dirLike: 

875 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.") 

876 local_src, is_temporary = self._as_local() 

877 local_uri = ResourcePath(local_src, isTemporary=is_temporary) 

878 

879 try: 

880 yield local_uri 

881 finally: 

882 # The caller might have relocated the temporary file. 

883 # Do not ever delete if the temporary matches self 

884 # (since it may have been that a temporary file was made local 

885 # but already was local). 

886 if self != local_uri and is_temporary and local_uri.exists(): 

887 local_uri.remove() 

888 

889 @classmethod 

890 @contextlib.contextmanager 

891 def temporary_uri( 

892 cls, prefix: Optional[ResourcePath] = None, suffix: Optional[str] = None 

893 ) -> Iterator[ResourcePath]: 

894 """Create a temporary file-like URI. 

895 

896 Parameters 

897 ---------- 

898 prefix : `ResourcePath`, optional 

899 Prefix to use. Without this the path will be formed as a local 

900 file URI in a temporary directory. Ensuring that the prefix 

901 location exists is the responsibility of the caller. 

902 suffix : `str`, optional 

903 A file suffix to be used. The ``.`` should be included in this 

904 suffix. 

905 

906 Yields 

907 ------ 

908 uri : `ResourcePath` 

909 The temporary URI. Will be removed when the context is completed. 

910 """ 

911 use_tempdir = False 

912 if prefix is None: 

913 prefix = ResourcePath(tempfile.mkdtemp(), forceDirectory=True, isTemporary=True) 

914 # Record that we need to delete this directory. Can not rely 

915 # on isTemporary flag since an external prefix may have that 

916 # set as well. 

917 use_tempdir = True 

918 

919 # Need to create a randomized file name. For consistency do not 

920 # use mkstemp for local and something else for remote. Additionally 

921 # this method does not create the file to prevent name clashes. 

922 characters = "abcdefghijklmnopqrstuvwxyz0123456789_" 

923 rng = Random() 

924 tempname = "".join(rng.choice(characters) for _ in range(16)) 

925 if suffix: 

926 tempname += suffix 

927 temporary_uri = prefix.join(tempname, isTemporary=True) 

928 if temporary_uri.dirLike: 

929 # If we had a safe way to clean up a remote temporary directory, we 

930 # could support this. 

931 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.") 

932 try: 

933 yield temporary_uri 

934 finally: 

935 if use_tempdir: 

936 shutil.rmtree(prefix.ospath, ignore_errors=True) 

937 else: 

938 try: 

939 # It's okay if this does not work because the user removed 

940 # the file. 

941 temporary_uri.remove() 

942 except FileNotFoundError: 

943 pass 

944 

945 def read(self, size: int = -1) -> bytes: 

946 """Open the resource and return the contents in bytes. 

947 

948 Parameters 

949 ---------- 

950 size : `int`, optional 

951 The number of bytes to read. Negative or omitted indicates 

952 that all data should be read. 

953 """ 

954 raise NotImplementedError() 

955 

956 def write(self, data: bytes, overwrite: bool = True) -> None: 

957 """Write the supplied bytes to the new resource. 

958 

959 Parameters 

960 ---------- 

961 data : `bytes` 

962 The bytes to write to the resource. The entire contents of the 

963 resource will be replaced. 

964 overwrite : `bool`, optional 

965 If `True` the resource will be overwritten if it exists. Otherwise 

966 the write will fail. 

967 """ 

968 raise NotImplementedError() 

969 

970 def mkdir(self) -> None: 

971 """For a dir-like URI, create the directory resource if needed.""" 

972 raise NotImplementedError() 

973 

974 def isdir(self) -> bool: 

975 """Return True if this URI looks like a directory, else False.""" 

976 return self.dirLike 

977 

978 def size(self) -> int: 

979 """For non-dir-like URI, return the size of the resource. 

980 

981 Returns 

982 ------- 

983 sz : `int` 

984 The size in bytes of the resource associated with this URI. 

985 Returns 0 if dir-like. 

986 """ 

987 raise NotImplementedError() 

988 

989 def __str__(self) -> str: 

990 """Convert the URI to its native string form.""" 

991 return self.geturl() 

992 

993 def __repr__(self) -> str: 

994 """Return string representation suitable for evaluation.""" 

995 return f'ResourcePath("{self.geturl()}")' 

996 

997 def __eq__(self, other: Any) -> bool: 

998 """Compare supplied object with this `ResourcePath`.""" 

999 if not isinstance(other, ResourcePath): 

1000 return NotImplemented 

1001 return self.geturl() == other.geturl() 

1002 

1003 def __hash__(self) -> int: 

1004 """Return hash of this object.""" 

1005 return hash(str(self)) 

1006 

1007 def __lt__(self, other: ResourcePath) -> bool: 

1008 return self.geturl() < other.geturl() 

1009 

1010 def __le__(self, other: ResourcePath) -> bool: 

1011 return self.geturl() <= other.geturl() 

1012 

1013 def __gt__(self, other: ResourcePath) -> bool: 

1014 return self.geturl() > other.geturl() 

1015 

1016 def __ge__(self, other: ResourcePath) -> bool: 

1017 return self.geturl() >= other.geturl()