Coverage for python/lsst/resources/_resourcePath.py: 28%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14import concurrent.futures 

15import contextlib 

16import copy 

17import io 

18import locale 

19import logging 

20import os 

21import posixpath 

22import re 

23import shutil 

24import tempfile 

25import urllib.parse 

26from pathlib import Path, PurePath, PurePosixPath 

27from random import Random 

28 

29__all__ = ("ResourcePath", "ResourcePathExpression") 

30 

31from collections.abc import Iterable, Iterator 

32from typing import TYPE_CHECKING, Any, Literal, overload 

33 

34from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol 

35from .utils import ensure_directory_is_writeable 

36 

37if TYPE_CHECKING: 

38 from .utils import TransactionProtocol 

39 

40 

41log = logging.getLogger(__name__) 

42 

43# Regex for looking for URI escapes 

44ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

45 

46# Precomputed escaped hash 

47ESCAPED_HASH = urllib.parse.quote("#") 

48 

49# Maximum number of worker threads for parallelized operations. 

50# If greater than 10, be aware that this number has to be consistent 

51# with connection pool sizing (for example in urllib3). 

52MAX_WORKERS = 10 

53 

54 

55class ResourcePath: 

56 """Convenience wrapper around URI parsers. 

57 

58 Provides access to URI components and can convert file 

59 paths into absolute path URIs. Scheme-less URIs are treated as if 

60 they are local file system paths and are converted to absolute URIs. 

61 

62 A specialist subclass is created for each supported URI scheme. 

63 

64 Parameters 

65 ---------- 

66 uri : `str`, `pathlib.Path`, `urllib.parse.ParseResult`, or `ResourcePath`. 

67 URI in string form. Can be scheme-less if referring to a relative 

68 path or an absolute path on the local file system. 

69 root : `str` or `ResourcePath`, optional 

70 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

71 use this as the root. Must be absolute. If `None` the current 

72 working directory will be used. Can be any supported URI scheme. 

73 Not used if ``forceAbsolute`` is `False`. 

74 forceAbsolute : `bool`, optional 

75 If `True`, scheme-less relative URI will be converted to an absolute 

76 path using a ``file`` scheme. If `False` scheme-less URI will remain 

77 scheme-less and will not be updated to ``file`` or absolute path unless 

78 it is already an absolute path, in which case it will be updated to 

79 a ``file`` scheme. 

80 forceDirectory: `bool`, optional 

81 If `True` forces the URI to end with a separator, otherwise given URI 

82 is interpreted as is. 

83 isTemporary : `bool`, optional 

84 If `True` indicates that this URI points to a temporary resource. 

85 The default is `False`, unless ``uri`` is already a `ResourcePath` 

86 instance and ``uri.isTemporary is True``. 

87 

88 Notes 

89 ----- 

90 A non-standard URI of the form ``file:dir/file.txt`` is always converted 

91 to an absolute ``file`` URI. 

92 """ 

93 

94 _pathLib: type[PurePath] = PurePosixPath 

95 """Path library to use for this scheme.""" 

96 

97 _pathModule = posixpath 

98 """Path module to use for this scheme.""" 

99 

100 transferModes: tuple[str, ...] = ("copy", "auto", "move") 

101 """Transfer modes supported by this implementation. 

102 

103 Move is special in that it is generally a copy followed by an unlink. 

104 Whether that unlink works depends critically on whether the source URI 

105 implements unlink. If it does not the move will be reported as a failure. 

106 """ 

107 

108 transferDefault: str = "copy" 

109 """Default mode to use for transferring if ``auto`` is specified.""" 

110 

111 quotePaths = True 

112 """True if path-like elements modifying a URI should be quoted. 

113 

114 All non-schemeless URIs have to internally use quoted paths. Therefore 

115 if a new file name is given (e.g. to updatedFile or join) a decision must 

116 be made whether to quote it to be consistent. 

117 """ 

118 

119 isLocal = False 

120 """If `True` this URI refers to a local file.""" 

121 

122 # This is not an ABC with abstract methods because the __new__ being 

123 # a factory confuses mypy such that it assumes that every constructor 

124 # returns a ResourcePath and then determines that all the abstract methods 

125 # are still abstract. If they are not marked abstract but just raise 

126 # mypy is fine with it. 

127 

128 # mypy is confused without these 

129 _uri: urllib.parse.ParseResult 

130 isTemporary: bool 

131 dirLike: bool 

132 

133 def __new__( 

134 cls, 

135 uri: ResourcePathExpression, 

136 root: str | ResourcePath | None = None, 

137 forceAbsolute: bool = True, 

138 forceDirectory: bool = False, 

139 isTemporary: bool | None = None, 

140 ) -> ResourcePath: 

141 """Create and return new specialist ResourcePath subclass.""" 

142 parsed: urllib.parse.ParseResult 

143 dirLike: bool = False 

144 subclass: type[ResourcePath] | None = None 

145 

146 # Force root to be a ResourcePath -- this simplifies downstream 

147 # code. 

148 if root is None: 

149 root_uri = None 

150 elif isinstance(root, str): 

151 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True) 

152 else: 

153 root_uri = root 

154 

155 if isinstance(uri, os.PathLike): 

156 uri = str(uri) 

157 

158 # Record if we need to post process the URI components 

159 # or if the instance is already fully configured 

160 if isinstance(uri, str): 

161 # Since local file names can have special characters in them 

162 # we need to quote them for the parser but we can unquote 

163 # later. Assume that all other URI schemes are quoted. 

164 # Since sometimes people write file:/a/b and not file:///a/b 

165 # we should not quote in the explicit case of file: 

166 if "://" not in uri and not uri.startswith("file:"): 

167 if ESCAPES_RE.search(uri): 

168 log.warning("Possible double encoding of %s", uri) 

169 else: 

170 # Fragments are generally not encoded so we must search 

171 # for the fragment boundary ourselves. This is making 

172 # an assumption that the filename does not include a "#" 

173 # and also that there is no "/" in the fragment itself. 

174 to_encode = uri 

175 fragment = "" 

176 if "#" in uri: 

177 dirpos = uri.rfind("/") 

178 trailing = uri[dirpos + 1 :] 

179 hashpos = trailing.rfind("#") 

180 if hashpos != -1: 

181 fragment = trailing[hashpos:] 

182 to_encode = uri[: dirpos + hashpos + 1] 

183 

184 uri = urllib.parse.quote(to_encode) + fragment 

185 

186 parsed = urllib.parse.urlparse(uri) 

187 elif isinstance(uri, urllib.parse.ParseResult): 

188 parsed = copy.copy(uri) 

189 # If we are being instantiated with a subclass, rather than 

190 # ResourcePath, ensure that that subclass is used directly. 

191 # This could lead to inconsistencies if this constructor 

192 # is used externally outside of the ResourcePath.replace() method. 

193 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt")) 

194 # will be a problem. 

195 # This is needed to prevent a schemeless absolute URI become 

196 # a file URI unexpectedly when calling updatedFile or 

197 # updatedExtension 

198 if cls is not ResourcePath: 

199 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory) 

200 subclass = cls 

201 

202 elif isinstance(uri, ResourcePath): 

203 # Since ResourcePath is immutable we can return the argument 

204 # unchanged if it already agrees with forceDirectory, isTemporary, 

205 # and forceAbsolute. 

206 # We invoke __new__ again with str(self) to add a scheme for 

207 # forceAbsolute, but for the others that seems more likely to paper 

208 # over logic errors than do something useful, so we just raise. 

209 if forceDirectory and not uri.dirLike: 

210 raise RuntimeError( 

211 f"{uri} is already a file-like ResourcePath; cannot force it to directory." 

212 ) 

213 if isTemporary is not None and isTemporary is not uri.isTemporary: 

214 raise RuntimeError( 

215 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} " 

216 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}." 

217 ) 

218 if forceAbsolute and not uri.scheme: 

219 return ResourcePath( 

220 str(uri), 

221 root=root, 

222 forceAbsolute=True, 

223 forceDirectory=uri.dirLike, 

224 isTemporary=uri.isTemporary, 

225 ) 

226 return uri 

227 else: 

228 raise ValueError( 

229 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'" 

230 ) 

231 

232 if subclass is None: 

233 # Work out the subclass from the URI scheme 

234 if not parsed.scheme: 

235 # Root may be specified as a ResourcePath that overrides 

236 # the schemeless determination. 

237 if ( 

238 root_uri is not None 

239 and root_uri.scheme != "file" # file scheme has different code path 

240 and not parsed.path.startswith("/") # Not already absolute path 

241 ): 

242 if not root_uri.dirLike: 

243 raise ValueError( 

244 f"Root URI ({root}) was not a directory so can not be joined with" 

245 f" path {parsed.path!r}" 

246 ) 

247 # If root is temporary or this schemeless is temporary we 

248 # assume this URI is temporary. 

249 isTemporary = isTemporary or root_uri.isTemporary 

250 joined = root_uri.join( 

251 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary 

252 ) 

253 

254 # Rather than returning this new ResourcePath directly we 

255 # instead extract the path and the scheme and adjust the 

256 # URI we were given -- we need to do this to preserve 

257 # fragments since join() will drop them. 

258 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc) 

259 subclass = type(joined) 

260 

261 # Clear the root parameter to indicate that it has 

262 # been applied already. 

263 root_uri = None 

264 else: 

265 from .schemeless import SchemelessResourcePath 

266 

267 subclass = SchemelessResourcePath 

268 elif parsed.scheme == "file": 

269 from .file import FileResourcePath 

270 

271 subclass = FileResourcePath 

272 elif parsed.scheme == "s3": 

273 from .s3 import S3ResourcePath 

274 

275 subclass = S3ResourcePath 

276 elif parsed.scheme.startswith("http"): 

277 from .http import HttpResourcePath 

278 

279 subclass = HttpResourcePath 

280 elif parsed.scheme == "gs": 

281 from .gs import GSResourcePath 

282 

283 subclass = GSResourcePath 

284 elif parsed.scheme == "resource": 

285 # Rules for scheme names disallow pkg_resource 

286 from .packageresource import PackageResourcePath 

287 

288 subclass = PackageResourcePath 

289 elif parsed.scheme == "mem": 

290 # in-memory datastore object 

291 from .mem import InMemoryResourcePath 

292 

293 subclass = InMemoryResourcePath 

294 else: 

295 raise NotImplementedError( 

296 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" 

297 ) 

298 

299 parsed, dirLike = subclass._fixupPathUri( 

300 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory 

301 ) 

302 

303 # It is possible for the class to change from schemeless 

304 # to file so handle that 

305 if parsed.scheme == "file": 

306 from .file import FileResourcePath 

307 

308 subclass = FileResourcePath 

309 

310 # Now create an instance of the correct subclass and set the 

311 # attributes directly 

312 self = object.__new__(subclass) 

313 self._uri = parsed 

314 self.dirLike = dirLike 

315 if isTemporary is None: 

316 isTemporary = False 

317 self.isTemporary = isTemporary 

318 return self 

319 

320 @property 

321 def scheme(self) -> str: 

322 """Return the URI scheme. 

323 

324 Notes 

325 ----- 

326 (``://`` is not part of the scheme). 

327 """ 

328 return self._uri.scheme 

329 

330 @property 

331 def netloc(self) -> str: 

332 """Return the URI network location.""" 

333 return self._uri.netloc 

334 

335 @property 

336 def path(self) -> str: 

337 """Return the path component of the URI.""" 

338 return self._uri.path 

339 

340 @property 

341 def unquoted_path(self) -> str: 

342 """Return path component of the URI with any URI quoting reversed.""" 

343 return urllib.parse.unquote(self._uri.path) 

344 

345 @property 

346 def ospath(self) -> str: 

347 """Return the path component of the URI localized to current OS.""" 

348 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

349 

350 @property 

351 def relativeToPathRoot(self) -> str: 

352 """Return path relative to network location. 

353 

354 Effectively, this is the path property with posix separator stripped 

355 from the left hand side of the path. 

356 

357 Always unquotes. 

358 """ 

359 p = self._pathLib(self.path) 

360 relToRoot = str(p.relative_to(p.root)) 

361 if self.dirLike and not relToRoot.endswith("/"): 

362 relToRoot += "/" 

363 return urllib.parse.unquote(relToRoot) 

364 

365 @property 

366 def is_root(self) -> bool: 

367 """Return whether this URI points to the root of the network location. 

368 

369 This means that the path components refers to the top level. 

370 """ 

371 relpath = self.relativeToPathRoot 

372 if relpath == "./": 

373 return True 

374 return False 

375 

376 @property 

377 def fragment(self) -> str: 

378 """Return the fragment component of the URI.""" 

379 return self._uri.fragment 

380 

381 @property 

382 def params(self) -> str: 

383 """Return any parameters included in the URI.""" 

384 return self._uri.params 

385 

386 @property 

387 def query(self) -> str: 

388 """Return any query strings included in the URI.""" 

389 return self._uri.query 

390 

391 def geturl(self) -> str: 

392 """Return the URI in string form. 

393 

394 Returns 

395 ------- 

396 url : `str` 

397 String form of URI. 

398 """ 

399 return self._uri.geturl() 

400 

401 def root_uri(self) -> ResourcePath: 

402 """Return the base root URI. 

403 

404 Returns 

405 ------- 

406 uri : `ResourcePath` 

407 root URI. 

408 """ 

409 return self.replace(path="", forceDirectory=True) 

410 

411 def split(self) -> tuple[ResourcePath, str]: 

412 """Split URI into head and tail. 

413 

414 Returns 

415 ------- 

416 head: `ResourcePath` 

417 Everything leading up to tail, expanded and normalized as per 

418 ResourcePath rules. 

419 tail : `str` 

420 Last path component. Tail will be empty if path ends on a 

421 separator. Tail will never contain separators. It will be 

422 unquoted. 

423 

424 Notes 

425 ----- 

426 Equivalent to `os.path.split` where head preserves the URI 

427 components. 

428 """ 

429 head, tail = self._pathModule.split(self.path) 

430 headuri = self._uri._replace(path=head) 

431 

432 # The file part should never include quoted metacharacters 

433 tail = urllib.parse.unquote(tail) 

434 

435 # Schemeless is special in that it can be a relative path 

436 # We need to ensure that it stays that way. All other URIs will 

437 # be absolute already. 

438 forceAbsolute = self._pathModule.isabs(self.path) 

439 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

440 

441 def basename(self) -> str: 

442 """Return the base name, last element of path, of the URI. 

443 

444 Returns 

445 ------- 

446 tail : `str` 

447 Last part of the path attribute. Trail will be empty if path ends 

448 on a separator. 

449 

450 Notes 

451 ----- 

452 If URI ends on a slash returns an empty string. This is the second 

453 element returned by `split()`. 

454 

455 Equivalent of `os.path.basename`. 

456 """ 

457 return self.split()[1] 

458 

459 def dirname(self) -> ResourcePath: 

460 """Return the directory component of the path as a new `ResourcePath`. 

461 

462 Returns 

463 ------- 

464 head : `ResourcePath` 

465 Everything except the tail of path attribute, expanded and 

466 normalized as per ResourcePath rules. 

467 

468 Notes 

469 ----- 

470 Equivalent of `os.path.dirname`. 

471 """ 

472 return self.split()[0] 

473 

474 def parent(self) -> ResourcePath: 

475 """Return a `ResourcePath` of the parent directory. 

476 

477 Returns 

478 ------- 

479 head : `ResourcePath` 

480 Everything except the tail of path attribute, expanded and 

481 normalized as per `ResourcePath` rules. 

482 

483 Notes 

484 ----- 

485 For a file-like URI this will be the same as calling `dirname()`. 

486 """ 

487 # When self is file-like, return self.dirname() 

488 if not self.dirLike: 

489 return self.dirname() 

490 # When self is dir-like, return its parent directory, 

491 # regardless of the presence of a trailing separator 

492 originalPath = self._pathLib(self.path) 

493 parentPath = originalPath.parent 

494 return self.replace(path=str(parentPath), forceDirectory=True) 

495 

496 def replace(self, forceDirectory: bool = False, isTemporary: bool = False, **kwargs: Any) -> ResourcePath: 

497 """Return new `ResourcePath` with specified components replaced. 

498 

499 Parameters 

500 ---------- 

501 forceDirectory : `bool`, optional 

502 Parameter passed to ResourcePath constructor to force this 

503 new URI to be dir-like. 

504 isTemporary : `bool`, optional 

505 Indicate that the resulting URI is temporary resource. 

506 **kwargs 

507 Components of a `urllib.parse.ParseResult` that should be 

508 modified for the newly-created `ResourcePath`. 

509 

510 Returns 

511 ------- 

512 new : `ResourcePath` 

513 New `ResourcePath` object with updated values. 

514 

515 Notes 

516 ----- 

517 Does not, for now, allow a change in URI scheme. 

518 """ 

519 # Disallow a change in scheme 

520 if "scheme" in kwargs: 

521 raise ValueError(f"Can not use replace() method to change URI scheme for {self}") 

522 return self.__class__( 

523 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary 

524 ) 

525 

526 def updatedFile(self, newfile: str) -> ResourcePath: 

527 """Return new URI with an updated final component of the path. 

528 

529 Parameters 

530 ---------- 

531 newfile : `str` 

532 File name with no path component. 

533 

534 Returns 

535 ------- 

536 updated : `ResourcePath` 

537 

538 Notes 

539 ----- 

540 Forces the ResourcePath.dirLike attribute to be false. The new file 

541 path will be quoted if necessary. 

542 """ 

543 if self.quotePaths: 

544 newfile = urllib.parse.quote(newfile) 

545 dir, _ = self._pathModule.split(self.path) 

546 newpath = self._pathModule.join(dir, newfile) 

547 

548 updated = self.replace(path=newpath) 

549 updated.dirLike = False 

550 return updated 

551 

552 def updatedExtension(self, ext: str | None) -> ResourcePath: 

553 """Return a new `ResourcePath` with updated file extension. 

554 

555 All file extensions are replaced. 

556 

557 Parameters 

558 ---------- 

559 ext : `str` or `None` 

560 New extension. If an empty string is given any extension will 

561 be removed. If `None` is given there will be no change. 

562 

563 Returns 

564 ------- 

565 updated : `ResourcePath` 

566 URI with the specified extension. Can return itself if 

567 no extension was specified. 

568 """ 

569 if ext is None: 

570 return self 

571 

572 # Get the extension 

573 current = self.getExtension() 

574 

575 # Nothing to do if the extension already matches 

576 if current == ext: 

577 return self 

578 

579 # Remove the current extension from the path 

580 # .fits.gz counts as one extension do not use os.path.splitext 

581 path = self.path 

582 if current: 

583 path = path[: -len(current)] 

584 

585 # Ensure that we have a leading "." on file extension (and we do not 

586 # try to modify the empty string) 

587 if ext and not ext.startswith("."): 

588 ext = "." + ext 

589 

590 return self.replace(path=path + ext) 

591 

592 def getExtension(self) -> str: 

593 """Return the file extension(s) associated with this URI path. 

594 

595 Returns 

596 ------- 

597 ext : `str` 

598 The file extension (including the ``.``). Can be empty string 

599 if there is no file extension. Usually returns only the last 

600 file extension unless there is a special extension modifier 

601 indicating file compression, in which case the combined 

602 extension (e.g. ``.fits.gz``) will be returned. 

603 """ 

604 special = {".gz", ".bz2", ".xz", ".fz"} 

605 

606 # Get the file part of the path so as not to be confused by 

607 # "." in directory names. 

608 basename = self.basename() 

609 extensions = self._pathLib(basename).suffixes 

610 

611 if not extensions: 

612 return "" 

613 

614 ext = extensions.pop() 

615 

616 # Multiple extensions, decide whether to include the final two 

617 if extensions and ext in special: 

618 ext = f"{extensions[-1]}{ext}" 

619 

620 return ext 

621 

622 def join( 

623 self, path: str | ResourcePath, isTemporary: bool | None = None, forceDirectory: bool = False 

624 ) -> ResourcePath: 

625 """Return new `ResourcePath` with additional path components. 

626 

627 Parameters 

628 ---------- 

629 path : `str`, `ResourcePath` 

630 Additional file components to append to the current URI. Assumed 

631 to include a file at the end. Will be quoted depending on the 

632 associated URI scheme. If the path looks like a URI with a scheme 

633 referring to an absolute location, it will be returned 

634 directly (matching the behavior of `os.path.join`). It can 

635 also be a `ResourcePath`. 

636 isTemporary : `bool`, optional 

637 Indicate that the resulting URI represents a temporary resource. 

638 Default is ``self.isTemporary``. 

639 forceDirectory : `bool`, optional 

640 If `True` forces the URI to end with a separator, otherwise given 

641 URI is interpreted as is. 

642 

643 Returns 

644 ------- 

645 new : `ResourcePath` 

646 New URI with any file at the end replaced with the new path 

647 components. 

648 

649 Notes 

650 ----- 

651 Schemeless URIs assume local path separator but all other URIs assume 

652 POSIX separator if the supplied path has directory structure. It 

653 may be this never becomes a problem but datastore templates assume 

654 POSIX separator is being used. 

655 

656 If an absolute `ResourcePath` is given for ``path`` is is assumed that 

657 this should be returned directly. Giving a ``path`` of an absolute 

658 scheme-less URI is not allowed for safety reasons as it may indicate 

659 a mistake in the calling code. 

660 

661 Raises 

662 ------ 

663 ValueError 

664 Raised if the ``path`` is an absolute scheme-less URI. In that 

665 situation it is unclear whether the intent is to return a 

666 ``file`` URI or it was a mistake and a relative scheme-less URI 

667 was meant. 

668 RuntimeError 

669 Raised if this attempts to join a temporary URI to a non-temporary 

670 URI. 

671 """ 

672 if isTemporary is None: 

673 isTemporary = self.isTemporary 

674 elif not isTemporary and self.isTemporary: 

675 raise RuntimeError("Cannot join temporary URI to non-temporary URI.") 

676 # If we have a full URI in path we will use it directly 

677 # but without forcing to absolute so that we can trap the 

678 # expected option of relative path. 

679 path_uri = ResourcePath( 

680 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary 

681 ) 

682 if path_uri.scheme: 

683 # Check for scheme so can distinguish explicit URIs from 

684 # absolute scheme-less URIs. 

685 return path_uri 

686 

687 if path_uri.isabs(): 

688 # Absolute scheme-less path. 

689 raise ValueError(f"Can not join absolute scheme-less {path_uri!r} to another URI.") 

690 

691 # If this was originally a ResourcePath extract the unquoted path from 

692 # it. Otherwise we use the string we were given to allow "#" to appear 

693 # in the filename if given as a plain string. 

694 if not isinstance(path, str): 

695 path = path_uri.unquoted_path 

696 

697 new = self.dirname() # By definition a directory URI 

698 

699 # new should be asked about quoting, not self, since dirname can 

700 # change the URI scheme for schemeless -> file 

701 if new.quotePaths: 

702 path = urllib.parse.quote(path) 

703 

704 newpath = self._pathModule.normpath(self._pathModule.join(new.path, path)) 

705 

706 # normpath can strip trailing / so we force directory if the supplied 

707 # path ended with a / 

708 return new.replace( 

709 path=newpath, 

710 forceDirectory=(forceDirectory or path.endswith(self._pathModule.sep)), 

711 isTemporary=isTemporary, 

712 ) 

713 

714 def relative_to(self, other: ResourcePath) -> str | None: 

715 """Return the relative path from this URI to the other URI. 

716 

717 Parameters 

718 ---------- 

719 other : `ResourcePath` 

720 URI to use to calculate the relative path. Must be a parent 

721 of this URI. 

722 

723 Returns 

724 ------- 

725 subpath : `str` 

726 The sub path of this URI relative to the supplied other URI. 

727 Returns `None` if there is no parent child relationship. 

728 Scheme and netloc must match. 

729 """ 

730 # Scheme-less absolute other is treated as if it's a file scheme. 

731 # Scheme-less relative other can only return non-None if self 

732 # is also scheme-less relative and that is handled specifically 

733 # in a subclass. 

734 if not other.scheme and other.isabs(): 

735 other = other.abspath() 

736 

737 # Scheme-less self is handled elsewhere. 

738 if self.scheme != other.scheme: 

739 return None 

740 if self.netloc != other.netloc: 

741 # Special case for localhost vs empty string. 

742 # There can be many variants of localhost. 

743 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"} 

744 if not {self.netloc, other.netloc}.issubset(local_netlocs): 

745 return None 

746 

747 enclosed_path = self._pathLib(self.relativeToPathRoot) 

748 parent_path = other.relativeToPathRoot 

749 subpath: str | None 

750 try: 

751 subpath = str(enclosed_path.relative_to(parent_path)) 

752 except ValueError: 

753 subpath = None 

754 else: 

755 subpath = urllib.parse.unquote(subpath) 

756 return subpath 

757 

758 def exists(self) -> bool: 

759 """Indicate that the resource is available. 

760 

761 Returns 

762 ------- 

763 exists : `bool` 

764 `True` if the resource exists. 

765 """ 

766 raise NotImplementedError() 

767 

768 @classmethod 

769 def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

770 """Check for existence of multiple URIs at once. 

771 

772 Parameters 

773 ---------- 

774 uris : iterable of `ResourcePath` 

775 The URIs to test. 

776 

777 Returns 

778 ------- 

779 existence : `dict` of [`ResourcePath`, `bool`] 

780 Mapping of original URI to boolean indicating existence. 

781 """ 

782 # Group by scheme to allow a subclass to be able to use 

783 # specialized implementations. 

784 grouped: dict[type, list[ResourcePath]] = {} 

785 for uri in uris: 

786 uri_class = uri.__class__ 

787 if uri_class not in grouped: 

788 grouped[uri_class] = [] 

789 grouped[uri_class].append(uri) 

790 

791 existence: dict[ResourcePath, bool] = {} 

792 for uri_class in grouped: 

793 existence.update(uri_class._mexists(grouped[uri_class])) 

794 

795 return existence 

796 

797 @classmethod 

798 def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

799 """Check for existence of multiple URIs at once. 

800 

801 Implementation helper method for `mexists`. 

802 

803 Parameters 

804 ---------- 

805 uris : iterable of `ResourcePath` 

806 The URIs to test. 

807 

808 Returns 

809 ------- 

810 existence : `dict` of [`ResourcePath`, `bool`] 

811 Mapping of original URI to boolean indicating existence. 

812 """ 

813 exists_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 

814 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris} 

815 

816 results: dict[ResourcePath, bool] = {} 

817 for future in concurrent.futures.as_completed(future_exists): 

818 uri = future_exists[future] 

819 try: 

820 exists = future.result() 

821 except Exception: 

822 exists = False 

823 results[uri] = exists 

824 return results 

825 

826 def remove(self) -> None: 

827 """Remove the resource.""" 

828 raise NotImplementedError() 

829 

830 def isabs(self) -> bool: 

831 """Indicate that the resource is fully specified. 

832 

833 For non-schemeless URIs this is always true. 

834 

835 Returns 

836 ------- 

837 isabs : `bool` 

838 `True` in all cases except schemeless URI. 

839 """ 

840 return True 

841 

842 def abspath(self) -> ResourcePath: 

843 """Return URI using an absolute path. 

844 

845 Returns 

846 ------- 

847 abs : `ResourcePath` 

848 Absolute URI. For non-schemeless URIs this always returns itself. 

849 Schemeless URIs are upgraded to file URIs. 

850 """ 

851 return self 

852 

853 def _as_local(self) -> tuple[str, bool]: 

854 """Return the location of the (possibly remote) resource as local file. 

855 

856 This is a helper function for `as_local` context manager. 

857 

858 Returns 

859 ------- 

860 path : `str` 

861 If this is a remote resource, it will be a copy of the resource 

862 on the local file system, probably in a temporary directory. 

863 For a local resource this should be the actual path to the 

864 resource. 

865 is_temporary : `bool` 

866 Indicates if the local path is a temporary file or not. 

867 """ 

868 raise NotImplementedError() 

869 

870 @contextlib.contextmanager 

871 def as_local(self) -> Iterator[ResourcePath]: 

872 """Return the location of the (possibly remote) resource as local file. 

873 

874 Yields 

875 ------ 

876 local : `ResourcePath` 

877 If this is a remote resource, it will be a copy of the resource 

878 on the local file system, probably in a temporary directory. 

879 For a local resource this should be the actual path to the 

880 resource. 

881 

882 Notes 

883 ----- 

884 The context manager will automatically delete any local temporary 

885 file. 

886 

887 Examples 

888 -------- 

889 Should be used as a context manager: 

890 

891 .. code-block:: py 

892 

893 with uri.as_local() as local: 

894 ospath = local.ospath 

895 """ 

896 if self.dirLike: 

897 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.") 

898 local_src, is_temporary = self._as_local() 

899 local_uri = ResourcePath(local_src, isTemporary=is_temporary) 

900 

901 try: 

902 yield local_uri 

903 finally: 

904 # The caller might have relocated the temporary file. 

905 # Do not ever delete if the temporary matches self 

906 # (since it may have been that a temporary file was made local 

907 # but already was local). 

908 if self != local_uri and is_temporary and local_uri.exists(): 

909 local_uri.remove() 

910 

911 @classmethod 

912 @contextlib.contextmanager 

913 def temporary_uri( 

914 cls, prefix: ResourcePath | None = None, suffix: str | None = None 

915 ) -> Iterator[ResourcePath]: 

916 """Create a temporary file-like URI. 

917 

918 Parameters 

919 ---------- 

920 prefix : `ResourcePath`, optional 

921 Prefix to use. Without this the path will be formed as a local 

922 file URI in a temporary directory. Ensuring that the prefix 

923 location exists is the responsibility of the caller. 

924 suffix : `str`, optional 

925 A file suffix to be used. The ``.`` should be included in this 

926 suffix. 

927 

928 Yields 

929 ------ 

930 uri : `ResourcePath` 

931 The temporary URI. Will be removed when the context is completed. 

932 """ 

933 use_tempdir = False 

934 if prefix is None: 

935 directory = tempfile.mkdtemp() 

936 # If the user has set a umask that restricts the owner-write bit, 

937 # the directory returned from mkdtemp may not initially be 

938 # writeable by us 

939 ensure_directory_is_writeable(directory) 

940 

941 prefix = ResourcePath(directory, forceDirectory=True, isTemporary=True) 

942 # Record that we need to delete this directory. Can not rely 

943 # on isTemporary flag since an external prefix may have that 

944 # set as well. 

945 use_tempdir = True 

946 

947 # Need to create a randomized file name. For consistency do not 

948 # use mkstemp for local and something else for remote. Additionally 

949 # this method does not create the file to prevent name clashes. 

950 characters = "abcdefghijklmnopqrstuvwxyz0123456789_" 

951 rng = Random() 

952 tempname = "".join(rng.choice(characters) for _ in range(16)) 

953 if suffix: 

954 tempname += suffix 

955 temporary_uri = prefix.join(tempname, isTemporary=True) 

956 if temporary_uri.dirLike: 

957 # If we had a safe way to clean up a remote temporary directory, we 

958 # could support this. 

959 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.") 

960 try: 

961 yield temporary_uri 

962 finally: 

963 if use_tempdir: 

964 shutil.rmtree(prefix.ospath, ignore_errors=True) 

965 else: 

966 with contextlib.suppress(FileNotFoundError): 

967 # It's okay if this does not work because the user removed 

968 # the file. 

969 temporary_uri.remove() 

970 

971 def read(self, size: int = -1) -> bytes: 

972 """Open the resource and return the contents in bytes. 

973 

974 Parameters 

975 ---------- 

976 size : `int`, optional 

977 The number of bytes to read. Negative or omitted indicates 

978 that all data should be read. 

979 """ 

980 raise NotImplementedError() 

981 

982 def write(self, data: bytes, overwrite: bool = True) -> None: 

983 """Write the supplied bytes to the new resource. 

984 

985 Parameters 

986 ---------- 

987 data : `bytes` 

988 The bytes to write to the resource. The entire contents of the 

989 resource will be replaced. 

990 overwrite : `bool`, optional 

991 If `True` the resource will be overwritten if it exists. Otherwise 

992 the write will fail. 

993 """ 

994 raise NotImplementedError() 

995 

996 def mkdir(self) -> None: 

997 """For a dir-like URI, create the directory resource if needed.""" 

998 raise NotImplementedError() 

999 

1000 def isdir(self) -> bool: 

1001 """Return True if this URI looks like a directory, else False.""" 

1002 return self.dirLike 

1003 

1004 def size(self) -> int: 

1005 """For non-dir-like URI, return the size of the resource. 

1006 

1007 Returns 

1008 ------- 

1009 sz : `int` 

1010 The size in bytes of the resource associated with this URI. 

1011 Returns 0 if dir-like. 

1012 """ 

1013 raise NotImplementedError() 

1014 

1015 def __str__(self) -> str: 

1016 """Convert the URI to its native string form.""" 

1017 return self.geturl() 

1018 

1019 def __repr__(self) -> str: 

1020 """Return string representation suitable for evaluation.""" 

1021 return f'ResourcePath("{self.geturl()}")' 

1022