Coverage for python/lsst/resources/_resourcePath.py: 28%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14import concurrent.futures 

15import contextlib 

16import copy 

17import io 

18import locale 

19import logging 

20import os 

21import posixpath 

22import re 

23import shutil 

24import tempfile 

25import urllib.parse 

26from pathlib import Path, PurePath, PurePosixPath 

27from random import Random 

28 

29__all__ = ("ResourcePath", "ResourcePathExpression") 

30 

31from collections.abc import Iterable, Iterator 

32from typing import TYPE_CHECKING, Any, Literal, overload 

33 

34from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol 

35from .utils import ensure_directory_is_writeable 

36 

37if TYPE_CHECKING: 

38 from .utils import TransactionProtocol 

39 

40 

41log = logging.getLogger(__name__) 

42 

43# Regex for looking for URI escapes 

44ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

45 

46# Precomputed escaped hash 

47ESCAPED_HASH = urllib.parse.quote("#") 

48 

49# Maximum number of worker threads for parallelized operations. 

50# If greater than 10, be aware that this number has to be consistent 

51# with connection pool sizing (for example in urllib3). 

52MAX_WORKERS = 10 

53 

54 

55class ResourcePath: # numpydoc ignore=PR02 

56 """Convenience wrapper around URI parsers. 

57 

58 Provides access to URI components and can convert file 

59 paths into absolute path URIs. Scheme-less URIs are treated as if 

60 they are local file system paths and are converted to absolute URIs. 

61 

62 A specialist subclass is created for each supported URI scheme. 

63 

64 Parameters 

65 ---------- 

66 uri : `str`, `pathlib.Path`, `urllib.parse.ParseResult`, or `ResourcePath` 

67 URI in string form. Can be scheme-less if referring to a relative 

68 path or an absolute path on the local file system. 

69 root : `str` or `ResourcePath`, optional 

70 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

71 use this as the root. Must be absolute. If `None` the current 

72 working directory will be used. Can be any supported URI scheme. 

73 Not used if ``forceAbsolute`` is `False`. 

74 forceAbsolute : `bool`, optional 

75 If `True`, scheme-less relative URI will be converted to an absolute 

76 path using a ``file`` scheme. If `False` scheme-less URI will remain 

77 scheme-less and will not be updated to ``file`` or absolute path unless 

78 it is already an absolute path, in which case it will be updated to 

79 a ``file`` scheme. 

80 forceDirectory : `bool` or `None`, optional 

81 If `True` forces the URI to end with a separator. If `False` the URI 

82 is interpreted as a file-like entity. Default, `None`, is that the 

83 given URI is interpreted as a directory if there is a trailing ``/`` or 

84 for some schemes the system will check to see if it is a file or a 

85 directory. 

86 isTemporary : `bool`, optional 

87 If `True` indicates that this URI points to a temporary resource. 

88 The default is `False`, unless ``uri`` is already a `ResourcePath` 

89 instance and ``uri.isTemporary is True``. 

90 

91 Notes 

92 ----- 

93 A non-standard URI of the form ``file:dir/file.txt`` is always converted 

94 to an absolute ``file`` URI. 

95 """ 

96 

97 _pathLib: type[PurePath] = PurePosixPath 

98 """Path library to use for this scheme.""" 

99 

100 _pathModule = posixpath 

101 """Path module to use for this scheme.""" 

102 

103 transferModes: tuple[str, ...] = ("copy", "auto", "move") 

104 """Transfer modes supported by this implementation. 

105 

106 Move is special in that it is generally a copy followed by an unlink. 

107 Whether that unlink works depends critically on whether the source URI 

108 implements unlink. If it does not the move will be reported as a failure. 

109 """ 

110 

111 transferDefault: str = "copy" 

112 """Default mode to use for transferring if ``auto`` is specified.""" 

113 

114 quotePaths = True 

115 """True if path-like elements modifying a URI should be quoted. 

116 

117 All non-schemeless URIs have to internally use quoted paths. Therefore 

118 if a new file name is given (e.g. to updatedFile or join) a decision must 

119 be made whether to quote it to be consistent. 

120 """ 

121 

122 isLocal = False 

123 """If `True` this URI refers to a local file.""" 

124 

125 # This is not an ABC with abstract methods because the __new__ being 

126 # a factory confuses mypy such that it assumes that every constructor 

127 # returns a ResourcePath and then determines that all the abstract methods 

128 # are still abstract. If they are not marked abstract but just raise 

129 # mypy is fine with it. 

130 

131 # mypy is confused without these 

132 _uri: urllib.parse.ParseResult 

133 isTemporary: bool 

134 dirLike: bool | None 

135 """Whether the resource looks like a directory resource. `None` means that 

136 the status is uncertain.""" 

137 

138 def __new__( 

139 cls, 

140 uri: ResourcePathExpression, 

141 root: str | ResourcePath | None = None, 

142 forceAbsolute: bool = True, 

143 forceDirectory: bool | None = None, 

144 isTemporary: bool | None = None, 

145 ) -> ResourcePath: 

146 """Create and return new specialist ResourcePath subclass.""" 

147 parsed: urllib.parse.ParseResult 

148 dirLike: bool | None = forceDirectory 

149 subclass: type[ResourcePath] | None = None 

150 

151 # Force root to be a ResourcePath -- this simplifies downstream 

152 # code. 

153 if root is None: 

154 root_uri = None 

155 elif isinstance(root, str): 

156 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True) 

157 else: 

158 root_uri = root 

159 

160 if isinstance(uri, os.PathLike): 

161 uri = str(uri) 

162 

163 # Record if we need to post process the URI components 

164 # or if the instance is already fully configured 

165 if isinstance(uri, str): 

166 # Since local file names can have special characters in them 

167 # we need to quote them for the parser but we can unquote 

168 # later. Assume that all other URI schemes are quoted. 

169 # Since sometimes people write file:/a/b and not file:///a/b 

170 # we should not quote in the explicit case of file: 

171 if "://" not in uri and not uri.startswith("file:"): 

172 if ESCAPES_RE.search(uri): 

173 log.warning("Possible double encoding of %s", uri) 

174 else: 

175 # Fragments are generally not encoded so we must search 

176 # for the fragment boundary ourselves. This is making 

177 # an assumption that the filename does not include a "#" 

178 # and also that there is no "/" in the fragment itself. 

179 to_encode = uri 

180 fragment = "" 

181 if "#" in uri: 

182 dirpos = uri.rfind("/") 

183 trailing = uri[dirpos + 1 :] 

184 hashpos = trailing.rfind("#") 

185 if hashpos != -1: 

186 fragment = trailing[hashpos:] 

187 to_encode = uri[: dirpos + hashpos + 1] 

188 

189 uri = urllib.parse.quote(to_encode) + fragment 

190 

191 parsed = urllib.parse.urlparse(uri) 

192 elif isinstance(uri, urllib.parse.ParseResult): 

193 parsed = copy.copy(uri) 

194 # If we are being instantiated with a subclass, rather than 

195 # ResourcePath, ensure that that subclass is used directly. 

196 # This could lead to inconsistencies if this constructor 

197 # is used externally outside of the ResourcePath.replace() method. 

198 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt")) 

199 # will be a problem. 

200 # This is needed to prevent a schemeless absolute URI become 

201 # a file URI unexpectedly when calling updatedFile or 

202 # updatedExtension 

203 if cls is not ResourcePath: 

204 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory) 

205 subclass = cls 

206 

207 elif isinstance(uri, ResourcePath): 

208 # Since ResourcePath is immutable we can return the argument 

209 # unchanged if it already agrees with forceDirectory, isTemporary, 

210 # and forceAbsolute. 

211 # We invoke __new__ again with str(self) to add a scheme for 

212 # forceAbsolute, but for the others that seems more likely to paper 

213 # over logic errors than do something useful, so we just raise. 

214 if forceDirectory is not None and uri.dirLike is not None and forceDirectory is not uri.dirLike: 

215 # Can not force a file-like URI to become a dir-like one or 

216 # vice versa. 

217 raise RuntimeError( 

218 f"{uri} can not be forced to change directory vs file state when previously declared." 

219 ) 

220 if isTemporary is not None and isTemporary is not uri.isTemporary: 

221 raise RuntimeError( 

222 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} " 

223 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}." 

224 ) 

225 

226 if forceAbsolute and not uri.scheme: 

227 # Create new absolute from relative. 

228 return ResourcePath( 

229 str(uri), 

230 root=root, 

231 forceAbsolute=forceAbsolute, 

232 forceDirectory=forceDirectory or uri.dirLike, 

233 isTemporary=uri.isTemporary, 

234 ) 

235 elif forceDirectory is not None and uri.dirLike is None: 

236 # Clone but with a new dirLike status. 

237 return uri.replace(forceDirectory=forceDirectory) 

238 return uri 

239 else: 

240 raise ValueError( 

241 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'" 

242 ) 

243 

244 if subclass is None: 

245 # Work out the subclass from the URI scheme 

246 if not parsed.scheme: 

247 # Root may be specified as a ResourcePath that overrides 

248 # the schemeless determination. 

249 if ( 

250 root_uri is not None 

251 and root_uri.scheme != "file" # file scheme has different code path 

252 and not parsed.path.startswith("/") # Not already absolute path 

253 ): 

254 if root_uri.dirLike is False: 

255 raise ValueError( 

256 f"Root URI ({root}) was not a directory so can not be joined with" 

257 f" path {parsed.path!r}" 

258 ) 

259 # If root is temporary or this schemeless is temporary we 

260 # assume this URI is temporary. 

261 isTemporary = isTemporary or root_uri.isTemporary 

262 joined = root_uri.join( 

263 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary 

264 ) 

265 

266 # Rather than returning this new ResourcePath directly we 

267 # instead extract the path and the scheme and adjust the 

268 # URI we were given -- we need to do this to preserve 

269 # fragments since join() will drop them. 

270 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc) 

271 subclass = type(joined) 

272 

273 # Clear the root parameter to indicate that it has 

274 # been applied already. 

275 root_uri = None 

276 else: 

277 from .schemeless import SchemelessResourcePath 

278 

279 subclass = SchemelessResourcePath 

280 elif parsed.scheme == "file": 

281 from .file import FileResourcePath 

282 

283 subclass = FileResourcePath 

284 elif parsed.scheme == "s3": 

285 from .s3 import S3ResourcePath 

286 

287 subclass = S3ResourcePath 

288 elif parsed.scheme.startswith("http"): 

289 from .http import HttpResourcePath 

290 

291 subclass = HttpResourcePath 

292 elif parsed.scheme == "gs": 

293 from .gs import GSResourcePath 

294 

295 subclass = GSResourcePath 

296 elif parsed.scheme == "resource": 

297 # Rules for scheme names disallow pkg_resource 

298 from .packageresource import PackageResourcePath 

299 

300 subclass = PackageResourcePath 

301 elif parsed.scheme == "mem": 

302 # in-memory datastore object 

303 from .mem import InMemoryResourcePath 

304 

305 subclass = InMemoryResourcePath 

306 else: 

307 raise NotImplementedError( 

308 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" 

309 ) 

310 

311 parsed, dirLike = subclass._fixupPathUri( 

312 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory 

313 ) 

314 

315 # It is possible for the class to change from schemeless 

316 # to file so handle that 

317 if parsed.scheme == "file": 

318 from .file import FileResourcePath 

319 

320 subclass = FileResourcePath 

321 

322 # Now create an instance of the correct subclass and set the 

323 # attributes directly 

324 self = object.__new__(subclass) 

325 self._uri = parsed 

326 self.dirLike = dirLike 

327 if isTemporary is None: 

328 isTemporary = False 

329 self.isTemporary = isTemporary 

330 return self 

331 

332 @property 

333 def scheme(self) -> str: 

334 """Return the URI scheme. 

335 

336 Notes 

337 ----- 

338 (``://`` is not part of the scheme). 

339 """ 

340 return self._uri.scheme 

341 

342 @property 

343 def netloc(self) -> str: 

344 """Return the URI network location.""" 

345 return self._uri.netloc 

346 

347 @property 

348 def path(self) -> str: 

349 """Return the path component of the URI.""" 

350 return self._uri.path 

351 

352 @property 

353 def unquoted_path(self) -> str: 

354 """Return path component of the URI with any URI quoting reversed.""" 

355 return urllib.parse.unquote(self._uri.path) 

356 

357 @property 

358 def ospath(self) -> str: 

359 """Return the path component of the URI localized to current OS.""" 

360 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

361 

362 @property 

363 def relativeToPathRoot(self) -> str: 

364 """Return path relative to network location. 

365 

366 This is the path property with posix separator stripped 

367 from the left hand side of the path. 

368 

369 Always unquotes. 

370 """ 

371 relToRoot = self.path.lstrip("/") 

372 if relToRoot == "": 

373 return "./" 

374 return urllib.parse.unquote(relToRoot) 

375 

376 @property 

377 def is_root(self) -> bool: 

378 """Return whether this URI points to the root of the network location. 

379 

380 This means that the path components refers to the top level. 

381 """ 

382 relpath = self.relativeToPathRoot 

383 if relpath == "./": 

384 return True 

385 return False 

386 

387 @property 

388 def fragment(self) -> str: 

389 """Return the fragment component of the URI.""" 

390 return self._uri.fragment 

391 

392 @property 

393 def params(self) -> str: 

394 """Return any parameters included in the URI.""" 

395 return self._uri.params 

396 

397 @property 

398 def query(self) -> str: 

399 """Return any query strings included in the URI.""" 

400 return self._uri.query 

401 

402 def geturl(self) -> str: 

403 """Return the URI in string form. 

404 

405 Returns 

406 ------- 

407 url : `str` 

408 String form of URI. 

409 """ 

410 return self._uri.geturl() 

411 

412 def root_uri(self) -> ResourcePath: 

413 """Return the base root URI. 

414 

415 Returns 

416 ------- 

417 uri : `ResourcePath` 

418 Root URI. 

419 """ 

420 return self.replace(path="", forceDirectory=True) 

421 

422 def split(self) -> tuple[ResourcePath, str]: 

423 """Split URI into head and tail. 

424 

425 Returns 

426 ------- 

427 head: `ResourcePath` 

428 Everything leading up to tail, expanded and normalized as per 

429 ResourcePath rules. 

430 tail : `str` 

431 Last path component. Tail will be empty if path ends on a 

432 separator or if the URI is known to be associated with a directory. 

433 Tail will never contain separators. It will be unquoted. 

434 

435 Notes 

436 ----- 

437 Equivalent to `os.path.split` where head preserves the URI 

438 components. In some cases this method can result in a file system 

439 check to verify whether the URI is a directory or not (only if 

440 ``forceDirectory`` was `None` during construction). For a scheme-less 

441 URI this can mean that the result might change depending on current 

442 working directory. 

443 """ 

444 if self.isdir(): 

445 # This is known to be a directory so must return itself and 

446 # the empty string. 

447 return self, "" 

448 

449 head, tail = self._pathModule.split(self.path) 

450 headuri = self._uri._replace(path=head) 

451 

452 # The file part should never include quoted metacharacters 

453 tail = urllib.parse.unquote(tail) 

454 

455 # Schemeless is special in that it can be a relative path. 

456 # We need to ensure that it stays that way. All other URIs will 

457 # be absolute already. 

458 forceAbsolute = self.isabs() 

459 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

460 

461 def basename(self) -> str: 

462 """Return the base name, last element of path, of the URI. 

463 

464 Returns 

465 ------- 

466 tail : `str` 

467 Last part of the path attribute. Trail will be empty if path ends 

468 on a separator. 

469 

470 Notes 

471 ----- 

472 If URI ends on a slash returns an empty string. This is the second 

473 element returned by `split()`. 

474 

475 Equivalent of `os.path.basename`. 

476 """ 

477 return self.split()[1] 

478 

479 def dirname(self) -> ResourcePath: 

480 """Return the directory component of the path as a new `ResourcePath`. 

481 

482 Returns 

483 ------- 

484 head : `ResourcePath` 

485 Everything except the tail of path attribute, expanded and 

486 normalized as per ResourcePath rules. 

487 

488 Notes 

489 ----- 

490 Equivalent of `os.path.dirname`. If this is a directory URI it will 

491 be returned unchanged. If the parent directory is always required 

492 use `parent`. 

493 """ 

494 return self.split()[0] 

495 

496 def parent(self) -> ResourcePath: 

497 """Return a `ResourcePath` of the parent directory. 

498 

499 Returns 

500 ------- 

501 head : `ResourcePath` 

502 Everything except the tail of path attribute, expanded and 

503 normalized as per `ResourcePath` rules. 

504 

505 Notes 

506 ----- 

507 For a file-like URI this will be the same as calling `dirname`. 

508 For a directory-like URI this will always return the parent directory 

509 whereas `dirname()` will return the original URI. This is consistent 

510 with `os.path.dirname` compared to the `pathlib.Path` property 

511 ``parent``. 

512 """ 

513 if self.dirLike is False: 

514 # os.path.split() is slightly faster than calling Path().parent. 

515 return self.dirname() 

516 # When self is dir-like, returns its parent directory, 

517 # regardless of the presence of a trailing separator 

518 originalPath = self._pathLib(self.path) 

519 parentPath = originalPath.parent 

520 return self.replace(path=str(parentPath), forceDirectory=True) 

521 

522 def replace( 

523 self, forceDirectory: bool | None = None, isTemporary: bool = False, **kwargs: Any 

524 ) -> ResourcePath: 

525 """Return new `ResourcePath` with specified components replaced. 

526 

527 Parameters 

528 ---------- 

529 forceDirectory : `bool` or `None`, optional 

530 Parameter passed to ResourcePath constructor to force this 

531 new URI to be dir-like or file-like. 

532 isTemporary : `bool`, optional 

533 Indicate that the resulting URI is temporary resource. 

534 **kwargs 

535 Components of a `urllib.parse.ParseResult` that should be 

536 modified for the newly-created `ResourcePath`. 

537 

538 Returns 

539 ------- 

540 new : `ResourcePath` 

541 New `ResourcePath` object with updated values. 

542 

543 Notes 

544 ----- 

545 Does not, for now, allow a change in URI scheme. 

546 """ 

547 # Disallow a change in scheme 

548 if "scheme" in kwargs: 

549 raise ValueError(f"Can not use replace() method to change URI scheme for {self}") 

550 return self.__class__( 

551 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary 

552 ) 

553 

554 def updatedFile(self, newfile: str) -> ResourcePath: 

555 """Return new URI with an updated final component of the path. 

556 

557 Parameters 

558 ---------- 

559 newfile : `str` 

560 File name with no path component. 

561 

562 Returns 

563 ------- 

564 updated : `ResourcePath` 

565 Updated `ResourcePath` with new updated final component. 

566 

567 Notes 

568 ----- 

569 Forces the ``ResourcePath.dirLike`` attribute to be false. The new file 

570 path will be quoted if necessary. If the current URI is known to 

571 refer to a directory, the new file will be joined to the current file. 

572 It is recommended that this behavior no longer be used and a call 

573 to `isdir` by the caller should be used to decide whether to join or 

574 replace. In the future this method may be modified to always replace 

575 the final element of the path. 

576 """ 

577 if self.dirLike: 

578 return self.join(newfile, forceDirectory=False) 

579 return self.parent().join(newfile, forceDirectory=False) 

580 

581 def updatedExtension(self, ext: str | None) -> ResourcePath: 

582 """Return a new `ResourcePath` with updated file extension. 

583 

584 All file extensions are replaced. 

585 

586 Parameters 

587 ---------- 

588 ext : `str` or `None` 

589 New extension. If an empty string is given any extension will 

590 be removed. If `None` is given there will be no change. 

591 

592 Returns 

593 ------- 

594 updated : `ResourcePath` 

595 URI with the specified extension. Can return itself if 

596 no extension was specified. 

597 """ 

598 if ext is None: 

599 return self 

600 

601 # Get the extension 

602 current = self.getExtension() 

603 

604 # Nothing to do if the extension already matches 

605 if current == ext: 

606 return self 

607 

608 # Remove the current extension from the path 

609 # .fits.gz counts as one extension do not use os.path.splitext 

610 path = self.path 

611 if current: 

612 path = path.removesuffix(current) 

613 

614 # Ensure that we have a leading "." on file extension (and we do not 

615 # try to modify the empty string) 

616 if ext and not ext.startswith("."): 

617 ext = "." + ext 

618 

619 return self.replace(path=path + ext, forceDirectory=False) 

620 

621 def getExtension(self) -> str: 

622 """Return the extension(s) associated with this URI path. 

623 

624 Returns 

625 ------- 

626 ext : `str` 

627 The file extension (including the ``.``). Can be empty string 

628 if there is no file extension. Usually returns only the last 

629 file extension unless there is a special extension modifier 

630 indicating file compression, in which case the combined 

631 extension (e.g. ``.fits.gz``) will be returned. 

632 

633 Notes 

634 ----- 

635 Does not distinguish between file and directory URIs when determining 

636 a suffix. An extension is only determined from the final component 

637 of the path. 

638 """ 

639 special = {".gz", ".bz2", ".xz", ".fz"} 

640 

641 # path lib will ignore any "." in directories. 

642 # path lib works well: 

643 # extensions = self._pathLib(self.path).suffixes 

644 # But the constructor is slow. Therefore write our own implementation. 

645 # Strip trailing separator if present, do not care if this is a 

646 # directory or not. 

647 parts = self.path.rstrip("/").rsplit(self._pathModule.sep, 1) 

648 _, *extensions = parts[-1].split(".") 

649 

650 if not extensions: 

651 return "" 

652 extensions = ["." + x for x in extensions] 

653 

654 ext = extensions.pop() 

655 

656 # Multiple extensions, decide whether to include the final two 

657 if extensions and ext in special: 

658 ext = f"{extensions[-1]}{ext}" 

659 

660 return ext 

661 

662 def join( 

663 self, path: str | ResourcePath, isTemporary: bool | None = None, forceDirectory: bool | None = None 

664 ) -> ResourcePath: 

665 """Return new `ResourcePath` with additional path components. 

666 

667 Parameters 

668 ---------- 

669 path : `str`, `ResourcePath` 

670 Additional file components to append to the current URI. Will be 

671 quoted depending on the associated URI scheme. If the path looks 

672 like a URI referring to an absolute location, it will be returned 

673 directly (matching the behavior of `os.path.join`). It can 

674 also be a `ResourcePath`. 

675 isTemporary : `bool`, optional 

676 Indicate that the resulting URI represents a temporary resource. 

677 Default is ``self.isTemporary``. 

678 forceDirectory : `bool` or `None`, optional 

679 If `True` forces the URI to end with a separator. If `False` the 

680 resultant URI is declared to refer to a file. `None` indicates 

681 that the file directory status is unknown. 

682 

683 Returns 

684 ------- 

685 new : `ResourcePath` 

686 New URI with the path appended. 

687 

688 Notes 

689 ----- 

690 Schemeless URIs assume local path separator but all other URIs assume 

691 POSIX separator if the supplied path has directory structure. It 

692 may be this never becomes a problem but datastore templates assume 

693 POSIX separator is being used. 

694 

695 If an absolute `ResourcePath` is given for ``path`` is is assumed that 

696 this should be returned directly. Giving a ``path`` of an absolute 

697 scheme-less URI is not allowed for safety reasons as it may indicate 

698 a mistake in the calling code. 

699 

700 It is an error to attempt to join to something that is known to 

701 refer to a file. Use `updatedFile` if the file is to be 

702 replaced. 

703 

704 Raises 

705 ------ 

706 ValueError 

707 Raised if the given path object refers to a directory but the 

708 ``forceDirectory`` parameter insists the outcome should be a file, 

709 and vice versa. Also raised if the URI being joined with is known 

710 to refer to a file. 

711 RuntimeError 

712 Raised if this attempts to join a temporary URI to a non-temporary 

713 URI. 

714 """ 

715 if self.dirLike is False: 

716 raise ValueError("Can not join a new path component to a file.") 

717 if isTemporary is None: 

718 isTemporary = self.isTemporary 

719 elif not isTemporary and self.isTemporary: 

720 raise RuntimeError("Cannot join temporary URI to non-temporary URI.") 

721 # If we have a full URI in path we will use it directly 

722 # but without forcing to absolute so that we can trap the 

723 # expected option of relative path. 

724 path_uri = ResourcePath( 

725 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary 

726 ) 

727 if forceDirectory is not None and path_uri.dirLike is not forceDirectory: 

728 raise ValueError( 

729 "The supplied path URI to join has inconsistent directory state " 

730 f"with forceDirectory parameter: {path_uri.dirLike} vs {forceDirectory}" 

731 ) 

732 forceDirectory = path_uri.dirLike 

733 

734 if path_uri.isabs(): 

735 # Absolute URI so return it directly. 

736 return path_uri 

737 

738 # If this was originally a ResourcePath extract the unquoted path from 

739 # it. Otherwise we use the string we were given to allow "#" to appear 

740 # in the filename if given as a plain string. 

741 if not isinstance(path, str): 

742 path = path_uri.unquoted_path 

743 

744 # Might need to quote the path. 

745 if self.quotePaths: 

746 path = urllib.parse.quote(path) 

747 

748 newpath = self._pathModule.normpath(self._pathModule.join(self.path, path)) 

749 

750 # normpath can strip trailing / so we force directory if the supplied 

751 # path ended with a / 

752 has_dir_sep = path.endswith(self._pathModule.sep) 

753 if forceDirectory is None and has_dir_sep: 

754 forceDirectory = True 

755 elif forceDirectory is False and has_dir_sep: 

756 raise ValueError("Path to join has trailing / but is being forced to be a file.") 

757 return self.replace( 

758 path=newpath, 

759 forceDirectory=forceDirectory, 

760 isTemporary=isTemporary, 

761 ) 

762 

763 def relative_to(self, other: ResourcePath) -> str | None: 

764 """Return the relative path from this URI to the other URI. 

765 

766 Parameters 

767 ---------- 

768 other : `ResourcePath` 

769 URI to use to calculate the relative path. Must be a parent 

770 of this URI. 

771 

772 Returns 

773 ------- 

774 subpath : `str` 

775 The sub path of this URI relative to the supplied other URI. 

776 Returns `None` if there is no parent child relationship. 

777 Scheme and netloc must match. 

778 """ 

779 # Scheme-less self is handled elsewhere. 

780 if self.scheme != other.scheme: 

781 return None 

782 if self.netloc != other.netloc: 

783 # Special case for localhost vs empty string. 

784 # There can be many variants of localhost. 

785 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"} 

786 if not {self.netloc, other.netloc}.issubset(local_netlocs): 

787 return None 

788 

789 enclosed_path = self._pathLib(self.relativeToPathRoot) 

790 parent_path = other.relativeToPathRoot 

791 subpath: str | None 

792 try: 

793 subpath = str(enclosed_path.relative_to(parent_path)) 

794 except ValueError: 

795 subpath = None 

796 else: 

797 subpath = urllib.parse.unquote(subpath) 

798 return subpath 

799 

800 def exists(self) -> bool: 

801 """Indicate that the resource is available. 

802 

803 Returns 

804 ------- 

805 exists : `bool` 

806 `True` if the resource exists. 

807 """ 

808 raise NotImplementedError() 

809 

810 @classmethod 

811 def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

812 """Check for existence of multiple URIs at once. 

813 

814 Parameters 

815 ---------- 

816 uris : iterable of `ResourcePath` 

817 The URIs to test. 

818 

819 Returns 

820 ------- 

821 existence : `dict` of [`ResourcePath`, `bool`] 

822 Mapping of original URI to boolean indicating existence. 

823 """ 

824 # Group by scheme to allow a subclass to be able to use 

825 # specialized implementations. 

826 grouped: dict[type, list[ResourcePath]] = {} 

827 for uri in uris: 

828 uri_class = uri.__class__ 

829 if uri_class not in grouped: 

830 grouped[uri_class] = [] 

831 grouped[uri_class].append(uri) 

832 

833 existence: dict[ResourcePath, bool] = {} 

834 for uri_class in grouped: 

835 existence.update(uri_class._mexists(grouped[uri_class])) 

836 

837 return existence 

838 

839 @classmethod 

840 def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

841 """Check for existence of multiple URIs at once. 

842 

843 Implementation helper method for `mexists`. 

844 

845 Parameters 

846 ---------- 

847 uris : iterable of `ResourcePath` 

848 The URIs to test. 

849 

850 Returns 

851 ------- 

852 existence : `dict` of [`ResourcePath`, `bool`] 

853 Mapping of original URI to boolean indicating existence. 

854 """ 

855 exists_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 

856 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris} 

857 

858 results: dict[ResourcePath, bool] = {} 

859 for future in concurrent.futures.as_completed(future_exists): 

860 uri = future_exists[future] 

861 try: 

862 exists = future.result() 

863 except Exception: 

864 exists = False 

865 results[uri] = exists 

866 return results 

867 

868 def remove(self) -> None: 

869 """Remove the resource.""" 

870 raise NotImplementedError() 

871 

872 def isabs(self) -> bool: 

873 """Indicate that the resource is fully specified. 

874 

875 For non-schemeless URIs this is always true. 

876 

877 Returns 

878 ------- 

879 isabs : `bool` 

880 `True` in all cases except schemeless URI. 

881 """ 

882 return True 

883 

884 def abspath(self) -> ResourcePath: 

885 """Return URI using an absolute path. 

886 

887 Returns 

888 ------- 

889 abs : `ResourcePath` 

890 Absolute URI. For non-schemeless URIs this always returns itself. 

891 Schemeless URIs are upgraded to file URIs. 

892 """ 

893 return self 

894 

895 def _as_local(self) -> tuple[str, bool]: 

896 """Return the location of the (possibly remote) resource as local file. 

897 

898 This is a helper function for `as_local` context manager. 

899 

900 Returns 

901 ------- 

902 path : `str` 

903 If this is a remote resource, it will be a copy of the resource 

904 on the local file system, probably in a temporary directory. 

905 For a local resource this should be the actual path to the 

906 resource. 

907 is_temporary : `bool` 

908 Indicates if the local path is a temporary file or not. 

909 """ 

910 raise NotImplementedError() 

911 

912 @contextlib.contextmanager 

913 def as_local(self) -> Iterator[ResourcePath]: 

914 """Return the location of the (possibly remote) resource as local file. 

915 

916 Yields 

917 ------ 

918 local : `ResourcePath` 

919 If this is a remote resource, it will be a copy of the resource 

920 on the local file system, probably in a temporary directory. 

921 For a local resource this should be the actual path to the 

922 resource. 

923 

924 Notes 

925 ----- 

926 The context manager will automatically delete any local temporary 

927 file. 

928 

929 Examples 

930 -------- 

931 Should be used as a context manager: 

932 

933 .. code-block:: py 

934 

935 with uri.as_local() as local: 

936 ospath = local.ospath 

937 """ 

938 if self.isdir(): 

939 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.") 

940 local_src, is_temporary = self._as_local() 

941 local_uri = ResourcePath(local_src, isTemporary=is_temporary) 

942 

943 try: 

944 yield local_uri 

945 finally: 

946 # The caller might have relocated the temporary file. 

947 # Do not ever delete if the temporary matches self 

948 # (since it may have been that a temporary file was made local 

949 # but already was local). 

950 if self != local_uri and is_temporary and local_uri.exists(): 

951 local_uri.remove() 

952 

953 @classmethod 

954 @contextlib.contextmanager 

955 def temporary_uri( 

956 cls, prefix: ResourcePath | None = None, suffix: str | None = None 

957 ) -> Iterator[ResourcePath]: 

958 """Create a temporary file-like URI. 

959 

960 Parameters 

961 ---------- 

962 prefix : `ResourcePath`, optional 

963 Prefix to use. Without this the path will be formed as a local 

964 file URI in a temporary directory. Ensuring that the prefix 

965 location exists is the responsibility of the caller. 

966 suffix : `str`, optional 

967 A file suffix to be used. The ``.`` should be included in this 

968 suffix. 

969 

970 Yields 

971 ------ 

972 uri : `ResourcePath` 

973 The temporary URI. Will be removed when the context is completed. 

974 """ 

975 use_tempdir = False 

976 if prefix is None: 

977 directory = tempfile.mkdtemp() 

978 # If the user has set a umask that restricts the owner-write bit, 

979 # the directory returned from mkdtemp may not initially be 

980 # writeable by us 

981 ensure_directory_is_writeable(directory) 

982 

983 prefix = ResourcePath(directory, forceDirectory=True, isTemporary=True) 

984 # Record that we need to delete this directory. Can not rely 

985 # on isTemporary flag since an external prefix may have that 

986 # set as well. 

987 use_tempdir = True 

988 

989 # Need to create a randomized file name. For consistency do not 

990 # use mkstemp for local and something else for remote. Additionally 

991 # this method does not create the file to prevent name clashes. 

992 characters = "abcdefghijklmnopqrstuvwxyz0123456789_" 

993 rng = Random() 

994 tempname = "".join(rng.choice(characters) for _ in range(16)) 

995 if suffix: 

996 tempname += suffix 

997 temporary_uri = prefix.join(tempname, isTemporary=True) 

998 if temporary_uri.isdir(): 

999 # If we had a safe way to clean up a remote temporary directory, we 

1000 # could support this. 

1001 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.") 

1002 try: 

1003 yield temporary_uri 

1004 finally: 

1005 if use_tempdir: 

1006 shutil.rmtree(prefix.ospath, ignore_errors=True) 

1007 else: 

1008 with contextlib.suppress(FileNotFoundError): 

1009 # It's okay if this does not work because the user removed 

1010 # the file. 

1011 temporary_uri.remove() 

1012 

1013 def read(self, size: int = -1) -> bytes: 

1014 """Open the resource and return the contents in bytes. 

1015 

1016 Parameters 

1017 ---------- 

1018 size : `int`, optional 

1019 The number of bytes to read. Negative or omitted indicates 

1020 that all data should be read. 

1021 """ 

1022 raise NotImplementedError() 

1023 

1024 def write(self, data: bytes, overwrite: bool = True) -> None: