Coverage for python/lsst/resources/_resourcePath.py: 28%

435 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-16 02:51 -0700

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14import concurrent.futures 

15import contextlib 

16import copy 

17import io 

18import locale 

19import logging 

20import os 

21import posixpath 

22import re 

23import shutil 

24import tempfile 

25import urllib.parse 

26from pathlib import Path, PurePath, PurePosixPath 

27from random import Random 

28 

29__all__ = ("ResourcePath", "ResourcePathExpression") 

30 

31from collections.abc import Iterable, Iterator 

32from typing import TYPE_CHECKING, Any, Literal, overload 

33 

34from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol 

35from .utils import ensure_directory_is_writeable 

36 

37if TYPE_CHECKING: 

38 from .utils import TransactionProtocol 

39 

40 

41log = logging.getLogger(__name__) 

42 

43# Regex for looking for URI escapes 

44ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

45 

46# Precomputed escaped hash 

47ESCAPED_HASH = urllib.parse.quote("#") 

48 

49# Maximum number of worker threads for parallelized operations. 

50# If greater than 10, be aware that this number has to be consistent 

51# with connection pool sizing (for example in urllib3). 

52MAX_WORKERS = 10 

53 

54 

55class ResourcePath: # numpydoc ignore=PR02 

56 """Convenience wrapper around URI parsers. 

57 

58 Provides access to URI components and can convert file 

59 paths into absolute path URIs. Scheme-less URIs are treated as if 

60 they are local file system paths and are converted to absolute URIs. 

61 

62 A specialist subclass is created for each supported URI scheme. 

63 

64 Parameters 

65 ---------- 

66 uri : `str`, `pathlib.Path`, `urllib.parse.ParseResult`, or `ResourcePath` 

67 URI in string form. Can be scheme-less if referring to a relative 

68 path or an absolute path on the local file system. 

69 root : `str` or `ResourcePath`, optional 

70 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

71 use this as the root. Must be absolute. If `None` the current 

72 working directory will be used. Can be any supported URI scheme. 

73 Not used if ``forceAbsolute`` is `False`. 

74 forceAbsolute : `bool`, optional 

75 If `True`, scheme-less relative URI will be converted to an absolute 

76 path using a ``file`` scheme. If `False` scheme-less URI will remain 

77 scheme-less and will not be updated to ``file`` or absolute path unless 

78 it is already an absolute path, in which case it will be updated to 

79 a ``file`` scheme. 

80 forceDirectory : `bool` or `None`, optional 

81 If `True` forces the URI to end with a separator. If `False` the URI 

82 is interpreted as a file-like entity. Default, `None`, is that the 

83 given URI is interpreted as a directory if there is a trailing ``/`` or 

84 for some schemes the system will check to see if it is a file or a 

85 directory. 

86 isTemporary : `bool`, optional 

87 If `True` indicates that this URI points to a temporary resource. 

88 The default is `False`, unless ``uri`` is already a `ResourcePath` 

89 instance and ``uri.isTemporary is True``. 

90 

91 Notes 

92 ----- 

93 A non-standard URI of the form ``file:dir/file.txt`` is always converted 

94 to an absolute ``file`` URI. 

95 """ 

96 

97 _pathLib: type[PurePath] = PurePosixPath 

98 """Path library to use for this scheme.""" 

99 

100 _pathModule = posixpath 

101 """Path module to use for this scheme.""" 

102 

103 transferModes: tuple[str, ...] = ("copy", "auto", "move") 

104 """Transfer modes supported by this implementation. 

105 

106 Move is special in that it is generally a copy followed by an unlink. 

107 Whether that unlink works depends critically on whether the source URI 

108 implements unlink. If it does not the move will be reported as a failure. 

109 """ 

110 

111 transferDefault: str = "copy" 

112 """Default mode to use for transferring if ``auto`` is specified.""" 

113 

114 quotePaths = True 

115 """True if path-like elements modifying a URI should be quoted. 

116 

117 All non-schemeless URIs have to internally use quoted paths. Therefore 

118 if a new file name is given (e.g. to updatedFile or join) a decision must 

119 be made whether to quote it to be consistent. 

120 """ 

121 

122 isLocal = False 

123 """If `True` this URI refers to a local file.""" 

124 

125 # This is not an ABC with abstract methods because the __new__ being 

126 # a factory confuses mypy such that it assumes that every constructor 

127 # returns a ResourcePath and then determines that all the abstract methods 

128 # are still abstract. If they are not marked abstract but just raise 

129 # mypy is fine with it. 

130 

131 # mypy is confused without these 

132 _uri: urllib.parse.ParseResult 

133 isTemporary: bool 

134 dirLike: bool | None 

135 """Whether the resource looks like a directory resource. `None` means that 

136 the status is uncertain.""" 

137 

138 def __new__( 

139 cls, 

140 uri: ResourcePathExpression, 

141 root: str | ResourcePath | None = None, 

142 forceAbsolute: bool = True, 

143 forceDirectory: bool | None = None, 

144 isTemporary: bool | None = None, 

145 ) -> ResourcePath: 

146 """Create and return new specialist ResourcePath subclass.""" 

147 parsed: urllib.parse.ParseResult 

148 dirLike: bool | None = forceDirectory 

149 subclass: type[ResourcePath] | None = None 

150 

151 # Force root to be a ResourcePath -- this simplifies downstream 

152 # code. 

153 if root is None: 

154 root_uri = None 

155 elif isinstance(root, str): 

156 root_uri = ResourcePath(root, forceDirectory=True, forceAbsolute=True) 

157 else: 

158 root_uri = root 

159 

160 if isinstance(uri, os.PathLike): 

161 uri = str(uri) 

162 

163 # Record if we need to post process the URI components 

164 # or if the instance is already fully configured 

165 if isinstance(uri, str): 

166 # Since local file names can have special characters in them 

167 # we need to quote them for the parser but we can unquote 

168 # later. Assume that all other URI schemes are quoted. 

169 # Since sometimes people write file:/a/b and not file:///a/b 

170 # we should not quote in the explicit case of file: 

171 if "://" not in uri and not uri.startswith("file:"): 

172 if ESCAPES_RE.search(uri): 

173 log.warning("Possible double encoding of %s", uri) 

174 else: 

175 # Fragments are generally not encoded so we must search 

176 # for the fragment boundary ourselves. This is making 

177 # an assumption that the filename does not include a "#" 

178 # and also that there is no "/" in the fragment itself. 

179 to_encode = uri 

180 fragment = "" 

181 if "#" in uri: 

182 dirpos = uri.rfind("/") 

183 trailing = uri[dirpos + 1 :] 

184 hashpos = trailing.rfind("#") 

185 if hashpos != -1: 

186 fragment = trailing[hashpos:] 

187 to_encode = uri[: dirpos + hashpos + 1] 

188 

189 uri = urllib.parse.quote(to_encode) + fragment 

190 

191 parsed = urllib.parse.urlparse(uri) 

192 elif isinstance(uri, urllib.parse.ParseResult): 

193 parsed = copy.copy(uri) 

194 # If we are being instantiated with a subclass, rather than 

195 # ResourcePath, ensure that that subclass is used directly. 

196 # This could lead to inconsistencies if this constructor 

197 # is used externally outside of the ResourcePath.replace() method. 

198 # S3ResourcePath(urllib.parse.urlparse("file://a/b.txt")) 

199 # will be a problem. 

200 # This is needed to prevent a schemeless absolute URI become 

201 # a file URI unexpectedly when calling updatedFile or 

202 # updatedExtension 

203 if cls is not ResourcePath: 

204 parsed, dirLike = cls._fixDirectorySep(parsed, forceDirectory) 

205 subclass = cls 

206 

207 elif isinstance(uri, ResourcePath): 

208 # Since ResourcePath is immutable we can return the argument 

209 # unchanged if it already agrees with forceDirectory, isTemporary, 

210 # and forceAbsolute. 

211 # We invoke __new__ again with str(self) to add a scheme for 

212 # forceAbsolute, but for the others that seems more likely to paper 

213 # over logic errors than do something useful, so we just raise. 

214 if forceDirectory is not None and uri.dirLike is not None and forceDirectory is not uri.dirLike: 

215 # Can not force a file-like URI to become a dir-like one or 

216 # vice versa. 

217 raise RuntimeError( 

218 f"{uri} can not be forced to change directory vs file state when previously declared." 

219 ) 

220 if isTemporary is not None and isTemporary is not uri.isTemporary: 

221 raise RuntimeError( 

222 f"{uri} is already a {'temporary' if uri.isTemporary else 'permanent'} " 

223 f"ResourcePath; cannot make it {'temporary' if isTemporary else 'permanent'}." 

224 ) 

225 

226 if forceAbsolute and not uri.scheme: 

227 # Create new absolute from relative. 

228 return ResourcePath( 

229 str(uri), 

230 root=root, 

231 forceAbsolute=forceAbsolute, 

232 forceDirectory=forceDirectory or uri.dirLike, 

233 isTemporary=uri.isTemporary, 

234 ) 

235 elif forceDirectory is not None and uri.dirLike is None: 

236 # Clone but with a new dirLike status. 

237 return uri.replace(forceDirectory=forceDirectory) 

238 return uri 

239 else: 

240 raise ValueError( 

241 f"Supplied URI must be string, Path, ResourcePath, or ParseResult but got '{uri!r}'" 

242 ) 

243 

244 if subclass is None: 

245 # Work out the subclass from the URI scheme 

246 if not parsed.scheme: 

247 # Root may be specified as a ResourcePath that overrides 

248 # the schemeless determination. 

249 if ( 

250 root_uri is not None 

251 and root_uri.scheme != "file" # file scheme has different code path 

252 and not parsed.path.startswith("/") # Not already absolute path 

253 ): 

254 if root_uri.dirLike is False: 

255 raise ValueError( 

256 f"Root URI ({root}) was not a directory so can not be joined with" 

257 f" path {parsed.path!r}" 

258 ) 

259 # If root is temporary or this schemeless is temporary we 

260 # assume this URI is temporary. 

261 isTemporary = isTemporary or root_uri.isTemporary 

262 joined = root_uri.join( 

263 parsed.path, forceDirectory=forceDirectory, isTemporary=isTemporary 

264 ) 

265 

266 # Rather than returning this new ResourcePath directly we 

267 # instead extract the path and the scheme and adjust the 

268 # URI we were given -- we need to do this to preserve 

269 # fragments since join() will drop them. 

270 parsed = parsed._replace(scheme=joined.scheme, path=joined.path, netloc=joined.netloc) 

271 subclass = type(joined) 

272 

273 # Clear the root parameter to indicate that it has 

274 # been applied already. 

275 root_uri = None 

276 else: 

277 from .schemeless import SchemelessResourcePath 

278 

279 subclass = SchemelessResourcePath 

280 elif parsed.scheme == "file": 

281 from .file import FileResourcePath 

282 

283 subclass = FileResourcePath 

284 elif parsed.scheme == "s3": 

285 from .s3 import S3ResourcePath 

286 

287 subclass = S3ResourcePath 

288 elif parsed.scheme.startswith("http"): 

289 from .http import HttpResourcePath 

290 

291 subclass = HttpResourcePath 

292 elif parsed.scheme == "gs": 

293 from .gs import GSResourcePath 

294 

295 subclass = GSResourcePath 

296 elif parsed.scheme == "resource": 

297 # Rules for scheme names disallow pkg_resource 

298 from .packageresource import PackageResourcePath 

299 

300 subclass = PackageResourcePath 

301 elif parsed.scheme == "mem": 

302 # in-memory datastore object 

303 from .mem import InMemoryResourcePath 

304 

305 subclass = InMemoryResourcePath 

306 else: 

307 raise NotImplementedError( 

308 f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" 

309 ) 

310 

311 parsed, dirLike = subclass._fixupPathUri( 

312 parsed, root=root_uri, forceAbsolute=forceAbsolute, forceDirectory=forceDirectory 

313 ) 

314 

315 # It is possible for the class to change from schemeless 

316 # to file so handle that 

317 if parsed.scheme == "file": 

318 from .file import FileResourcePath 

319 

320 subclass = FileResourcePath 

321 

322 # Now create an instance of the correct subclass and set the 

323 # attributes directly 

324 self = object.__new__(subclass) 

325 self._uri = parsed 

326 self.dirLike = dirLike 

327 if isTemporary is None: 

328 isTemporary = False 

329 self.isTemporary = isTemporary 

330 return self 

331 

332 @property 

333 def scheme(self) -> str: 

334 """Return the URI scheme. 

335 

336 Notes 

337 ----- 

338 (``://`` is not part of the scheme). 

339 """ 

340 return self._uri.scheme 

341 

342 @property 

343 def netloc(self) -> str: 

344 """Return the URI network location.""" 

345 return self._uri.netloc 

346 

347 @property 

348 def path(self) -> str: 

349 """Return the path component of the URI.""" 

350 return self._uri.path 

351 

352 @property 

353 def unquoted_path(self) -> str: 

354 """Return path component of the URI with any URI quoting reversed.""" 

355 return urllib.parse.unquote(self._uri.path) 

356 

357 @property 

358 def ospath(self) -> str: 

359 """Return the path component of the URI localized to current OS.""" 

360 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

361 

362 @property 

363 def relativeToPathRoot(self) -> str: 

364 """Return path relative to network location. 

365 

366 This is the path property with posix separator stripped 

367 from the left hand side of the path. 

368 

369 Always unquotes. 

370 """ 

371 relToRoot = self.path.lstrip("/") 

372 if relToRoot == "": 

373 return "./" 

374 return urllib.parse.unquote(relToRoot) 

375 

376 @property 

377 def is_root(self) -> bool: 

378 """Return whether this URI points to the root of the network location. 

379 

380 This means that the path components refers to the top level. 

381 """ 

382 relpath = self.relativeToPathRoot 

383 if relpath == "./": 

384 return True 

385 return False 

386 

387 @property 

388 def fragment(self) -> str: 

389 """Return the fragment component of the URI.""" 

390 return self._uri.fragment 

391 

392 @property 

393 def params(self) -> str: 

394 """Return any parameters included in the URI.""" 

395 return self._uri.params 

396 

397 @property 

398 def query(self) -> str: 

399 """Return any query strings included in the URI.""" 

400 return self._uri.query 

401 

402 def geturl(self) -> str: 

403 """Return the URI in string form. 

404 

405 Returns 

406 ------- 

407 url : `str` 

408 String form of URI. 

409 """ 

410 return self._uri.geturl() 

411 

412 def root_uri(self) -> ResourcePath: 

413 """Return the base root URI. 

414 

415 Returns 

416 ------- 

417 uri : `ResourcePath` 

418 Root URI. 

419 """ 

420 return self.replace(path="", query="", fragment="", params="", forceDirectory=True) 

421 

422 def split(self) -> tuple[ResourcePath, str]: 

423 """Split URI into head and tail. 

424 

425 Returns 

426 ------- 

427 head: `ResourcePath` 

428 Everything leading up to tail, expanded and normalized as per 

429 ResourcePath rules. 

430 tail : `str` 

431 Last path component. Tail will be empty if path ends on a 

432 separator or if the URI is known to be associated with a directory. 

433 Tail will never contain separators. It will be unquoted. 

434 

435 Notes 

436 ----- 

437 Equivalent to `os.path.split` where head preserves the URI 

438 components. In some cases this method can result in a file system 

439 check to verify whether the URI is a directory or not (only if 

440 ``forceDirectory`` was `None` during construction). For a scheme-less 

441 URI this can mean that the result might change depending on current 

442 working directory. 

443 """ 

444 if self.isdir(): 

445 # This is known to be a directory so must return itself and 

446 # the empty string. 

447 return self, "" 

448 

449 head, tail = self._pathModule.split(self.path) 

450 headuri = self._uri._replace(path=head) 

451 

452 # The file part should never include quoted metacharacters 

453 tail = urllib.parse.unquote(tail) 

454 

455 # Schemeless is special in that it can be a relative path. 

456 # We need to ensure that it stays that way. All other URIs will 

457 # be absolute already. 

458 forceAbsolute = self.isabs() 

459 return ResourcePath(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

460 

461 def basename(self) -> str: 

462 """Return the base name, last element of path, of the URI. 

463 

464 Returns 

465 ------- 

466 tail : `str` 

467 Last part of the path attribute. Trail will be empty if path ends 

468 on a separator. 

469 

470 Notes 

471 ----- 

472 If URI ends on a slash returns an empty string. This is the second 

473 element returned by `split()`. 

474 

475 Equivalent of `os.path.basename`. 

476 """ 

477 return self.split()[1] 

478 

479 def dirname(self) -> ResourcePath: 

480 """Return the directory component of the path as a new `ResourcePath`. 

481 

482 Returns 

483 ------- 

484 head : `ResourcePath` 

485 Everything except the tail of path attribute, expanded and 

486 normalized as per ResourcePath rules. 

487 

488 Notes 

489 ----- 

490 Equivalent of `os.path.dirname`. If this is a directory URI it will 

491 be returned unchanged. If the parent directory is always required 

492 use `parent`. 

493 """ 

494 return self.split()[0] 

495 

496 def parent(self) -> ResourcePath: 

497 """Return a `ResourcePath` of the parent directory. 

498 

499 Returns 

500 ------- 

501 head : `ResourcePath` 

502 Everything except the tail of path attribute, expanded and 

503 normalized as per `ResourcePath` rules. 

504 

505 Notes 

506 ----- 

507 For a file-like URI this will be the same as calling `dirname`. 

508 For a directory-like URI this will always return the parent directory 

509 whereas `dirname()` will return the original URI. This is consistent 

510 with `os.path.dirname` compared to the `pathlib.Path` property 

511 ``parent``. 

512 """ 

513 if self.dirLike is False: 

514 # os.path.split() is slightly faster than calling Path().parent. 

515 return self.dirname() 

516 # When self is dir-like, returns its parent directory, 

517 # regardless of the presence of a trailing separator 

518 originalPath = self._pathLib(self.path) 

519 parentPath = originalPath.parent 

520 return self.replace(path=str(parentPath), forceDirectory=True) 

521 

522 def replace( 

523 self, forceDirectory: bool | None = None, isTemporary: bool = False, **kwargs: Any 

524 ) -> ResourcePath: 

525 """Return new `ResourcePath` with specified components replaced. 

526 

527 Parameters 

528 ---------- 

529 forceDirectory : `bool` or `None`, optional 

530 Parameter passed to ResourcePath constructor to force this 

531 new URI to be dir-like or file-like. 

532 isTemporary : `bool`, optional 

533 Indicate that the resulting URI is temporary resource. 

534 **kwargs 

535 Components of a `urllib.parse.ParseResult` that should be 

536 modified for the newly-created `ResourcePath`. 

537 

538 Returns 

539 ------- 

540 new : `ResourcePath` 

541 New `ResourcePath` object with updated values. 

542 

543 Notes 

544 ----- 

545 Does not, for now, allow a change in URI scheme. 

546 """ 

547 # Disallow a change in scheme 

548 if "scheme" in kwargs: 

549 raise ValueError(f"Can not use replace() method to change URI scheme for {self}") 

550 return self.__class__( 

551 self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary 

552 ) 

553 

554 def updatedFile(self, newfile: str) -> ResourcePath: 

555 """Return new URI with an updated final component of the path. 

556 

557 Parameters 

558 ---------- 

559 newfile : `str` 

560 File name with no path component. 

561 

562 Returns 

563 ------- 

564 updated : `ResourcePath` 

565 Updated `ResourcePath` with new updated final component. 

566 

567 Notes 

568 ----- 

569 Forces the ``ResourcePath.dirLike`` attribute to be false. The new file 

570 path will be quoted if necessary. If the current URI is known to 

571 refer to a directory, the new file will be joined to the current file. 

572 It is recommended that this behavior no longer be used and a call 

573 to `isdir` by the caller should be used to decide whether to join or 

574 replace. In the future this method may be modified to always replace 

575 the final element of the path. 

576 """ 

577 if self.dirLike: 

578 return self.join(newfile, forceDirectory=False) 

579 return self.parent().join(newfile, forceDirectory=False) 

580 

581 def updatedExtension(self, ext: str | None) -> ResourcePath: 

582 """Return a new `ResourcePath` with updated file extension. 

583 

584 All file extensions are replaced. 

585 

586 Parameters 

587 ---------- 

588 ext : `str` or `None` 

589 New extension. If an empty string is given any extension will 

590 be removed. If `None` is given there will be no change. 

591 

592 Returns 

593 ------- 

594 updated : `ResourcePath` 

595 URI with the specified extension. Can return itself if 

596 no extension was specified. 

597 """ 

598 if ext is None: 

599 return self 

600 

601 # Get the extension 

602 current = self.getExtension() 

603 

604 # Nothing to do if the extension already matches 

605 if current == ext: 

606 return self 

607 

608 # Remove the current extension from the path 

609 # .fits.gz counts as one extension do not use os.path.splitext 

610 path = self.path 

611 if current: 

612 path = path.removesuffix(current) 

613 

614 # Ensure that we have a leading "." on file extension (and we do not 

615 # try to modify the empty string) 

616 if ext and not ext.startswith("."): 

617 ext = "." + ext 

618 

619 return self.replace(path=path + ext, forceDirectory=False) 

620 

621 def getExtension(self) -> str: 

622 """Return the extension(s) associated with this URI path. 

623 

624 Returns 

625 ------- 

626 ext : `str` 

627 The file extension (including the ``.``). Can be empty string 

628 if there is no file extension. Usually returns only the last 

629 file extension unless there is a special extension modifier 

630 indicating file compression, in which case the combined 

631 extension (e.g. ``.fits.gz``) will be returned. 

632 

633 Notes 

634 ----- 

635 Does not distinguish between file and directory URIs when determining 

636 a suffix. An extension is only determined from the final component 

637 of the path. 

638 """ 

639 special = {".gz", ".bz2", ".xz", ".fz"} 

640 

641 # path lib will ignore any "." in directories. 

642 # path lib works well: 

643 # extensions = self._pathLib(self.path).suffixes 

644 # But the constructor is slow. Therefore write our own implementation. 

645 # Strip trailing separator if present, do not care if this is a 

646 # directory or not. 

647 parts = self.path.rstrip("/").rsplit(self._pathModule.sep, 1) 

648 _, *extensions = parts[-1].split(".") 

649 

650 if not extensions: 

651 return "" 

652 extensions = ["." + x for x in extensions] 

653 

654 ext = extensions.pop() 

655 

656 # Multiple extensions, decide whether to include the final two 

657 if extensions and ext in special: 

658 ext = f"{extensions[-1]}{ext}" 

659 

660 return ext 

661 

662 def join( 

663 self, path: str | ResourcePath, isTemporary: bool | None = None, forceDirectory: bool | None = None 

664 ) -> ResourcePath: 

665 """Return new `ResourcePath` with additional path components. 

666 

667 Parameters 

668 ---------- 

669 path : `str`, `ResourcePath` 

670 Additional file components to append to the current URI. Will be 

671 quoted depending on the associated URI scheme. If the path looks 

672 like a URI referring to an absolute location, it will be returned 

673 directly (matching the behavior of `os.path.join`). It can 

674 also be a `ResourcePath`. 

675 isTemporary : `bool`, optional 

676 Indicate that the resulting URI represents a temporary resource. 

677 Default is ``self.isTemporary``. 

678 forceDirectory : `bool` or `None`, optional 

679 If `True` forces the URI to end with a separator. If `False` the 

680 resultant URI is declared to refer to a file. `None` indicates 

681 that the file directory status is unknown. 

682 

683 Returns 

684 ------- 

685 new : `ResourcePath` 

686 New URI with the path appended. 

687 

688 Notes 

689 ----- 

690 Schemeless URIs assume local path separator but all other URIs assume 

691 POSIX separator if the supplied path has directory structure. It 

692 may be this never becomes a problem but datastore templates assume 

693 POSIX separator is being used. 

694 

695 If an absolute `ResourcePath` is given for ``path`` is is assumed that 

696 this should be returned directly. Giving a ``path`` of an absolute 

697 scheme-less URI is not allowed for safety reasons as it may indicate 

698 a mistake in the calling code. 

699 

700 It is an error to attempt to join to something that is known to 

701 refer to a file. Use `updatedFile` if the file is to be 

702 replaced. 

703 

704 Raises 

705 ------ 

706 ValueError 

707 Raised if the given path object refers to a directory but the 

708 ``forceDirectory`` parameter insists the outcome should be a file, 

709 and vice versa. Also raised if the URI being joined with is known 

710 to refer to a file. 

711 RuntimeError 

712 Raised if this attempts to join a temporary URI to a non-temporary 

713 URI. 

714 """ 

715 if self.dirLike is False: 

716 raise ValueError("Can not join a new path component to a file.") 

717 if isTemporary is None: 

718 isTemporary = self.isTemporary 

719 elif not isTemporary and self.isTemporary: 

720 raise RuntimeError("Cannot join temporary URI to non-temporary URI.") 

721 # If we have a full URI in path we will use it directly 

722 # but without forcing to absolute so that we can trap the 

723 # expected option of relative path. 

724 path_uri = ResourcePath( 

725 path, forceAbsolute=False, forceDirectory=forceDirectory, isTemporary=isTemporary 

726 ) 

727 if forceDirectory is not None and path_uri.dirLike is not forceDirectory: 

728 raise ValueError( 

729 "The supplied path URI to join has inconsistent directory state " 

730 f"with forceDirectory parameter: {path_uri.dirLike} vs {forceDirectory}" 

731 ) 

732 forceDirectory = path_uri.dirLike 

733 

734 if path_uri.isabs(): 

735 # Absolute URI so return it directly. 

736 return path_uri 

737 

738 # If this was originally a ResourcePath extract the unquoted path from 

739 # it. Otherwise we use the string we were given to allow "#" to appear 

740 # in the filename if given as a plain string. 

741 if not isinstance(path, str): 

742 path = path_uri.unquoted_path 

743 

744 # Might need to quote the path. 

745 if self.quotePaths: 

746 path = urllib.parse.quote(path) 

747 

748 newpath = self._pathModule.normpath(self._pathModule.join(self.path, path)) 

749 

750 # normpath can strip trailing / so we force directory if the supplied 

751 # path ended with a / 

752 has_dir_sep = path.endswith(self._pathModule.sep) 

753 if forceDirectory is None and has_dir_sep: 

754 forceDirectory = True 

755 elif forceDirectory is False and has_dir_sep: 

756 raise ValueError("Path to join has trailing / but is being forced to be a file.") 

757 return self.replace( 

758 path=newpath, 

759 forceDirectory=forceDirectory, 

760 isTemporary=isTemporary, 

761 ) 

762 

763 def relative_to(self, other: ResourcePath) -> str | None: 

764 """Return the relative path from this URI to the other URI. 

765 

766 Parameters 

767 ---------- 

768 other : `ResourcePath` 

769 URI to use to calculate the relative path. Must be a parent 

770 of this URI. 

771 

772 Returns 

773 ------- 

774 subpath : `str` 

775 The sub path of this URI relative to the supplied other URI. 

776 Returns `None` if there is no parent child relationship. 

777 Scheme and netloc must match. 

778 """ 

779 # Scheme-less self is handled elsewhere. 

780 if self.scheme != other.scheme: 

781 return None 

782 if self.netloc != other.netloc: 

783 # Special case for localhost vs empty string. 

784 # There can be many variants of localhost. 

785 local_netlocs = {"", "localhost", "localhost.localdomain", "127.0.0.1"} 

786 if not {self.netloc, other.netloc}.issubset(local_netlocs): 

787 return None 

788 

789 enclosed_path = self._pathLib(self.relativeToPathRoot) 

790 parent_path = other.relativeToPathRoot 

791 subpath: str | None 

792 try: 

793 subpath = str(enclosed_path.relative_to(parent_path)) 

794 except ValueError: 

795 subpath = None 

796 else: 

797 subpath = urllib.parse.unquote(subpath) 

798 return subpath 

799 

800 def exists(self) -> bool: 

801 """Indicate that the resource is available. 

802 

803 Returns 

804 ------- 

805 exists : `bool` 

806 `True` if the resource exists. 

807 """ 

808 raise NotImplementedError() 

809 

810 @classmethod 

811 def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

812 """Check for existence of multiple URIs at once. 

813 

814 Parameters 

815 ---------- 

816 uris : iterable of `ResourcePath` 

817 The URIs to test. 

818 

819 Returns 

820 ------- 

821 existence : `dict` of [`ResourcePath`, `bool`] 

822 Mapping of original URI to boolean indicating existence. 

823 """ 

824 # Group by scheme to allow a subclass to be able to use 

825 # specialized implementations. 

826 grouped: dict[type, list[ResourcePath]] = {} 

827 for uri in uris: 

828 uri_class = uri.__class__ 

829 if uri_class not in grouped: 

830 grouped[uri_class] = [] 

831 grouped[uri_class].append(uri) 

832 

833 existence: dict[ResourcePath, bool] = {} 

834 for uri_class in grouped: 

835 existence.update(uri_class._mexists(grouped[uri_class])) 

836 

837 return existence 

838 

839 @classmethod 

840 def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]: 

841 """Check for existence of multiple URIs at once. 

842 

843 Implementation helper method for `mexists`. 

844 

845 Parameters 

846 ---------- 

847 uris : iterable of `ResourcePath` 

848 The URIs to test. 

849 

850 Returns 

851 ------- 

852 existence : `dict` of [`ResourcePath`, `bool`] 

853 Mapping of original URI to boolean indicating existence. 

854 """ 

855 exists_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 

856 future_exists = {exists_executor.submit(uri.exists): uri for uri in uris} 

857 

858 results: dict[ResourcePath, bool] = {} 

859 for future in concurrent.futures.as_completed(future_exists): 

860 uri = future_exists[future] 

861 try: 

862 exists = future.result() 

863 except Exception: 

864 exists = False 

865 results[uri] = exists 

866 return results 

867 

868 def remove(self) -> None: 

869 """Remove the resource.""" 

870 raise NotImplementedError() 

871 

872 def isabs(self) -> bool: 

873 """Indicate that the resource is fully specified. 

874 

875 For non-schemeless URIs this is always true. 

876 

877 Returns 

878 ------- 

879 isabs : `bool` 

880 `True` in all cases except schemeless URI. 

881 """ 

882 return True 

883 

884 def abspath(self) -> ResourcePath: 

885 """Return URI using an absolute path. 

886 

887 Returns 

888 ------- 

889 abs : `ResourcePath` 

890 Absolute URI. For non-schemeless URIs this always returns itself. 

891 Schemeless URIs are upgraded to file URIs. 

892 """ 

893 return self 

894 

895 def _as_local(self) -> tuple[str, bool]: 

896 """Return the location of the (possibly remote) resource as local file. 

897 

898 This is a helper function for `as_local` context manager. 

899 

900 Returns 

901 ------- 

902 path : `str` 

903 If this is a remote resource, it will be a copy of the resource 

904 on the local file system, probably in a temporary directory. 

905 For a local resource this should be the actual path to the 

906 resource. 

907 is_temporary : `bool` 

908 Indicates if the local path is a temporary file or not. 

909 """ 

910 raise NotImplementedError() 

911 

912 @contextlib.contextmanager 

913 def as_local(self) -> Iterator[ResourcePath]: 

914 """Return the location of the (possibly remote) resource as local file. 

915 

916 Yields 

917 ------ 

918 local : `ResourcePath` 

919 If this is a remote resource, it will be a copy of the resource 

920 on the local file system, probably in a temporary directory. 

921 For a local resource this should be the actual path to the 

922 resource. 

923 

924 Notes 

925 ----- 

926 The context manager will automatically delete any local temporary 

927 file. 

928 

929 Examples 

930 -------- 

931 Should be used as a context manager: 

932 

933 .. code-block:: py 

934 

935 with uri.as_local() as local: 

936 ospath = local.ospath 

937 """ 

938 if self.isdir(): 

939 raise IsADirectoryError(f"Directory-like URI {self} cannot be fetched as local.") 

940 local_src, is_temporary = self._as_local() 

941 local_uri = ResourcePath(local_src, isTemporary=is_temporary) 

942 

943 try: 

944 yield local_uri 

945 finally: 

946 # The caller might have relocated the temporary file. 

947 # Do not ever delete if the temporary matches self 

948 # (since it may have been that a temporary file was made local 

949 # but already was local). 

950 if self != local_uri and is_temporary and local_uri.exists(): 

951 local_uri.remove() 

952 

953 @classmethod 

954 @contextlib.contextmanager 

955 def temporary_uri( 

956 cls, prefix: ResourcePath | None = None, suffix: str | None = None 

957 ) -> Iterator[ResourcePath]: 

958 """Create a temporary file-like URI. 

959 

960 Parameters 

961 ---------- 

962 prefix : `ResourcePath`, optional 

963 Prefix to use. Without this the path will be formed as a local 

964 file URI in a temporary directory. Ensuring that the prefix 

965 location exists is the responsibility of the caller. 

966 suffix : `str`, optional 

967 A file suffix to be used. The ``.`` should be included in this 

968 suffix. 

969 

970 Yields 

971 ------ 

972 uri : `ResourcePath` 

973 The temporary URI. Will be removed when the context is completed. 

974 """ 

975 use_tempdir = False 

976 if prefix is None: 

977 directory = tempfile.mkdtemp() 

978 # If the user has set a umask that restricts the owner-write bit, 

979 # the directory returned from mkdtemp may not initially be 

980 # writeable by us 

981 ensure_directory_is_writeable(directory) 

982 

983 prefix = ResourcePath(directory, forceDirectory=True, isTemporary=True) 

984 # Record that we need to delete this directory. Can not rely 

985 # on isTemporary flag since an external prefix may have that 

986 # set as well. 

987 use_tempdir = True 

988 

989 # Need to create a randomized file name. For consistency do not 

990 # use mkstemp for local and something else for remote. Additionally 

991 # this method does not create the file to prevent name clashes. 

992 characters = "abcdefghijklmnopqrstuvwxyz0123456789_" 

993 rng = Random() 

994 tempname = "".join(rng.choice(characters) for _ in range(16)) 

995 if suffix: 

996 tempname += suffix 

997 temporary_uri = prefix.join(tempname, isTemporary=True) 

998 if temporary_uri.isdir(): 

999 # If we had a safe way to clean up a remote temporary directory, we 

1000 # could support this. 

1001 raise NotImplementedError("temporary_uri cannot be used to create a temporary directory.") 

1002 try: 

1003 yield temporary_uri 

1004 finally: 

1005 if use_tempdir: 

1006 shutil.rmtree(prefix.ospath, ignore_errors=True) 

1007 else: 

1008 with contextlib.suppress(FileNotFoundError): 

1009 # It's okay if this does not work because the user removed 

1010 # the file. 

1011 temporary_uri.remove() 

1012 

1013 def read(self, size: int = -1) -> bytes: 

1014 """Open the resource and return the contents in bytes. 

1015 

1016 Parameters 

1017 ---------- 

1018 size : `int`, optional 

1019 The number of bytes to read. Negative or omitted indicates 

1020 that all data should be read. 

1021 """ 

1022 raise NotImplementedError() 

1023 

1024 def write(self, data: bytes, overwrite: bool = True) -> None: 

1025 """Write the supplied bytes to the new resource. 

1026 

1027 Parameters 

1028 ---------- 

1029 data : `bytes` 

1030 The bytes to write to the resource. The entire contents of the 

1031 resource will be replaced. 

1032 overwrite : `bool`, optional 

1033 If `True` the resource will be overwritten if it exists. Otherwise 

1034 the write will fail. 

1035 """ 

1036 raise NotImplementedError() 

1037 

1038 def mkdir(self) -> None: 

1039 """For a dir-like URI, create the directory resource if needed.""" 

1040 raise NotImplementedError() 

1041 

1042 def isdir(self) -> bool: 

1043 """Return True if this URI looks like a directory, else False.""" 

1044 return bool(self.dirLike) 

1045 

1046 def size(self) -> int: 

1047 """For non-dir-like URI, return the size of the resource. 

1048 

1049 Returns 

1050 ------- 

1051 sz : `int` 

1052 The size in bytes of the resource associated with this URI. 

1053 Returns 0 if dir-like. 

1054 """ 

1055 raise NotImplementedError() 

1056 

1057 def __str__(self) -> str: 

1058 """Convert the URI to its native string form.""" 

1059 return self.geturl() 

1060 

1061 def __repr__(self) -> str: 

1062 """Return string representation suitable for evaluation.""" 

1063 return f'ResourcePath("{self.geturl()}")' 

1064 

1065 def __eq__(self, other: Any) -> bool: 

1066 """Compare supplied object with this `ResourcePath`.""" 

1067 if not isinstance(other, ResourcePath): 

1068 return NotImplemented 

1069 return self.geturl() == other.geturl() 

1070 

1071 def __hash__(self) -> int: 

1072 """Return hash of this object.""" 

1073 return hash(str(self)) 

1074 

1075 def __lt__(self, other: ResourcePath) -> bool: 

1076 return self.geturl() < other.geturl() 

1077 

1078 def __le__(self, other: ResourcePath) -> bool: 

1079 return self.geturl() <= other.geturl() 

1080 

1081 def __gt__(self, other: ResourcePath) -> bool: 

1082 return self.geturl() > other.geturl() 

1083 

1084 def __ge__(self, other: ResourcePath) -> bool: 

1085 return self.geturl() >= other.geturl() 

1086 

1087 def __copy__(self) -> ResourcePath: 

1088 """Copy constructor. 

1089 

1090 Object is immutable so copy can return itself. 

1091 """ 

1092 # Implement here because the __new__ method confuses things 

1093 return self 

1094 

1095 def __deepcopy__(self, memo: Any) -> ResourcePath: 

1096 """Deepcopy the object. 

1097 

1098 Object is immutable so copy can return itself. 

1099 """ 

1100 # Implement here because the __new__ method confuses things 

1101 return self 

1102 

1103 def __getnewargs__(self) -> tuple: 

1104 """Support pickling.""" 

1105 return (str(self),) 

1106 

1107 @classmethod 

1108 def _fixDirectorySep( 

1109 cls, parsed: urllib.parse.ParseResult, forceDirectory: bool | None = None 

1110 ) -> tuple[urllib.parse.ParseResult, bool | None]: 

1111 """Ensure that a path separator is present on directory paths. 

1112 

1113 Parameters 

1114 ---------- 

1115 parsed : `~urllib.parse.ParseResult` 

1116 The result from parsing a URI using `urllib.parse`. 

1117 forceDirectory : `bool` or `None`, optional 

1118 If `True` forces the URI to end with a separator, otherwise given 

1119 URI is interpreted as is. Specifying that the URI is conceptually 

1120 equivalent to a directory can break some ambiguities when 

1121 interpreting the last element of a path. 

1122 

1123 Returns 

1124 ------- 

1125 modified : `~urllib.parse.ParseResult` 

1126 Update result if a URI is being handled. 

1127 dirLike : `bool` or `None` 

1128 `True` if given parsed URI has a trailing separator or 

1129 ``forceDirectory`` is `True`. Otherwise returns the given value of 

1130 ``forceDirectory``. 

1131 """ 

1132 # Assume the forceDirectory flag can give us a clue. 

1133 dirLike = forceDirectory 

1134 

1135 # Directory separator 

1136 sep = cls._pathModule.sep 

1137 

1138 # URI is dir-like if explicitly stated or if it ends on a separator 

1139 endsOnSep = parsed.path.endswith(sep) 

1140 

1141 if forceDirectory is False and endsOnSep: 

1142 raise ValueError( 

1143 f"URI {parsed.geturl()} ends with {sep} but " 

1144 "forceDirectory parameter declares it to be a file." 

1145 ) 

1146 

1147 if forceDirectory or endsOnSep: 

1148 dirLike = True 

1149 # only add the separator if it's not already there 

1150 if not endsOnSep: 

1151 parsed = parsed._replace(path=parsed.path + sep) 

1152 

1153 return parsed, dirLike 

1154 

1155 @classmethod 

1156 def _fixupPathUri( 

1157 cls, 

1158 parsed: urllib.parse.ParseResult, 

1159 root: ResourcePath | None = None, 

1160 forceAbsolute: bool = False, 

1161 forceDirectory: bool | None = None, 

1162 ) -> tuple[urllib.parse.ParseResult, bool | None]: 

1163 """Correct any issues with the supplied URI. 

1164 

1165 Parameters 

1166 ---------- 

1167 parsed : `~urllib.parse.ParseResult` 

1168 The result from parsing a URI using `urllib.parse`. 

1169 root : `ResourcePath`, ignored 

1170 Not used by the this implementation since all URIs are 

1171 absolute except for those representing the local file system. 

1172 forceAbsolute : `bool`, ignored. 

1173 Not used by this implementation. URIs are generally always 

1174 absolute. 

1175 forceDirectory : `bool` or `None`, optional 

1176 If `True` forces the URI to end with a separator, otherwise given 

1177 URI is interpreted as is. Specifying that the URI is conceptually 

1178 equivalent to a directory can break some ambiguities when 

1179 interpreting the last element of a path. 

1180 

1181 Returns 

1182 ------- 

1183 modified : `~urllib.parse.ParseResult` 

1184 Update result if a URI is being handled. 

1185 dirLike : `bool` 

1186 `True` if given parsed URI has a trailing separator or 

1187 ``forceDirectory`` is `True`. Otherwise returns the given value 

1188 of ``forceDirectory``. 

1189 

1190 Notes 

1191 ----- 

1192 Relative paths are explicitly not supported by RFC8089 but `urllib` 

1193 does accept URIs of the form ``file:relative/path.ext``. They need 

1194 to be turned into absolute paths before they can be used. This is 

1195 always done regardless of the ``forceAbsolute`` parameter. 

1196 

1197 AWS S3 differentiates between keys with trailing POSIX separators (i.e 

1198 ``/dir`` and ``/dir/``) whereas POSIX does not necessarily. 

1199 

1200 Scheme-less paths are normalized. 

1201 """ 

1202 return cls._fixDirectorySep(parsed, forceDirectory) 

1203 

1204 def transfer_from( 

1205 self, 

1206 src: ResourcePath, 

1207 transfer: str, 

1208 overwrite: bool = False, 

1209 transaction: TransactionProtocol | None = None, 

1210 ) -> None: 

1211 """Transfer to this URI from another. 

1212 

1213 Parameters 

1214 ---------- 

1215 src : `ResourcePath` 

1216 Source URI. 

1217 transfer : `str` 

1218 Mode to use for transferring the resource. Generically there are 

1219 many standard options: copy, link, symlink, hardlink, relsymlink. 

1220 Not all URIs support all modes. 

1221 overwrite : `bool`, optional 

1222 Allow an existing file to be overwritten. Defaults to `False`. 

1223 transaction : `~lsst.resources.utils.TransactionProtocol`, optional 

1224 A transaction object that can (depending on implementation) 

1225 rollback transfers on error. Not guaranteed to be implemented. 

1226 

1227 Notes 

1228 ----- 

1229 Conceptually this is hard to scale as the number of URI schemes 

1230 grow. The destination URI is more important than the source URI 

1231 since that is where all the transfer modes are relevant (with the 

1232 complication that "move" deletes the source). 

1233 

1234 Local file to local file is the fundamental use case but every 

1235 other scheme has to support "copy" to local file (with implicit 

1236 support for "move") and copy from local file. 

1237 All the "link" options tend to be specific to local file systems. 

1238 

1239 "move" is a "copy" where the remote resource is deleted at the end. 

1240 Whether this works depends on the source URI rather than the 

1241 destination URI. Reverting a move on transaction rollback is 

1242 expected to be problematic if a remote resource was involved. 

1243 """ 

1244 raise NotImplementedError(f"No transfer modes supported by URI scheme {self.scheme}") 

1245 

1246 def walk( 

1247 self, file_filter: str | re.Pattern | None = None 

1248 ) -> Iterator[list | tuple[ResourcePath, list[str], list[str]]]: 

1249 """Walk the directory tree returning matching files and directories. 

1250 

1251 Parameters 

1252 ---------- 

1253 file_filter : `str` or `re.Pattern`, optional 

1254 Regex to filter out files from the list before it is returned. 

1255 

1256 Yields 

1257 ------ 

1258 dirpath : `ResourcePath` 

1259 Current directory being examined. 

1260 dirnames : `list` of `str` 

1261 Names of subdirectories within dirpath. 

1262 filenames : `list` of `str` 

1263 Names of all the files within dirpath. 

1264 """ 

1265 raise NotImplementedError() 

1266 

1267 @overload 

1268 @classmethod 

1269 def findFileResources( 1269 ↛ exitline 1269 didn't jump to the function exit

1270 cls, 

1271 candidates: Iterable[ResourcePathExpression], 

1272 file_filter: str | re.Pattern | None, 

1273 grouped: Literal[True], 

1274 ) -> Iterator[Iterator[ResourcePath]]: ... 

1275 

1276 @overload 

1277 @classmethod 

1278 def findFileResources( 1278 ↛ exitline 1278 didn't jump to the function exit

1279 cls, 

1280 candidates: Iterable[ResourcePathExpression], 

1281 *, 

1282 grouped: Literal[True], 

1283 ) -> Iterator[Iterator[ResourcePath]]: ... 

1284 

1285 @overload 

1286 @classmethod 

1287 def findFileResources( 1287 ↛ exitline 1287 didn't jump to the function exit

1288 cls, 

1289 candidates: Iterable[ResourcePathExpression], 

1290 file_filter: str | re.Pattern | None = None, 

1291 grouped: Literal[False] = False, 

1292 ) -> Iterator[ResourcePath]: ... 

1293 

1294 @classmethod 

1295 def findFileResources( 

1296 cls, 

1297 candidates: Iterable[ResourcePathExpression], 

1298 file_filter: str | re.Pattern | None = None, 

1299 grouped: bool = False, 

1300 ) -> Iterator[ResourcePath | Iterator[ResourcePath]]: 

1301 """Get all the files from a list of values. 

1302 

1303 Parameters 

1304 ---------- 

1305 candidates : iterable [`str` or `ResourcePath`] 

1306 The files to return and directories in which to look for files to 

1307 return. 

1308 file_filter : `str` or `re.Pattern`, optional 

1309 The regex to use when searching for files within directories. 

1310 By default returns all the found files. 

1311 grouped : `bool`, optional 

1312 If `True` the results will be grouped by directory and each 

1313 yielded value will be an iterator over URIs. If `False` each 

1314 URI will be returned separately. 

1315 

1316 Yields 

1317 ------ 

1318 found_file: `ResourcePath` 

1319 The passed-in URIs and URIs found in passed-in directories. 

1320 If grouping is enabled, each of the yielded values will be an 

1321 iterator yielding members of the group. Files given explicitly 

1322 will be returned as a single group at the end. 

1323 

1324 Notes 

1325 ----- 

1326 If a value is a file it is yielded immediately without checking that it 

1327 exists. If a value is a directory, all the files in the directory 

1328 (recursively) that match the regex will be yielded in turn. 

1329 """ 

1330 fileRegex = None if file_filter is None else re.compile(file_filter) 

1331 

1332 singles = [] 

1333 

1334 # Find all the files of interest 

1335 for location in candidates: 

1336 uri = ResourcePath(location) 

1337 if uri.isdir(): 

1338 for found in uri.walk(fileRegex): 

1339 if not found: 

1340 # This means the uri does not exist and by 

1341 # convention we ignore it 

1342 continue 

1343 root, dirs, files = found 

1344 if not files: 

1345 continue 

1346 if grouped: 

1347 yield (root.join(name) for name in files) 

1348 else: 

1349 for name in files: 

1350 yield root.join(name) 

1351 else: 

1352 if grouped: 

1353 singles.append(uri) 

1354 else: 

1355 yield uri 

1356 

1357 # Finally, return any explicitly given files in one group 

1358 if grouped and singles: 

1359 yield iter(singles) 

1360 

1361 @contextlib.contextmanager 

1362 def open( 

1363 self, 

1364 mode: str = "r", 

1365 *, 

1366 encoding: str | None = None, 

1367 prefer_file_temporary: bool = False, 

1368 ) -> Iterator[ResourceHandleProtocol]: 

1369 """Return a context manager that wraps an object that behaves like an 

1370 open file at the location of the URI. 

1371 

1372 Parameters 

1373 ---------- 

1374 mode : `str` 

1375 String indicating the mode in which to open the file. Values are 

1376 the same as those accepted by `open`, though intrinsically 

1377 read-only URI types may only support read modes, and 

1378 `io.IOBase.seekable` is not guaranteed to be `True` on the returned 

1379 object. 

1380 encoding : `str`, optional 

1381 Unicode encoding for text IO; ignored for binary IO. Defaults to 

1382 ``locale.getpreferredencoding(False)``, just as `open` 

1383 does. 

1384 prefer_file_temporary : `bool`, optional 

1385 If `True`, for implementations that require transfers from a remote 

1386 system to temporary local storage and/or back, use a temporary file 

1387 instead of an in-memory buffer; this is generally slower, but it 

1388 may be necessary to avoid excessive memory usage by large files. 

1389 Ignored by implementations that do not require a temporary. 

1390 

1391 Yields 

1392 ------ 

1393 cm : `~contextlib.AbstractContextManager` 

1394 A context manager that wraps a `ResourceHandleProtocol` file-like 

1395 object. 

1396 

1397 Notes 

1398 ----- 

1399 The default implementation of this method uses a local temporary buffer 

1400 (in-memory or file, depending on ``prefer_file_temporary``) with calls 

1401 to `read`, `write`, `as_local`, and `transfer_from` as necessary to 

1402 read and write from/to remote systems. Remote writes thus occur only 

1403 when the context manager is exited. `ResourcePath` implementations 

1404 that can return a more efficient native buffer should do so whenever 

1405 possible (as is guaranteed for local files). `ResourcePath` 

1406 implementations for which `as_local` does not return a temporary are 

1407 required to reimplement `open`, though they may delegate to `super` 

1408 when ``prefer_file_temporary`` is `False`. 

1409 """ 

1410 if self.isdir(): 

1411 raise IsADirectoryError(f"Directory-like URI {self} cannot be opened.") 

1412 if "x" in mode and self.exists(): 

1413 raise FileExistsError(f"File at {self} already exists.") 

1414 if prefer_file_temporary: 

1415 if "r" in mode or "a" in mode: 

1416 local_cm = self.as_local() 

1417 else: 

1418 local_cm = self.temporary_uri(suffix=self.getExtension()) 

1419 with local_cm as local_uri: 

1420 assert local_uri.isTemporary, ( 

1421 "ResourcePath implementations for which as_local is not " 

1422 "a temporary must reimplement `open`." 

1423 ) 

1424 with open(local_uri.ospath, mode=mode, encoding=encoding) as file_buffer: 

1425 if "a" in mode: 

1426 file_buffer.seek(0, io.SEEK_END) 

1427 yield file_buffer 

1428 if "r" not in mode or "+" in mode: 

1429 self.transfer_from(local_uri, transfer="copy", overwrite=("x" not in mode)) 

1430 else: 

1431 with self._openImpl(mode, encoding=encoding) as handle: 

1432 yield handle 

1433 

1434 @contextlib.contextmanager 

1435 def _openImpl(self, mode: str = "r", *, encoding: str | None = None) -> Iterator[ResourceHandleProtocol]: 

1436 """Implement opening of a resource handle. 

1437 

1438 This private method may be overridden by specific `ResourcePath` 

1439 implementations to provide a customized handle like interface. 

1440 

1441 Parameters 

1442 ---------- 

1443 mode : `str` 

1444 The mode the handle should be opened with 

1445 encoding : `str`, optional 

1446 The byte encoding of any binary text 

1447 

1448 Yields 

1449 ------ 

1450 handle : `~._resourceHandles.BaseResourceHandle` 

1451 A handle that conforms to the 

1452 `~._resourceHandles.BaseResourceHandle` interface 

1453 

1454 Notes 

1455 ----- 

1456 The base implementation of a file handle reads in a files entire 

1457 contents into a buffer for manipulation, and then writes it back out 

1458 upon close. Subclasses of this class may offer more fine grained 

1459 control. 

1460 """ 

1461 in_bytes = self.read() if "r" in mode or "a" in mode else b"" 

1462 if "b" in mode: 

1463 bytes_buffer = io.BytesIO(in_bytes) 

1464 if "a" in mode: 

1465 bytes_buffer.seek(0, io.SEEK_END) 

1466 yield bytes_buffer 

1467 out_bytes = bytes_buffer.getvalue() 

1468 else: 

1469 if encoding is None: 

1470 encoding = locale.getpreferredencoding(False) 

1471 str_buffer = io.StringIO(in_bytes.decode(encoding)) 

1472 if "a" in mode: 

1473 str_buffer.seek(0, io.SEEK_END) 

1474 yield str_buffer 

1475 out_bytes = str_buffer.getvalue().encode(encoding) 

1476 if "r" not in mode or "+" in mode: 

1477 self.write(out_bytes, overwrite=("x" not in mode)) 

1478 

1479 def generate_presigned_get_url(self, *, expiration_time_seconds: int) -> str: 

1480 """Return a pre-signed URL that can be used to retrieve this resource 

1481 using an HTTP GET without supplying any access credentials. 

1482 

1483 Parameters 

1484 ---------- 

1485 expiration_time_seconds : `int` 

1486 Number of seconds until the generated URL is no longer valid. 

1487 

1488 Returns 

1489 ------- 

1490 url : `str` 

1491 HTTP URL signed for GET. 

1492 """ 

1493 raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'") 

1494 

1495 def generate_presigned_put_url(self, *, expiration_time_seconds: int) -> str: 

1496 """Return a pre-signed URL that can be used to upload a file to this 

1497 path using an HTTP PUT without supplying any access credentials. 

1498 

1499 Parameters 

1500 ---------- 

1501 expiration_time_seconds : `int` 

1502 Number of seconds until the generated URL is no longer valid. 

1503 

1504 Returns 

1505 ------- 

1506 url : `str` 

1507 HTTP URL signed for PUT. 

1508 """ 

1509 raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'") 

1510 

1511 

1512ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path 

1513"""Type-annotation alias for objects that can be coerced to ResourcePath. 

1514"""