Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import contextlib 

25import urllib.parse 

26import posixpath 

27import copy 

28import logging 

29import re 

30 

31from pathlib import Path, PurePath, PurePosixPath 

32 

33__all__ = ('ButlerURI',) 

34 

35from typing import ( 

36 TYPE_CHECKING, 

37 Any, 

38 Iterator, 

39 Optional, 

40 Tuple, 

41 Type, 

42 Union, 

43) 

44 

45from .utils import NoTransaction 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from ..datastore import DatastoreTransaction 

49 

50 

51log = logging.getLogger(__name__) 

52 

53# Regex for looking for URI escapes 

54ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

55 

56# Precomputed escaped hash 

57ESCAPED_HASH = urllib.parse.quote("#") 

58 

59 

60class ButlerURI: 

61 """Convenience wrapper around URI parsers. 

62 

63 Provides access to URI components and can convert file 

64 paths into absolute path URIs. Scheme-less URIs are treated as if 

65 they are local file system paths and are converted to absolute URIs. 

66 

67 A specialist subclass is created for each supported URI scheme. 

68 

69 Parameters 

70 ---------- 

71 uri : `str` or `urllib.parse.ParseResult` 

72 URI in string form. Can be scheme-less if referring to a local 

73 filesystem path. 

74 root : `str` or `ButlerURI`, optional 

75 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

76 use this as the root. Must be absolute. If `None` the current 

77 working directory will be used. Can be a file URI. 

78 forceAbsolute : `bool`, optional 

79 If `True`, scheme-less relative URI will be converted to an absolute 

80 path using a ``file`` scheme. If `False` scheme-less URI will remain 

81 scheme-less and will not be updated to ``file`` or absolute path. 

82 forceDirectory: `bool`, optional 

83 If `True` forces the URI to end with a separator, otherwise given URI 

84 is interpreted as is. 

85 isTemporary : `bool`, optional 

86 If `True` indicates that this URI points to a temporary resource. 

87 """ 

88 

89 _pathLib: Type[PurePath] = PurePosixPath 

90 """Path library to use for this scheme.""" 

91 

92 _pathModule = posixpath 

93 """Path module to use for this scheme.""" 

94 

95 transferModes: Tuple[str, ...] = ("copy", "auto", "move") 

96 """Transfer modes supported by this implementation. 

97 

98 Move is special in that it is generally a copy followed by an unlink. 

99 Whether that unlink works depends critically on whether the source URI 

100 implements unlink. If it does not the move will be reported as a failure. 

101 """ 

102 

103 transferDefault: str = "copy" 

104 """Default mode to use for transferring if ``auto`` is specified.""" 

105 

106 quotePaths = True 

107 """True if path-like elements modifying a URI should be quoted. 

108 

109 All non-schemeless URIs have to internally use quoted paths. Therefore 

110 if a new file name is given (e.g. to updateFile or join) a decision must 

111 be made whether to quote it to be consistent. 

112 """ 

113 

114 isLocal = False 

115 """If `True` this URI refers to a local file.""" 

116 

117 # This is not an ABC with abstract methods because the __new__ being 

118 # a factory confuses mypy such that it assumes that every constructor 

119 # returns a ButlerURI and then determines that all the abstract methods 

120 # are still abstract. If they are not marked abstract but just raise 

121 # mypy is fine with it. 

122 

123 # mypy is confused without these 

124 _uri: urllib.parse.ParseResult 

125 isTemporary: bool 

126 

127 def __new__(cls, uri: Union[str, urllib.parse.ParseResult, ButlerURI, Path], 

128 root: Optional[Union[str, ButlerURI]] = None, forceAbsolute: bool = True, 

129 forceDirectory: bool = False, isTemporary: bool = False) -> ButlerURI: 

130 parsed: urllib.parse.ParseResult 

131 dirLike: bool = False 

132 subclass: Optional[Type] = None 

133 

134 if isinstance(uri, Path): 134 ↛ 135line 134 didn't jump to line 135, because the condition on line 134 was never true

135 uri = str(uri) 

136 

137 # Record if we need to post process the URI components 

138 # or if the instance is already fully configured 

139 if isinstance(uri, str): 

140 # Since local file names can have special characters in them 

141 # we need to quote them for the parser but we can unquote 

142 # later. Assume that all other URI schemes are quoted. 

143 # Since sometimes people write file:/a/b and not file:///a/b 

144 # we should not quote in the explicit case of file: 

145 if "://" not in uri and not uri.startswith("file:"): 

146 if ESCAPES_RE.search(uri): 146 ↛ 147line 146 didn't jump to line 147, because the condition on line 146 was never true

147 log.warning("Possible double encoding of %s", uri) 

148 else: 

149 uri = urllib.parse.quote(uri) 

150 # Special case hash since we must support fragments 

151 # even in schemeless URIs -- although try to only replace 

152 # them in file part and not directory part 

153 if ESCAPED_HASH in uri: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 dirpos = uri.rfind("/") 

155 # Do replacement after this / 

156 uri = uri[:dirpos+1] + uri[dirpos+1:].replace(ESCAPED_HASH, "#") 

157 

158 parsed = urllib.parse.urlparse(uri) 

159 elif isinstance(uri, urllib.parse.ParseResult): 

160 parsed = copy.copy(uri) 

161 elif isinstance(uri, ButlerURI): 161 ↛ 167line 161 didn't jump to line 167, because the condition on line 161 was never false

162 parsed = copy.copy(uri._uri) 

163 dirLike = uri.dirLike 

164 # No further parsing required and we know the subclass 

165 subclass = type(uri) 

166 else: 

167 raise ValueError("Supplied URI must be string, Path, " 

168 f"ButlerURI, or ParseResult but got '{uri!r}'") 

169 

170 if subclass is None: 

171 # Work out the subclass from the URI scheme 

172 if not parsed.scheme: 

173 from .schemeless import ButlerSchemelessURI 

174 subclass = ButlerSchemelessURI 

175 elif parsed.scheme == "file": 175 ↛ 176line 175 didn't jump to line 176, because the condition on line 175 was never true

176 from .file import ButlerFileURI 

177 subclass = ButlerFileURI 

178 elif parsed.scheme == "s3": 178 ↛ 179line 178 didn't jump to line 179, because the condition on line 178 was never true

179 from .s3 import ButlerS3URI 

180 subclass = ButlerS3URI 

181 elif parsed.scheme.startswith("http"): 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true

182 from .http import ButlerHttpURI 

183 subclass = ButlerHttpURI 

184 elif parsed.scheme == "resource": 184 ↛ 188line 184 didn't jump to line 188, because the condition on line 184 was never false

185 # Rules for scheme names disallow pkg_resource 

186 from .packageresource import ButlerPackageResourceURI 

187 subclass = ButlerPackageResourceURI 

188 elif parsed.scheme == "mem": 

189 # in-memory datastore object 

190 from .mem import ButlerInMemoryURI 

191 subclass = ButlerInMemoryURI 

192 else: 

193 raise NotImplementedError(f"No URI support for scheme: '{parsed.scheme}'" 

194 " in {parsed.geturl()}") 

195 

196 parsed, dirLike = subclass._fixupPathUri(parsed, root=root, 

197 forceAbsolute=forceAbsolute, 

198 forceDirectory=forceDirectory) 

199 

200 # It is possible for the class to change from schemeless 

201 # to file so handle that 

202 if parsed.scheme == "file": 202 ↛ 203line 202 didn't jump to line 203, because the condition on line 202 was never true

203 from .file import ButlerFileURI 

204 subclass = ButlerFileURI 

205 

206 # Now create an instance of the correct subclass and set the 

207 # attributes directly 

208 self = object.__new__(subclass) 

209 self._uri = parsed 

210 self.dirLike = dirLike 

211 self.isTemporary = isTemporary 

212 return self 

213 

214 @property 

215 def scheme(self) -> str: 

216 """The URI scheme (``://`` is not part of the scheme).""" 

217 return self._uri.scheme 

218 

219 @property 

220 def netloc(self) -> str: 

221 """The URI network location.""" 

222 return self._uri.netloc 

223 

224 @property 

225 def path(self) -> str: 

226 """The path component of the URI.""" 

227 return self._uri.path 

228 

229 @property 

230 def unquoted_path(self) -> str: 

231 """The path component of the URI with any URI quoting reversed.""" 

232 return urllib.parse.unquote(self._uri.path) 

233 

234 @property 

235 def ospath(self) -> str: 

236 """Path component of the URI localized to current OS.""" 

237 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

238 

239 @property 

240 def relativeToPathRoot(self) -> str: 

241 """Returns path relative to network location. 

242 

243 Effectively, this is the path property with posix separator stripped 

244 from the left hand side of the path. 

245 

246 Always unquotes. 

247 """ 

248 p = self._pathLib(self.path) 

249 relToRoot = str(p.relative_to(p.root)) 

250 if self.dirLike and not relToRoot.endswith("/"): 250 ↛ 251line 250 didn't jump to line 251, because the condition on line 250 was never true

251 relToRoot += "/" 

252 return urllib.parse.unquote(relToRoot) 

253 

254 @property 

255 def is_root(self) -> bool: 

256 """`True` if this URI points to the root of the network location. 

257 

258 This means that the path components refers to the top level. 

259 """ 

260 relpath = self.relativeToPathRoot 

261 if relpath == "./": 

262 return True 

263 return False 

264 

265 @property 

266 def fragment(self) -> str: 

267 """The fragment component of the URI.""" 

268 return self._uri.fragment 

269 

270 @property 

271 def params(self) -> str: 

272 """Any parameters included in the URI.""" 

273 return self._uri.params 

274 

275 @property 

276 def query(self) -> str: 

277 """Any query strings included in the URI.""" 

278 return self._uri.query 

279 

280 def geturl(self) -> str: 

281 """Return the URI in string form. 

282 

283 Returns 

284 ------- 

285 url : `str` 

286 String form of URI. 

287 """ 

288 return self._uri.geturl() 

289 

290 def split(self) -> Tuple[ButlerURI, str]: 

291 """Splits URI into head and tail. Equivalent to os.path.split where 

292 head preserves the URI components. 

293 

294 Returns 

295 ------- 

296 head: `ButlerURI` 

297 Everything leading up to tail, expanded and normalized as per 

298 ButlerURI rules. 

299 tail : `str` 

300 Last `self.path` component. Tail will be empty if path ends on a 

301 separator. Tail will never contain separators. It will be 

302 unquoted. 

303 """ 

304 head, tail = self._pathModule.split(self.path) 

305 headuri = self._uri._replace(path=head) 

306 

307 # The file part should never include quoted metacharacters 

308 tail = urllib.parse.unquote(tail) 

309 

310 # Schemeless is special in that it can be a relative path 

311 # We need to ensure that it stays that way. All other URIs will 

312 # be absolute already. 

313 forceAbsolute = self._pathModule.isabs(self.path) 

314 return ButlerURI(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

315 

316 def basename(self) -> str: 

317 """Returns the base name, last element of path, of the URI. If URI ends 

318 on a slash returns an empty string. This is the second element returned 

319 by split(). 

320 

321 Equivalent of os.path.basename(). 

322 

323 Returns 

324 ------- 

325 tail : `str` 

326 Last part of the path attribute. Trail will be empty if path ends 

327 on a separator. 

328 """ 

329 return self.split()[1] 

330 

331 def dirname(self) -> ButlerURI: 

332 """Returns a ButlerURI containing all the directories of the path 

333 attribute. 

334 

335 Equivalent of os.path.dirname() 

336 

337 Returns 

338 ------- 

339 head : `ButlerURI` 

340 Everything except the tail of path attribute, expanded and 

341 normalized as per ButlerURI rules. 

342 """ 

343 return self.split()[0] 

344 

345 def parent(self) -> ButlerURI: 

346 """Returns a ButlerURI containing all the directories of the path 

347 attribute, minus the last one. 

348 

349 Returns 

350 ------- 

351 head : `ButlerURI` 

352 Everything except the tail of path attribute, expanded and 

353 normalized as per ButlerURI rules. 

354 """ 

355 # When self is file-like, return self.dirname() 

356 if not self.dirLike: 

357 return self.dirname() 

358 # When self is dir-like, return its parent directory, 

359 # regardless of the presence of a trailing separator 

360 originalPath = self._pathLib(self.path) 

361 parentPath = originalPath.parent 

362 parentURI = self._uri._replace(path=str(parentPath)) 

363 

364 return ButlerURI(parentURI, forceDirectory=True) 

365 

366 def replace(self, **kwargs: Any) -> ButlerURI: 

367 """Replace components in a URI with new values and return a new 

368 instance. 

369 

370 Returns 

371 ------- 

372 new : `ButlerURI` 

373 New `ButlerURI` object with updated values. 

374 """ 

375 return self.__class__(self._uri._replace(**kwargs)) 

376 

377 def updateFile(self, newfile: str) -> None: 

378 """Update in place the final component of the path with the supplied 

379 file name. 

380 

381 Parameters 

382 ---------- 

383 newfile : `str` 

384 File name with no path component. 

385 

386 Notes 

387 ----- 

388 Updates the URI in place. 

389 Updates the ButlerURI.dirLike attribute. The new file path will 

390 be quoted if necessary. 

391 """ 

392 if self.quotePaths: 

393 newfile = urllib.parse.quote(newfile) 

394 dir, _ = self._pathModule.split(self.path) 

395 newpath = self._pathModule.join(dir, newfile) 

396 

397 self.dirLike = False 

398 self._uri = self._uri._replace(path=newpath) 

399 

400 def updateExtension(self, ext: Optional[str]) -> None: 

401 """Update the file extension associated with this `ButlerURI` in place. 

402 

403 All file extensions are replaced. 

404 

405 Parameters 

406 ---------- 

407 ext : `str` or `None` 

408 New extension. If an empty string is given any extension will 

409 be removed. If `None` is given there will be no change. 

410 """ 

411 if ext is None: 

412 return 

413 

414 # Get the extension and remove it from the path if one is found 

415 # .fits.gz counts as one extension do not use os.path.splitext 

416 current = self.getExtension() 

417 path = self.path 

418 if current: 

419 path = path[:-len(current)] 

420 

421 # Ensure that we have a leading "." on file extension (and we do not 

422 # try to modify the empty string) 

423 if ext and not ext.startswith("."): 

424 ext = "." + ext 

425 

426 self._uri = self._uri._replace(path=path + ext) 

427 

428 def getExtension(self) -> str: 

429 """Return the file extension(s) associated with this URI path. 

430 

431 Returns 

432 ------- 

433 ext : `str` 

434 The file extension (including the ``.``). Can be empty string 

435 if there is no file extension. Usually returns only the last 

436 file extension unless there is a special extension modifier 

437 indicating file compression, in which case the combined 

438 extension (e.g. ``.fits.gz``) will be returned. 

439 """ 

440 special = {".gz", ".bz2", ".xz", ".fz"} 

441 

442 extensions = self._pathLib(self.path).suffixes 

443 

444 if not extensions: 444 ↛ 445line 444 didn't jump to line 445, because the condition on line 444 was never true

445 return "" 

446 

447 ext = extensions.pop() 

448 

449 # Multiple extensions, decide whether to include the final two 

450 if extensions and ext in special: 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true

451 ext = f"{extensions[-1]}{ext}" 

452 

453 return ext 

454 

455 def join(self, path: Union[str, ButlerURI]) -> ButlerURI: 

456 """Create a new `ButlerURI` with additional path components including 

457 a file. 

458 

459 Parameters 

460 ---------- 

461 path : `str`, `ButlerURI` 

462 Additional file components to append to the current URI. Assumed 

463 to include a file at the end. Will be quoted depending on the 

464 associated URI scheme. If the path looks like a URI with a scheme 

465 referring to an absolute location, it will be returned 

466 directly (matching the behavior of `os.path.join()`). It can 

467 also be a `ButlerURI`. 

468 

469 Returns 

470 ------- 

471 new : `ButlerURI` 

472 New URI with any file at the end replaced with the new path 

473 components. 

474 

475 Notes 

476 ----- 

477 Schemeless URIs assume local path separator but all other URIs assume 

478 POSIX separator if the supplied path has directory structure. It 

479 may be this never becomes a problem but datastore templates assume 

480 POSIX separator is being used. 

481 """ 

482 # If we have a full URI in path we will use it directly 

483 # but without forcing to absolute so that we can trap the 

484 # expected option of relative path. 

485 path_uri = ButlerURI(path, forceAbsolute=False) 

486 if path_uri.scheme: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true

487 return path_uri 

488 

489 # Force back to string 

490 path = path_uri.path 

491 

492 new = self.dirname() # By definition a directory URI 

493 

494 # new should be asked about quoting, not self, since dirname can 

495 # change the URI scheme for schemeless -> file 

496 if new.quotePaths: 496 ↛ 499line 496 didn't jump to line 499, because the condition on line 496 was never false

497 path = urllib.parse.quote(path) 

498 

499 newpath = self._pathModule.normpath(self._pathModule.join(new.path, path)) 

500 new._uri = new._uri._replace(path=newpath) 

501 # Declare the new URI not be dirLike unless path ended in / 

502 if not path.endswith(self._pathModule.sep): 502 ↛ 504line 502 didn't jump to line 504, because the condition on line 502 was never false

503 new.dirLike = False 

504 return new 

505 

506 def relative_to(self, other: ButlerURI) -> Optional[str]: 

507 """Return the relative path from this URI to the other URI. 

508 

509 Parameters 

510 ---------- 

511 other : `ButlerURI` 

512 URI to use to calculate the relative path. Must be a parent 

513 of this URI. 

514 

515 Returns 

516 ------- 

517 subpath : `str` 

518 The sub path of this URI relative to the supplied other URI. 

519 Returns `None` if there is no parent child relationship. 

520 Scheme and netloc must match. 

521 """ 

522 if self.scheme != other.scheme or self.netloc != other.netloc: 

523 return None 

524 

525 enclosed_path = self._pathLib(self.relativeToPathRoot) 

526 parent_path = other.relativeToPathRoot 

527 subpath: Optional[str] 

528 try: 

529 subpath = str(enclosed_path.relative_to(parent_path)) 

530 except ValueError: 

531 subpath = None 

532 else: 

533 subpath = urllib.parse.unquote(subpath) 

534 return subpath 

535 

536 def exists(self) -> bool: 

537 """Indicate that the resource is available. 

538 

539 Returns 

540 ------- 

541 exists : `bool` 

542 `True` if the resource exists. 

543 """ 

544 raise NotImplementedError() 

545 

546 def remove(self) -> None: 

547 """Remove the resource.""" 

548 raise NotImplementedError() 

549 

550 def isabs(self) -> bool: 

551 """Indicate that the resource is fully specified. 

552 

553 For non-schemeless URIs this is always true. 

554 

555 Returns 

556 ------- 

557 isabs : `bool` 

558 `True` in all cases except schemeless URI. 

559 """ 

560 return True 

561 

562 def _as_local(self) -> Tuple[str, bool]: 

563 """Return the location of the (possibly remote) resource in the 

564 local file system. 

565 

566 This is a helper function for ``as_local`` context manager. 

567 

568 Returns 

569 ------- 

570 path : `str` 

571 If this is a remote resource, it will be a copy of the resource 

572 on the local file system, probably in a temporary directory. 

573 For a local resource this should be the actual path to the 

574 resource. 

575 is_temporary : `bool` 

576 Indicates if the local path is a temporary file or not. 

577 """ 

578 raise NotImplementedError() 

579 

580 @contextlib.contextmanager 

581 def as_local(self) -> Iterator[ButlerURI]: 

582 """Return the location of the (possibly remote) resource in the 

583 local file system. 

584 

585 Yields 

586 ------ 

587 local : `ButlerURI` 

588 If this is a remote resource, it will be a copy of the resource 

589 on the local file system, probably in a temporary directory. 

590 For a local resource this should be the actual path to the 

591 resource. 

592 

593 Notes 

594 ----- 

595 The context manager will automatically delete any local temporary 

596 file. 

597 

598 Examples 

599 -------- 

600 Should be used as a context manager: 

601 

602 .. code-block:: py 

603 

604 with uri.as_local() as local: 

605 ospath = local.ospath 

606 """ 

607 local_src, is_temporary = self._as_local() 

608 local_uri = ButlerURI(local_src, isTemporary=is_temporary) 

609 

610 try: 

611 yield local_uri 

612 finally: 

613 # The caller might have relocated the temporary file 

614 if is_temporary and local_uri.exists(): 

615 local_uri.remove() 

616 

617 def read(self, size: int = -1) -> bytes: 

618 """Open the resource and return the contents in bytes. 

619 

620 Parameters 

621 ---------- 

622 size : `int`, optional 

623 The number of bytes to read. Negative or omitted indicates 

624 that all data should be read. 

625 """ 

626 raise NotImplementedError() 

627 

628 def write(self, data: bytes, overwrite: bool = True) -> None: 

629 """Write the supplied bytes to the new resource. 

630 

631 Parameters 

632 ---------- 

633 data : `bytes` 

634 The bytes to write to the resource. The entire contents of the 

635 resource will be replaced. 

636 overwrite : `bool`, optional 

637 If `True` the resource will be overwritten if it exists. Otherwise 

638 the write will fail. 

639 """ 

640 raise NotImplementedError() 

641 

642 def mkdir(self) -> None: 

643 """For a dir-like URI, create the directory resource if it does not 

644 already exist. 

645 """ 

646 raise NotImplementedError() 

647 

648 def size(self) -> int: 

649 """For non-dir-like URI, return the size of the resource. 

650 

651 Returns 

652 ------- 

653 sz : `int` 

654 The size in bytes of the resource associated with this URI. 

655 Returns 0 if dir-like. 

656 """ 

657 raise NotImplementedError() 

658 

659 def __str__(self) -> str: 

660 return self.geturl() 

661 

662 def __repr__(self) -> str: 

663 return f'ButlerURI("{self.geturl()}")' 

664 

665 def __eq__(self, other: Any) -> bool: 

666 if not isinstance(other, ButlerURI): 

667 return False 

668 return self.geturl() == other.geturl() 

669 

670 def __copy__(self) -> ButlerURI: 

671 # Implement here because the __new__ method confuses things 

672 # Be careful not to convert a relative schemeless URI to absolute 

673 return type(self)(str(self), forceAbsolute=self.isabs()) 

674 

675 def __deepcopy__(self, memo: Any) -> ButlerURI: 

676 # Implement here because the __new__ method confuses things 

677 return self.__copy__() 

678 

679 def __getnewargs__(self) -> Tuple: 

680 return (str(self),) 

681 

682 @staticmethod 

683 def _fixupPathUri(parsed: urllib.parse.ParseResult, root: Optional[Union[str, ButlerURI]] = None, 

684 forceAbsolute: bool = False, 

685 forceDirectory: bool = False) -> Tuple[urllib.parse.ParseResult, bool]: 

686 """Correct any issues with the supplied URI. 

687 

688 Parameters 

689 ---------- 

690 parsed : `~urllib.parse.ParseResult` 

691 The result from parsing a URI using `urllib.parse`. 

692 root : `str` or `ButlerURI`, ignored 

693 Not used by the this implementation since all URIs are 

694 absolute except for those representing the local file system. 

695 forceAbsolute : `bool`, ignored. 

696 Not used by this implementation. URIs are generally always 

697 absolute. 

698 forceDirectory : `bool`, optional 

699 If `True` forces the URI to end with a separator, otherwise given 

700 URI is interpreted as is. Specifying that the URI is conceptually 

701 equivalent to a directory can break some ambiguities when 

702 interpreting the last element of a path. 

703 

704 Returns 

705 ------- 

706 modified : `~urllib.parse.ParseResult` 

707 Update result if a URI is being handled. 

708 dirLike : `bool` 

709 `True` if given parsed URI has a trailing separator or 

710 forceDirectory is True. Otherwise `False`. 

711 

712 Notes 

713 ----- 

714 Relative paths are explicitly not supported by RFC8089 but `urllib` 

715 does accept URIs of the form ``file:relative/path.ext``. They need 

716 to be turned into absolute paths before they can be used. This is 

717 always done regardless of the ``forceAbsolute`` parameter. 

718 

719 AWS S3 differentiates between keys with trailing POSIX separators (i.e 

720 `/dir` and `/dir/`) whereas POSIX does not neccessarily. 

721 

722 Scheme-less paths are normalized. 

723 """ 

724 # assume we are not dealing with a directory like URI 

725 dirLike = False 

726 

727 # URI is dir-like if explicitly stated or if it ends on a separator 

728 endsOnSep = parsed.path.endswith(posixpath.sep) 

729 if forceDirectory or endsOnSep: 

730 dirLike = True 

731 # only add the separator if it's not already there 

732 if not endsOnSep: 732 ↛ 735line 732 didn't jump to line 735, because the condition on line 732 was never false

733 parsed = parsed._replace(path=parsed.path+posixpath.sep) 

734 

735 return parsed, dirLike 

736 

737 def transfer_from(self, src: ButlerURI, transfer: str, 

738 overwrite: bool = False, 

739 transaction: Optional[Union[DatastoreTransaction, NoTransaction]] = None) -> None: 

740 """Transfer the current resource to a new location. 

741 

742 Parameters 

743 ---------- 

744 src : `ButlerURI` 

745 Source URI. 

746 transfer : `str` 

747 Mode to use for transferring the resource. Generically there are 

748 many standard options: copy, link, symlink, hardlink, relsymlink. 

749 Not all URIs support all modes. 

750 overwrite : `bool`, optional 

751 Allow an existing file to be overwritten. Defaults to `False`. 

752 transaction : `DatastoreTransaction`, optional 

753 A transaction object that can (depending on implementation) 

754 rollback transfers on error. Not guaranteed to be implemented. 

755 

756 Notes 

757 ----- 

758 Conceptually this is hard to scale as the number of URI schemes 

759 grow. The destination URI is more important than the source URI 

760 since that is where all the transfer modes are relevant (with the 

761 complication that "move" deletes the source). 

762 

763 Local file to local file is the fundamental use case but every 

764 other scheme has to support "copy" to local file (with implicit 

765 support for "move") and copy from local file. 

766 All the "link" options tend to be specific to local file systems. 

767 

768 "move" is a "copy" where the remote resource is deleted at the end. 

769 Whether this works depends on the source URI rather than the 

770 destination URI. Reverting a move on transaction rollback is 

771 expected to be problematic if a remote resource was involved. 

772 """ 

773 raise NotImplementedError(f"No transfer modes supported by URI scheme {self.scheme}")