Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import contextlib 

25import urllib.parse 

26import posixpath 

27import copy 

28import logging 

29import re 

30 

31from pathlib import Path, PurePath, PurePosixPath 

32 

33__all__ = ('ButlerURI',) 

34 

35from typing import ( 

36 TYPE_CHECKING, 

37 Any, 

38 Iterator, 

39 Optional, 

40 Tuple, 

41 Type, 

42 Union, 

43) 

44 

45from .utils import NoTransaction 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from ..datastore import DatastoreTransaction 

49 

50 

51log = logging.getLogger(__name__) 

52 

53# Regex for looking for URI escapes 

54ESCAPES_RE = re.compile(r"%[A-F0-9]{2}") 

55 

56 

57class ButlerURI: 

58 """Convenience wrapper around URI parsers. 

59 

60 Provides access to URI components and can convert file 

61 paths into absolute path URIs. Scheme-less URIs are treated as if 

62 they are local file system paths and are converted to absolute URIs. 

63 

64 A specialist subclass is created for each supported URI scheme. 

65 

66 Parameters 

67 ---------- 

68 uri : `str` or `urllib.parse.ParseResult` 

69 URI in string form. Can be scheme-less if referring to a local 

70 filesystem path. 

71 root : `str` or `ButlerURI`, optional 

72 When fixing up a relative path in a ``file`` scheme or if scheme-less, 

73 use this as the root. Must be absolute. If `None` the current 

74 working directory will be used. Can be a file URI. 

75 forceAbsolute : `bool`, optional 

76 If `True`, scheme-less relative URI will be converted to an absolute 

77 path using a ``file`` scheme. If `False` scheme-less URI will remain 

78 scheme-less and will not be updated to ``file`` or absolute path. 

79 forceDirectory: `bool`, optional 

80 If `True` forces the URI to end with a separator, otherwise given URI 

81 is interpreted as is. 

82 isTemporary : `bool`, optional 

83 If `True` indicates that this URI points to a temporary resource. 

84 """ 

85 

86 _pathLib: Type[PurePath] = PurePosixPath 

87 """Path library to use for this scheme.""" 

88 

89 _pathModule = posixpath 

90 """Path module to use for this scheme.""" 

91 

92 transferModes: Tuple[str, ...] = ("copy", "auto", "move") 

93 """Transfer modes supported by this implementation. 

94 

95 Move is special in that it is generally a copy followed by an unlink. 

96 Whether that unlink works depends critically on whether the source URI 

97 implements unlink. If it does not the move will be reported as a failure. 

98 """ 

99 

100 transferDefault: str = "copy" 

101 """Default mode to use for transferring if ``auto`` is specified.""" 

102 

103 quotePaths = True 

104 """True if path-like elements modifying a URI should be quoted. 

105 

106 All non-schemeless URIs have to internally use quoted paths. Therefore 

107 if a new file name is given (e.g. to updateFile or join) a decision must 

108 be made whether to quote it to be consistent. 

109 """ 

110 

111 isLocal = False 

112 """If `True` this URI refers to a local file.""" 

113 

114 # This is not an ABC with abstract methods because the __new__ being 

115 # a factory confuses mypy such that it assumes that every constructor 

116 # returns a ButlerURI and then determines that all the abstract methods 

117 # are still abstract. If they are not marked abstract but just raise 

118 # mypy is fine with it. 

119 

120 # mypy is confused without these 

121 _uri: urllib.parse.ParseResult 

122 isTemporary: bool 

123 

124 def __new__(cls, uri: Union[str, urllib.parse.ParseResult, ButlerURI, Path], 

125 root: Optional[Union[str, ButlerURI]] = None, forceAbsolute: bool = True, 

126 forceDirectory: bool = False, isTemporary: bool = False) -> ButlerURI: 

127 parsed: urllib.parse.ParseResult 

128 dirLike: bool = False 

129 subclass: Optional[Type] = None 

130 

131 if isinstance(uri, Path): 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true

132 uri = str(uri) 

133 

134 # Record if we need to post process the URI components 

135 # or if the instance is already fully configured 

136 if isinstance(uri, str): 

137 # Since local file names can have special characters in them 

138 # we need to quote them for the parser but we can unquote 

139 # later. Assume that all other URI schemes are quoted. 

140 # Since sometimes people write file:/a/b and not file:///a/b 

141 # we should not quote in the explicit case of file: 

142 if "://" not in uri and not uri.startswith("file:"): 

143 if ESCAPES_RE.search(uri): 143 ↛ 144line 143 didn't jump to line 144, because the condition on line 143 was never true

144 log.warning("Possible double encoding of %s", uri) 

145 else: 

146 uri = urllib.parse.quote(uri) 

147 parsed = urllib.parse.urlparse(uri) 

148 elif isinstance(uri, urllib.parse.ParseResult): 

149 parsed = copy.copy(uri) 

150 elif isinstance(uri, ButlerURI): 150 ↛ 156line 150 didn't jump to line 156, because the condition on line 150 was never false

151 parsed = copy.copy(uri._uri) 

152 dirLike = uri.dirLike 

153 # No further parsing required and we know the subclass 

154 subclass = type(uri) 

155 else: 

156 raise ValueError("Supplied URI must be string, Path, " 

157 f"ButlerURI, or ParseResult but got '{uri!r}'") 

158 

159 if subclass is None: 

160 # Work out the subclass from the URI scheme 

161 if not parsed.scheme: 

162 from .schemeless import ButlerSchemelessURI 

163 subclass = ButlerSchemelessURI 

164 elif parsed.scheme == "file": 164 ↛ 165line 164 didn't jump to line 165, because the condition on line 164 was never true

165 from .file import ButlerFileURI 

166 subclass = ButlerFileURI 

167 elif parsed.scheme == "s3": 167 ↛ 168line 167 didn't jump to line 168, because the condition on line 167 was never true

168 from .s3 import ButlerS3URI 

169 subclass = ButlerS3URI 

170 elif parsed.scheme.startswith("http"): 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 from .http import ButlerHttpURI 

172 subclass = ButlerHttpURI 

173 elif parsed.scheme == "resource": 173 ↛ 177line 173 didn't jump to line 177, because the condition on line 173 was never false

174 # Rules for scheme names disallow pkg_resource 

175 from .packageresource import ButlerPackageResourceURI 

176 subclass = ButlerPackageResourceURI 

177 elif parsed.scheme == "mem": 

178 # in-memory datastore object 

179 from .mem import ButlerInMemoryURI 

180 subclass = ButlerInMemoryURI 

181 else: 

182 raise NotImplementedError(f"No URI support for scheme: '{parsed.scheme}'" 

183 " in {parsed.geturl()}") 

184 

185 parsed, dirLike = subclass._fixupPathUri(parsed, root=root, 

186 forceAbsolute=forceAbsolute, 

187 forceDirectory=forceDirectory) 

188 

189 # It is possible for the class to change from schemeless 

190 # to file so handle that 

191 if parsed.scheme == "file": 191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true

192 from .file import ButlerFileURI 

193 subclass = ButlerFileURI 

194 

195 # Now create an instance of the correct subclass and set the 

196 # attributes directly 

197 self = object.__new__(subclass) 

198 self._uri = parsed 

199 self.dirLike = dirLike 

200 self.isTemporary = isTemporary 

201 return self 

202 

203 @property 

204 def scheme(self) -> str: 

205 """The URI scheme (``://`` is not part of the scheme).""" 

206 return self._uri.scheme 

207 

208 @property 

209 def netloc(self) -> str: 

210 """The URI network location.""" 

211 return self._uri.netloc 

212 

213 @property 

214 def path(self) -> str: 

215 """The path component of the URI.""" 

216 return self._uri.path 

217 

218 @property 

219 def unquoted_path(self) -> str: 

220 """The path component of the URI with any URI quoting reversed.""" 

221 return urllib.parse.unquote(self._uri.path) 

222 

223 @property 

224 def ospath(self) -> str: 

225 """Path component of the URI localized to current OS.""" 

226 raise AttributeError(f"Non-file URI ({self}) has no local OS path.") 

227 

228 @property 

229 def relativeToPathRoot(self) -> str: 

230 """Returns path relative to network location. 

231 

232 Effectively, this is the path property with posix separator stripped 

233 from the left hand side of the path. 

234 

235 Always unquotes. 

236 """ 

237 p = self._pathLib(self.path) 

238 relToRoot = str(p.relative_to(p.root)) 

239 if self.dirLike and not relToRoot.endswith("/"): 239 ↛ 240line 239 didn't jump to line 240, because the condition on line 239 was never true

240 relToRoot += "/" 

241 return urllib.parse.unquote(relToRoot) 

242 

243 @property 

244 def is_root(self) -> bool: 

245 """`True` if this URI points to the root of the network location. 

246 

247 This means that the path components refers to the top level. 

248 """ 

249 relpath = self.relativeToPathRoot 

250 if relpath == "./": 

251 return True 

252 return False 

253 

254 @property 

255 def fragment(self) -> str: 

256 """The fragment component of the URI.""" 

257 return self._uri.fragment 

258 

259 @property 

260 def params(self) -> str: 

261 """Any parameters included in the URI.""" 

262 return self._uri.params 

263 

264 @property 

265 def query(self) -> str: 

266 """Any query strings included in the URI.""" 

267 return self._uri.query 

268 

269 def geturl(self) -> str: 

270 """Return the URI in string form. 

271 

272 Returns 

273 ------- 

274 url : `str` 

275 String form of URI. 

276 """ 

277 return self._uri.geturl() 

278 

279 def split(self) -> Tuple[ButlerURI, str]: 

280 """Splits URI into head and tail. Equivalent to os.path.split where 

281 head preserves the URI components. 

282 

283 Returns 

284 ------- 

285 head: `ButlerURI` 

286 Everything leading up to tail, expanded and normalized as per 

287 ButlerURI rules. 

288 tail : `str` 

289 Last `self.path` component. Tail will be empty if path ends on a 

290 separator. Tail will never contain separators. It will be 

291 unquoted. 

292 """ 

293 head, tail = self._pathModule.split(self.path) 

294 headuri = self._uri._replace(path=head) 

295 

296 # The file part should never include quoted metacharacters 

297 tail = urllib.parse.unquote(tail) 

298 

299 # Schemeless is special in that it can be a relative path 

300 # We need to ensure that it stays that way. All other URIs will 

301 # be absolute already. 

302 forceAbsolute = self._pathModule.isabs(self.path) 

303 return ButlerURI(headuri, forceDirectory=True, forceAbsolute=forceAbsolute), tail 

304 

305 def basename(self) -> str: 

306 """Returns the base name, last element of path, of the URI. If URI ends 

307 on a slash returns an empty string. This is the second element returned 

308 by split(). 

309 

310 Equivalent of os.path.basename(). 

311 

312 Returns 

313 ------- 

314 tail : `str` 

315 Last part of the path attribute. Trail will be empty if path ends 

316 on a separator. 

317 """ 

318 return self.split()[1] 

319 

320 def dirname(self) -> ButlerURI: 

321 """Returns a ButlerURI containing all the directories of the path 

322 attribute. 

323 

324 Equivalent of os.path.dirname() 

325 

326 Returns 

327 ------- 

328 head : `ButlerURI` 

329 Everything except the tail of path attribute, expanded and 

330 normalized as per ButlerURI rules. 

331 """ 

332 return self.split()[0] 

333 

334 def parent(self) -> ButlerURI: 

335 """Returns a ButlerURI containing all the directories of the path 

336 attribute, minus the last one. 

337 

338 Returns 

339 ------- 

340 head : `ButlerURI` 

341 Everything except the tail of path attribute, expanded and 

342 normalized as per ButlerURI rules. 

343 """ 

344 # When self is file-like, return self.dirname() 

345 if not self.dirLike: 

346 return self.dirname() 

347 # When self is dir-like, return its parent directory, 

348 # regardless of the presence of a trailing separator 

349 originalPath = self._pathLib(self.path) 

350 parentPath = originalPath.parent 

351 parentURI = self._uri._replace(path=str(parentPath)) 

352 

353 return ButlerURI(parentURI, forceDirectory=True) 

354 

355 def replace(self, **kwargs: Any) -> ButlerURI: 

356 """Replace components in a URI with new values and return a new 

357 instance. 

358 

359 Returns 

360 ------- 

361 new : `ButlerURI` 

362 New `ButlerURI` object with updated values. 

363 """ 

364 return self.__class__(self._uri._replace(**kwargs)) 

365 

366 def updateFile(self, newfile: str) -> None: 

367 """Update in place the final component of the path with the supplied 

368 file name. 

369 

370 Parameters 

371 ---------- 

372 newfile : `str` 

373 File name with no path component. 

374 

375 Notes 

376 ----- 

377 Updates the URI in place. 

378 Updates the ButlerURI.dirLike attribute. The new file path will 

379 be quoted if necessary. 

380 """ 

381 if self.quotePaths: 

382 newfile = urllib.parse.quote(newfile) 

383 dir, _ = self._pathModule.split(self.path) 

384 newpath = self._pathModule.join(dir, newfile) 

385 

386 self.dirLike = False 

387 self._uri = self._uri._replace(path=newpath) 

388 

389 def updateExtension(self, ext: Optional[str]) -> None: 

390 """Update the file extension associated with this `ButlerURI` in place. 

391 

392 All file extensions are replaced. 

393 

394 Parameters 

395 ---------- 

396 ext : `str` or `None` 

397 New extension. If an empty string is given any extension will 

398 be removed. If `None` is given there will be no change. 

399 """ 

400 if ext is None: 

401 return 

402 

403 # Get the extension and remove it from the path if one is found 

404 # .fits.gz counts as one extension do not use os.path.splitext 

405 current = self.getExtension() 

406 path = self.path 

407 if current: 

408 path = path[:-len(current)] 

409 

410 # Ensure that we have a leading "." on file extension (and we do not 

411 # try to modify the empty string) 

412 if ext and not ext.startswith("."): 

413 ext = "." + ext 

414 

415 self._uri = self._uri._replace(path=path + ext) 

416 

417 def getExtension(self) -> str: 

418 """Return the file extension(s) associated with this URI path. 

419 

420 Returns 

421 ------- 

422 ext : `str` 

423 The file extension (including the ``.``). Can be empty string 

424 if there is no file extension. Usually returns only the last 

425 file extension unless there is a special extension modifier 

426 indicating file compression, in which case the combined 

427 extension (e.g. ``.fits.gz``) will be returned. 

428 """ 

429 special = {".gz", ".bz2", ".xz", ".fz"} 

430 

431 extensions = self._pathLib(self.path).suffixes 

432 

433 if not extensions: 433 ↛ 434line 433 didn't jump to line 434, because the condition on line 433 was never true

434 return "" 

435 

436 ext = extensions.pop() 

437 

438 # Multiple extensions, decide whether to include the final two 

439 if extensions and ext in special: 439 ↛ 440line 439 didn't jump to line 440, because the condition on line 439 was never true

440 ext = f"{extensions[-1]}{ext}" 

441 

442 return ext 

443 

444 def join(self, path: Union[str, ButlerURI]) -> ButlerURI: 

445 """Create a new `ButlerURI` with additional path components including 

446 a file. 

447 

448 Parameters 

449 ---------- 

450 path : `str`, `ButlerURI` 

451 Additional file components to append to the current URI. Assumed 

452 to include a file at the end. Will be quoted depending on the 

453 associated URI scheme. If the path looks like a URI with a scheme 

454 referring to an absolute location, it will be returned 

455 directly (matching the behavior of `os.path.join()`). It can 

456 also be a `ButlerURI`. 

457 

458 Returns 

459 ------- 

460 new : `ButlerURI` 

461 New URI with any file at the end replaced with the new path 

462 components. 

463 

464 Notes 

465 ----- 

466 Schemeless URIs assume local path separator but all other URIs assume 

467 POSIX separator if the supplied path has directory structure. It 

468 may be this never becomes a problem but datastore templates assume 

469 POSIX separator is being used. 

470 """ 

471 # If we have a full URI in path we will use it directly 

472 # but without forcing to absolute so that we can trap the 

473 # expected option of relative path. 

474 path_uri = ButlerURI(path, forceAbsolute=False) 

475 if path_uri.scheme: 475 ↛ 476line 475 didn't jump to line 476, because the condition on line 475 was never true

476 return path_uri 

477 

478 # Force back to string 

479 path = path_uri.path 

480 

481 new = self.dirname() # By definition a directory URI 

482 

483 # new should be asked about quoting, not self, since dirname can 

484 # change the URI scheme for schemeless -> file 

485 if new.quotePaths: 485 ↛ 488line 485 didn't jump to line 488, because the condition on line 485 was never false

486 path = urllib.parse.quote(path) 

487 

488 newpath = self._pathModule.normpath(self._pathModule.join(new.path, path)) 

489 new._uri = new._uri._replace(path=newpath) 

490 # Declare the new URI not be dirLike unless path ended in / 

491 if not path.endswith(self._pathModule.sep): 491 ↛ 493line 491 didn't jump to line 493, because the condition on line 491 was never false

492 new.dirLike = False 

493 return new 

494 

495 def relative_to(self, other: ButlerURI) -> Optional[str]: 

496 """Return the relative path from this URI to the other URI. 

497 

498 Parameters 

499 ---------- 

500 other : `ButlerURI` 

501 URI to use to calculate the relative path. Must be a parent 

502 of this URI. 

503 

504 Returns 

505 ------- 

506 subpath : `str` 

507 The sub path of this URI relative to the supplied other URI. 

508 Returns `None` if there is no parent child relationship. 

509 Scheme and netloc must match. 

510 """ 

511 if self.scheme != other.scheme or self.netloc != other.netloc: 

512 return None 

513 

514 enclosed_path = self._pathLib(self.relativeToPathRoot) 

515 parent_path = other.relativeToPathRoot 

516 subpath: Optional[str] 

517 try: 

518 subpath = str(enclosed_path.relative_to(parent_path)) 

519 except ValueError: 

520 subpath = None 

521 else: 

522 subpath = urllib.parse.unquote(subpath) 

523 return subpath 

524 

525 def exists(self) -> bool: 

526 """Indicate that the resource is available. 

527 

528 Returns 

529 ------- 

530 exists : `bool` 

531 `True` if the resource exists. 

532 """ 

533 raise NotImplementedError() 

534 

535 def remove(self) -> None: 

536 """Remove the resource.""" 

537 raise NotImplementedError() 

538 

539 def isabs(self) -> bool: 

540 """Indicate that the resource is fully specified. 

541 

542 For non-schemeless URIs this is always true. 

543 

544 Returns 

545 ------- 

546 isabs : `bool` 

547 `True` in all cases except schemeless URI. 

548 """ 

549 return True 

550 

551 def _as_local(self) -> Tuple[str, bool]: 

552 """Return the location of the (possibly remote) resource in the 

553 local file system. 

554 

555 This is a helper function for ``as_local`` context manager. 

556 

557 Returns 

558 ------- 

559 path : `str` 

560 If this is a remote resource, it will be a copy of the resource 

561 on the local file system, probably in a temporary directory. 

562 For a local resource this should be the actual path to the 

563 resource. 

564 is_temporary : `bool` 

565 Indicates if the local path is a temporary file or not. 

566 """ 

567 raise NotImplementedError() 

568 

569 @contextlib.contextmanager 

570 def as_local(self) -> Iterator[ButlerURI]: 

571 """Return the location of the (possibly remote) resource in the 

572 local file system. 

573 

574 Yields 

575 ------ 

576 local : `ButlerURI` 

577 If this is a remote resource, it will be a copy of the resource 

578 on the local file system, probably in a temporary directory. 

579 For a local resource this should be the actual path to the 

580 resource. 

581 

582 Notes 

583 ----- 

584 The context manager will automatically delete any local temporary 

585 file. 

586 

587 Examples 

588 -------- 

589 Should be used as a context manager: 

590 

591 .. code-block:: py 

592 

593 with uri.as_local() as local: 

594 ospath = local.ospath 

595 """ 

596 local_src, is_temporary = self._as_local() 

597 local_uri = ButlerURI(local_src, isTemporary=is_temporary) 

598 

599 try: 

600 yield local_uri 

601 finally: 

602 # The caller might have relocated the temporary file 

603 if is_temporary and local_uri.exists(): 

604 local_uri.remove() 

605 

606 def read(self, size: int = -1) -> bytes: 

607 """Open the resource and return the contents in bytes. 

608 

609 Parameters 

610 ---------- 

611 size : `int`, optional 

612 The number of bytes to read. Negative or omitted indicates 

613 that all data should be read. 

614 """ 

615 raise NotImplementedError() 

616 

617 def write(self, data: bytes, overwrite: bool = True) -> None: 

618 """Write the supplied bytes to the new resource. 

619 

620 Parameters 

621 ---------- 

622 data : `bytes` 

623 The bytes to write to the resource. The entire contents of the 

624 resource will be replaced. 

625 overwrite : `bool`, optional 

626 If `True` the resource will be overwritten if it exists. Otherwise 

627 the write will fail. 

628 """ 

629 raise NotImplementedError() 

630 

631 def mkdir(self) -> None: 

632 """For a dir-like URI, create the directory resource if it does not 

633 already exist. 

634 """ 

635 raise NotImplementedError() 

636 

637 def size(self) -> int: 

638 """For non-dir-like URI, return the size of the resource. 

639 

640 Returns 

641 ------- 

642 sz : `int` 

643 The size in bytes of the resource associated with this URI. 

644 Returns 0 if dir-like. 

645 """ 

646 raise NotImplementedError() 

647 

648 def __str__(self) -> str: 

649 return self.geturl() 

650 

651 def __repr__(self) -> str: 

652 return f'ButlerURI("{self.geturl()}")' 

653 

654 def __eq__(self, other: Any) -> bool: 

655 if not isinstance(other, ButlerURI): 

656 return False 

657 return self.geturl() == other.geturl() 

658 

659 def __copy__(self) -> ButlerURI: 

660 # Implement here because the __new__ method confuses things 

661 # Be careful not to convert a relative schemeless URI to absolute 

662 return type(self)(str(self), forceAbsolute=self.isabs()) 

663 

664 def __deepcopy__(self, memo: Any) -> ButlerURI: 

665 # Implement here because the __new__ method confuses things 

666 return self.__copy__() 

667 

668 def __getnewargs__(self) -> Tuple: 

669 return (str(self),) 

670 

671 @staticmethod 

672 def _fixupPathUri(parsed: urllib.parse.ParseResult, root: Optional[Union[str, ButlerURI]] = None, 

673 forceAbsolute: bool = False, 

674 forceDirectory: bool = False) -> Tuple[urllib.parse.ParseResult, bool]: 

675 """Correct any issues with the supplied URI. 

676 

677 Parameters 

678 ---------- 

679 parsed : `~urllib.parse.ParseResult` 

680 The result from parsing a URI using `urllib.parse`. 

681 root : `str` or `ButlerURI`, ignored 

682 Not used by the this implementation since all URIs are 

683 absolute except for those representing the local file system. 

684 forceAbsolute : `bool`, ignored. 

685 Not used by this implementation. URIs are generally always 

686 absolute. 

687 forceDirectory : `bool`, optional 

688 If `True` forces the URI to end with a separator, otherwise given 

689 URI is interpreted as is. Specifying that the URI is conceptually 

690 equivalent to a directory can break some ambiguities when 

691 interpreting the last element of a path. 

692 

693 Returns 

694 ------- 

695 modified : `~urllib.parse.ParseResult` 

696 Update result if a URI is being handled. 

697 dirLike : `bool` 

698 `True` if given parsed URI has a trailing separator or 

699 forceDirectory is True. Otherwise `False`. 

700 

701 Notes 

702 ----- 

703 Relative paths are explicitly not supported by RFC8089 but `urllib` 

704 does accept URIs of the form ``file:relative/path.ext``. They need 

705 to be turned into absolute paths before they can be used. This is 

706 always done regardless of the ``forceAbsolute`` parameter. 

707 

708 AWS S3 differentiates between keys with trailing POSIX separators (i.e 

709 `/dir` and `/dir/`) whereas POSIX does not neccessarily. 

710 

711 Scheme-less paths are normalized. 

712 """ 

713 # assume we are not dealing with a directory like URI 

714 dirLike = False 

715 

716 # URI is dir-like if explicitly stated or if it ends on a separator 

717 endsOnSep = parsed.path.endswith(posixpath.sep) 

718 if forceDirectory or endsOnSep: 

719 dirLike = True 

720 # only add the separator if it's not already there 

721 if not endsOnSep: 721 ↛ 724line 721 didn't jump to line 724, because the condition on line 721 was never false

722 parsed = parsed._replace(path=parsed.path+posixpath.sep) 

723 

724 return parsed, dirLike 

725 

726 def transfer_from(self, src: ButlerURI, transfer: str, 

727 overwrite: bool = False, 

728 transaction: Optional[Union[DatastoreTransaction, NoTransaction]] = None) -> None: 

729 """Transfer the current resource to a new location. 

730 

731 Parameters 

732 ---------- 

733 src : `ButlerURI` 

734 Source URI. 

735 transfer : `str` 

736 Mode to use for transferring the resource. Generically there are 

737 many standard options: copy, link, symlink, hardlink, relsymlink. 

738 Not all URIs support all modes. 

739 overwrite : `bool`, optional 

740 Allow an existing file to be overwritten. Defaults to `False`. 

741 transaction : `DatastoreTransaction`, optional 

742 A transaction object that can (depending on implementation) 

743 rollback transfers on error. Not guaranteed to be implemented. 

744 

745 Notes 

746 ----- 

747 Conceptually this is hard to scale as the number of URI schemes 

748 grow. The destination URI is more important than the source URI 

749 since that is where all the transfer modes are relevant (with the 

750 complication that "move" deletes the source). 

751 

752 Local file to local file is the fundamental use case but every 

753 other scheme has to support "copy" to local file (with implicit 

754 support for "move") and copy from local file. 

755 All the "link" options tend to be specific to local file systems. 

756 

757 "move" is a "copy" where the remote resource is deleted at the end. 

758 Whether this works depends on the source URI rather than the 

759 destination URI. Reverting a move on transaction rollback is 

760 expected to be problematic if a remote resource was involved. 

761 """ 

762 raise NotImplementedError(f"No transfer modes supported by URI scheme {self.scheme}")