Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22""" 

23Support for generic data stores. 

24""" 

25 

26from __future__ import annotations 

27 

28__all__ = ("DatastoreConfig", "Datastore", "DatastoreValidationError") 

29 

30import contextlib 

31import logging 

32from collections import defaultdict 

33from typing import ( 

34 TYPE_CHECKING, 

35 Any, 

36 Callable, 

37 ClassVar, 

38 Dict, 

39 Iterable, 

40 Iterator, 

41 List, 

42 Mapping, 

43 Optional, 

44 Set, 

45 Tuple, 

46 Type, 

47 Union, 

48) 

49 

50from dataclasses import dataclass 

51from abc import ABCMeta, abstractmethod 

52 

53from lsst.utils import doImport 

54from .config import ConfigSubset, Config 

55from .exceptions import ValidationError, DatasetTypeNotSupportedError 

56from .constraints import Constraints 

57from .storageClass import StorageClassFactory 

58from .fileDataset import FileDataset 

59 

60if TYPE_CHECKING: 60 ↛ 61line 60 didn't jump to line 61, because the condition on line 60 was never true

61 from ..registry.interfaces import DatastoreRegistryBridgeManager 

62 from .datasets import DatasetRef, DatasetType 

63 from .configSupport import LookupKey 

64 from .storageClass import StorageClass 

65 from ._butlerUri import ButlerURI 

66 

67 

68class DatastoreConfig(ConfigSubset): 

69 component = "datastore" 

70 requiredKeys = ("cls",) 

71 defaultConfigFile = "datastore.yaml" 

72 

73 

74class DatastoreValidationError(ValidationError): 

75 """There is a problem with the Datastore configuration. 

76 """ 

77 pass 

78 

79 

80@dataclass(frozen=True) 

81class Event: 

82 __slots__ = {"name", "undoFunc", "args", "kwargs"} 

83 name: str 

84 undoFunc: Callable 

85 args: tuple 

86 kwargs: dict 

87 

88 

89class IngestPrepData: 

90 """A helper base class for `Datastore` ingest implementations. 

91 

92 Datastore implementations will generally need a custom implementation of 

93 this class. 

94 

95 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct 

96 import. 

97 

98 Parameters 

99 ---------- 

100 refs : iterable of `DatasetRef` 

101 References for the datasets that can be ingested by this datastore. 

102 """ 

103 def __init__(self, refs: Iterable[DatasetRef]): 

104 self.refs = {ref.id: ref for ref in refs} 

105 

106 

107class DatastoreTransaction: 

108 """Keeps a log of `Datastore` activity and allow rollback. 

109 

110 Parameters 

111 ---------- 

112 parent : `DatastoreTransaction`, optional 

113 The parent transaction (if any) 

114 """ 

115 Event: ClassVar[Type] = Event 

116 

117 parent: Optional['DatastoreTransaction'] 

118 """The parent transaction. (`DatastoreTransaction`, optional)""" 

119 

120 def __init__(self, parent: Optional[DatastoreTransaction] = None): 

121 self.parent = parent 

122 self._log: List[Event] = [] 

123 

124 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None: 

125 """Register event with undo function. 

126 

127 Parameters 

128 ---------- 

129 name : `str` 

130 Name of the event. 

131 undoFunc : func 

132 Function to undo this event. 

133 args : `tuple` 

134 Positional arguments to `undoFunc`. 

135 kwargs : `dict` 

136 Keyword arguments to `undoFunc`. 

137 """ 

138 self._log.append(self.Event(name, undoFunc, args, kwargs)) 

139 

140 @contextlib.contextmanager 

141 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]: 

142 """A context manager that calls `registerUndo` if the nested operation 

143 does not raise an exception. 

144 

145 This can be used to wrap individual undo-able statements within a 

146 DatastoreTransaction block. Multiple statements that can fail 

147 separately should not be part of the same `undoWith` block. 

148 

149 All arguments are forwarded directly to `registerUndo`. 

150 """ 

151 try: 

152 yield None 

153 except BaseException: 

154 raise 

155 else: 

156 self.registerUndo(name, undoFunc, *args, **kwargs) 

157 

158 def rollback(self) -> None: 

159 """Roll back all events in this transaction. 

160 """ 

161 log = logging.getLogger(__name__) 

162 while self._log: 

163 ev = self._log.pop() 

164 try: 

165 log.debug("Rolling back transaction: %s: %s(%s,%s)", ev.name, 

166 ev.undoFunc, 

167 ",".join(str(a) for a in ev.args), 

168 ",".join(f"{k}={v}" for k, v in ev.kwargs.items())) 

169 except Exception: 

170 # In case we had a problem in stringification of arguments 

171 log.warning("Rolling back transaction: %s", ev.name) 

172 try: 

173 ev.undoFunc(*ev.args, **ev.kwargs) 

174 except BaseException as e: 

175 # Deliberately swallow error that may occur in unrolling 

176 log.warning("Exception: %s caught while unrolling: %s", e, ev.name) 

177 pass 

178 

179 def commit(self) -> None: 

180 """Commit this transaction. 

181 """ 

182 if self.parent is None: 

183 # Just forget about the events, they have already happened. 

184 return 

185 else: 

186 # We may still want to events from this transaction as part of 

187 # the parent. 

188 self.parent._log.extend(self._log) 

189 

190 

191class Datastore(metaclass=ABCMeta): 

192 """Datastore interface. 

193 

194 Parameters 

195 ---------- 

196 config : `DatastoreConfig` or `str` 

197 Load configuration either from an existing config instance or by 

198 referring to a configuration file. 

199 bridgeManager : `DatastoreRegistryBridgeManager` 

200 Object that manages the interface between `Registry` and datastores. 

201 butlerRoot : `str`, optional 

202 New datastore root to use to override the configuration value. 

203 """ 

204 

205 defaultConfigFile: ClassVar[Optional[str]] = None 

206 """Path to configuration defaults. Accessed within the ``config`` resource 

207 or relative to a search path. Can be None if no defaults specified. 

208 """ 

209 

210 containerKey: ClassVar[Optional[str]] = None 

211 """Name of the key containing a list of subconfigurations that also 

212 need to be merged with defaults and will likely use different Python 

213 datastore classes (but all using DatastoreConfig). Assumed to be a 

214 list of configurations that can be represented in a DatastoreConfig 

215 and containing a "cls" definition. None indicates that no containers 

216 are expected in this Datastore.""" 

217 

218 isEphemeral: bool = False 

219 """Indicate whether this Datastore is ephemeral or not. An ephemeral 

220 datastore is one where the contents of the datastore will not exist 

221 across process restarts. This value can change per-instance.""" 

222 

223 config: DatastoreConfig 

224 """Configuration used to create Datastore.""" 

225 

226 name: str 

227 """Label associated with this Datastore.""" 

228 

229 storageClassFactory: StorageClassFactory 

230 """Factory for creating storage class instances from name.""" 

231 

232 constraints: Constraints 

233 """Constraints to apply when putting datasets into the datastore.""" 

234 

235 # MyPy does not like for this to be annotated as any kind of type, because 

236 # it can't do static checking on type variables that can change at runtime. 

237 IngestPrepData: ClassVar[Any] = IngestPrepData 

238 """Helper base class for ingest implementations. 

239 """ 

240 

241 @classmethod 

242 @abstractmethod 

243 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

244 """Set any filesystem-dependent config options for this Datastore to 

245 be appropriate for a new empty repository with the given root. 

246 

247 Parameters 

248 ---------- 

249 root : `str` 

250 Filesystem path to the root of the data repository. 

251 config : `Config` 

252 A `Config` to update. Only the subset understood by 

253 this component will be updated. Will not expand 

254 defaults. 

255 full : `Config` 

256 A complete config with all defaults expanded that can be 

257 converted to a `DatastoreConfig`. Read-only and will not be 

258 modified by this method. 

259 Repository-specific options that should not be obtained 

260 from defaults when Butler instances are constructed 

261 should be copied from ``full`` to ``config``. 

262 overwrite : `bool`, optional 

263 If `False`, do not modify a value in ``config`` if the value 

264 already exists. Default is always to overwrite with the provided 

265 ``root``. 

266 

267 Notes 

268 ----- 

269 If a keyword is explicitly defined in the supplied ``config`` it 

270 will not be overridden by this method if ``overwrite`` is `False`. 

271 This allows explicit values set in external configs to be retained. 

272 """ 

273 raise NotImplementedError() 

274 

275 @staticmethod 

276 def fromConfig(config: Config, bridgeManager: DatastoreRegistryBridgeManager, 

277 butlerRoot: Optional[Union[str, ButlerURI]] = None) -> 'Datastore': 

278 """Create datastore from type specified in config file. 

279 

280 Parameters 

281 ---------- 

282 config : `Config` 

283 Configuration instance. 

284 bridgeManager : `DatastoreRegistryBridgeManager` 

285 Object that manages the interface between `Registry` and 

286 datastores. 

287 butlerRoot : `str`, optional 

288 Butler root directory. 

289 """ 

290 cls = doImport(config["datastore", "cls"]) 

291 return cls(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot) 

292 

293 def __init__(self, config: Union[Config, str], 

294 bridgeManager: DatastoreRegistryBridgeManager, butlerRoot: str = None): 

295 self.config = DatastoreConfig(config) 

296 self.name = "ABCDataStore" 

297 self._transaction: Optional[DatastoreTransaction] = None 

298 

299 # All Datastores need storage classes and constraints 

300 self.storageClassFactory = StorageClassFactory() 

301 

302 # And read the constraints list 

303 constraintsConfig = self.config.get("constraints") 

304 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe) 

305 

306 def __str__(self) -> str: 

307 return self.name 

308 

309 def __repr__(self) -> str: 

310 return self.name 

311 

312 @property 

313 def names(self) -> Tuple[str, ...]: 

314 """Names associated with this datastore returned as a list. 

315 

316 Can be different to ``name`` for a chaining datastore. 

317 """ 

318 # Default implementation returns solely the name itself 

319 return (self.name, ) 

320 

321 @contextlib.contextmanager 

322 def transaction(self) -> Iterator[DatastoreTransaction]: 

323 """Context manager supporting `Datastore` transactions. 

324 

325 Transactions can be nested, and are to be used in combination with 

326 `Registry.transaction`. 

327 """ 

328 self._transaction = DatastoreTransaction(self._transaction) 

329 try: 

330 yield self._transaction 

331 except BaseException: 

332 self._transaction.rollback() 

333 raise 

334 else: 

335 self._transaction.commit() 

336 self._transaction = self._transaction.parent 

337 

338 @abstractmethod 

339 def exists(self, datasetRef: DatasetRef) -> bool: 

340 """Check if the dataset exists in the datastore. 

341 

342 Parameters 

343 ---------- 

344 datasetRef : `DatasetRef` 

345 Reference to the required dataset. 

346 

347 Returns 

348 ------- 

349 exists : `bool` 

350 `True` if the entity exists in the `Datastore`. 

351 """ 

352 raise NotImplementedError("Must be implemented by subclass") 

353 

354 @abstractmethod 

355 def get(self, datasetRef: DatasetRef, parameters: Mapping[str, Any] = None) -> Any: 

356 """Load an `InMemoryDataset` from the store. 

357 

358 Parameters 

359 ---------- 

360 datasetRef : `DatasetRef` 

361 Reference to the required Dataset. 

362 parameters : `dict` 

363 `StorageClass`-specific parameters that specify a slice of the 

364 Dataset to be loaded. 

365 

366 Returns 

367 ------- 

368 inMemoryDataset : `object` 

369 Requested Dataset or slice thereof as an InMemoryDataset. 

370 """ 

371 raise NotImplementedError("Must be implemented by subclass") 

372 

373 @abstractmethod 

374 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

375 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

376 

377 Parameters 

378 ---------- 

379 inMemoryDataset : `object` 

380 The Dataset to store. 

381 datasetRef : `DatasetRef` 

382 Reference to the associated Dataset. 

383 """ 

384 raise NotImplementedError("Must be implemented by subclass") 

385 

386 def _overrideTransferMode(self, *datasets: FileDataset, transfer: Optional[str] = None) -> Optional[str]: 

387 """Allow ingest transfer mode to be defaulted based on datasets. 

388 

389 Parameters 

390 ---------- 

391 datasets : `FileDataset` 

392 Each positional argument is a struct containing information about 

393 a file to be ingested, including its path (either absolute or 

394 relative to the datastore root, if applicable), a complete 

395 `DatasetRef` (with ``dataset_id not None``), and optionally a 

396 formatter class or its fully-qualified string name. If a formatter 

397 is not provided, this method should populate that attribute with 

398 the formatter the datastore would use for `put`. Subclasses are 

399 also permitted to modify the path attribute (typically to put it 

400 in what the datastore considers its standard form). 

401 transfer : `str`, optional 

402 How (and whether) the dataset should be added to the datastore. 

403 See `ingest` for details of transfer modes. 

404 

405 Returns 

406 ------- 

407 newTransfer : `str` 

408 Transfer mode to use. Will be identical to the supplied transfer 

409 mode unless "auto" is used. 

410 """ 

411 if transfer != "auto": 

412 return transfer 

413 raise RuntimeError(f"{transfer} is not allowed without specialization.") 

414 

415 def _prepIngest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> IngestPrepData: 

416 """Process datasets to identify which ones can be ingested into this 

417 Datastore. 

418 

419 Parameters 

420 ---------- 

421 datasets : `FileDataset` 

422 Each positional argument is a struct containing information about 

423 a file to be ingested, including its path (either absolute or 

424 relative to the datastore root, if applicable), a complete 

425 `DatasetRef` (with ``dataset_id not None``), and optionally a 

426 formatter class or its fully-qualified string name. If a formatter 

427 is not provided, this method should populate that attribute with 

428 the formatter the datastore would use for `put`. Subclasses are 

429 also permitted to modify the path attribute (typically to put it 

430 in what the datastore considers its standard form). 

431 transfer : `str`, optional 

432 How (and whether) the dataset should be added to the datastore. 

433 See `ingest` for details of transfer modes. 

434 

435 Returns 

436 ------- 

437 data : `IngestPrepData` 

438 An instance of a subclass of `IngestPrepData`, used to pass 

439 arbitrary data from `_prepIngest` to `_finishIngest`. This should 

440 include only the datasets this datastore can actually ingest; 

441 others should be silently ignored (`Datastore.ingest` will inspect 

442 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if 

443 necessary). 

444 

445 Raises 

446 ------ 

447 NotImplementedError 

448 Raised if the datastore does not support the given transfer mode 

449 (including the case where ingest is not supported at all). 

450 FileNotFoundError 

451 Raised if one of the given files does not exist. 

452 FileExistsError 

453 Raised if transfer is not `None` but the (internal) location the 

454 file would be moved to is already occupied. 

455 

456 Notes 

457 ----- 

458 This method (along with `_finishIngest`) should be implemented by 

459 subclasses to provide ingest support instead of implementing `ingest` 

460 directly. 

461 

462 `_prepIngest` should not modify the data repository or given files in 

463 any way; all changes should be deferred to `_finishIngest`. 

464 

465 When possible, exceptions should be raised in `_prepIngest` instead of 

466 `_finishIngest`. `NotImplementedError` exceptions that indicate that 

467 the transfer mode is not supported must be raised by `_prepIngest` 

468 instead of `_finishIngest`. 

469 """ 

470 raise NotImplementedError( 

471 "Datastore does not support direct file-based ingest." 

472 ) 

473 

474 def _finishIngest(self, prepData: IngestPrepData, *, transfer: Optional[str] = None) -> None: 

475 """Complete an ingest operation. 

476 

477 Parameters 

478 ---------- 

479 data : `IngestPrepData` 

480 An instance of a subclass of `IngestPrepData`. Guaranteed to be 

481 the direct result of a call to `_prepIngest` on this datastore. 

482 transfer : `str`, optional 

483 How (and whether) the dataset should be added to the datastore. 

484 See `ingest` for details of transfer modes. 

485 

486 Raises 

487 ------ 

488 FileNotFoundError 

489 Raised if one of the given files does not exist. 

490 FileExistsError 

491 Raised if transfer is not `None` but the (internal) location the 

492 file would be moved to is already occupied. 

493 

494 Notes 

495 ----- 

496 This method (along with `_prepIngest`) should be implemented by 

497 subclasses to provide ingest support instead of implementing `ingest` 

498 directly. 

499 """ 

500 raise NotImplementedError( 

501 "Datastore does not support direct file-based ingest." 

502 ) 

503 

504 def ingest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> None: 

505 """Ingest one or more files into the datastore. 

506 

507 Parameters 

508 ---------- 

509 datasets : `FileDataset` 

510 Each positional argument is a struct containing information about 

511 a file to be ingested, including its path (either absolute or 

512 relative to the datastore root, if applicable), a complete 

513 `DatasetRef` (with ``dataset_id not None``), and optionally a 

514 formatter class or its fully-qualified string name. If a formatter 

515 is not provided, the one the datastore would use for ``put`` on 

516 that dataset is assumed. 

517 transfer : `str`, optional 

518 How (and whether) the dataset should be added to the datastore. 

519 If `None` (default), the file must already be in a location 

520 appropriate for the datastore (e.g. within its root directory), 

521 and will not be modified. Other choices include "move", "copy", 

522 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

523 special transfer mode that will first try to make a hardlink and 

524 if that fails a symlink will be used instead. "relsymlink" creates 

525 a relative symlink rather than use an absolute path. 

526 Most datastores do not support all transfer modes. 

527 "auto" is a special option that will let the 

528 data store choose the most natural option for itself. 

529 

530 Raises 

531 ------ 

532 NotImplementedError 

533 Raised if the datastore does not support the given transfer mode 

534 (including the case where ingest is not supported at all). 

535 DatasetTypeNotSupportedError 

536 Raised if one or more files to be ingested have a dataset type that 

537 is not supported by the datastore. 

538 FileNotFoundError 

539 Raised if one of the given files does not exist. 

540 FileExistsError 

541 Raised if transfer is not `None` but the (internal) location the 

542 file would be moved to is already occupied. 

543 

544 Notes 

545 ----- 

546 Subclasses should implement `_prepIngest` and `_finishIngest` instead 

547 of implementing `ingest` directly. Datastores that hold and 

548 delegate to child datastores may want to call those methods as well. 

549 

550 Subclasses are encouraged to document their supported transfer modes 

551 in their class documentation. 

552 """ 

553 # Allow a datastore to select a default transfer mode 

554 transfer = self._overrideTransferMode(*datasets, transfer=transfer) 

555 prepData = self._prepIngest(*datasets, transfer=transfer) 

556 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs} 

557 if refs.keys() != prepData.refs.keys(): 

558 unsupported = refs.keys() - prepData.refs.keys() 

559 # Group unsupported refs by DatasetType for an informative 

560 # but still concise error message. 

561 byDatasetType = defaultdict(list) 

562 for datasetId in unsupported: 

563 ref = refs[datasetId] 

564 byDatasetType[ref.datasetType].append(ref) 

565 raise DatasetTypeNotSupportedError( 

566 "DatasetType(s) not supported in ingest: " 

567 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items()) 

568 ) 

569 self._finishIngest(prepData, transfer=transfer) 

570 

571 @abstractmethod 

572 def getURIs(self, datasetRef: DatasetRef, 

573 predict: bool = False) -> Tuple[Optional[ButlerURI], Dict[str, ButlerURI]]: 

574 """Return URIs associated with dataset. 

575 

576 Parameters 

577 ---------- 

578 ref : `DatasetRef` 

579 Reference to the required dataset. 

580 predict : `bool`, optional 

581 If the datastore does not know about the dataset, should it 

582 return a predicted URI or not? 

583 

584 Returns 

585 ------- 

586 primary : `ButlerURI` 

587 The URI to the primary artifact associated with this dataset. 

588 If the dataset was disassembled within the datastore this 

589 may be `None`. 

590 components : `dict` 

591 URIs to any components associated with the dataset artifact. 

592 Can be empty if there are no components. 

593 """ 

594 raise NotImplementedError() 

595 

596 @abstractmethod 

597 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ButlerURI: 

598 """URI to the Dataset. 

599 

600 Parameters 

601 ---------- 

602 datasetRef : `DatasetRef` 

603 Reference to the required Dataset. 

604 predict : `bool` 

605 If `True` attempt to predict the URI for a dataset if it does 

606 not exist in datastore. 

607 

608 Returns 

609 ------- 

610 uri : `str` 

611 URI string pointing to the Dataset within the datastore. If the 

612 Dataset does not exist in the datastore, the URI may be a guess. 

613 If the datastore does not have entities that relate well 

614 to the concept of a URI the returned URI string will be 

615 descriptive. The returned URI is not guaranteed to be obtainable. 

616 

617 Raises 

618 ------ 

619 FileNotFoundError 

620 A URI has been requested for a dataset that does not exist and 

621 guessing is not allowed. 

622 """ 

623 raise NotImplementedError("Must be implemented by subclass") 

624 

625 @abstractmethod 

626 def remove(self, datasetRef: DatasetRef) -> None: 

627 """Indicate to the Datastore that a Dataset can be removed. 

628 

629 Parameters 

630 ---------- 

631 datasetRef : `DatasetRef` 

632 Reference to the required Dataset. 

633 

634 Raises 

635 ------ 

636 FileNotFoundError 

637 When Dataset does not exist. 

638 

639 Notes 

640 ----- 

641 Some Datastores may implement this method as a silent no-op to 

642 disable Dataset deletion through standard interfaces. 

643 """ 

644 raise NotImplementedError("Must be implemented by subclass") 

645 

646 @abstractmethod 

647 def trash(self, datasetRef: DatasetRef, ignore_errors: bool = True) -> None: 

648 """Indicate to the Datastore that a Dataset can be moved to the trash. 

649 

650 Parameters 

651 ---------- 

652 datasetRef : `DatasetRef` 

653 Reference to the required Dataset. 

654 ignore_errors : `bool`, optional 

655 Determine whether errors should be ignored. 

656 

657 Raises 

658 ------ 

659 FileNotFoundError 

660 When Dataset does not exist. 

661 

662 Notes 

663 ----- 

664 Some Datastores may implement this method as a silent no-op to 

665 disable Dataset deletion through standard interfaces. 

666 """ 

667 raise NotImplementedError("Must be implemented by subclass") 

668 

669 @abstractmethod 

670 def emptyTrash(self, ignore_errors: bool = True) -> None: 

671 """Remove all datasets from the trash. 

672 

673 Parameters 

674 ---------- 

675 ignore_errors : `bool`, optional 

676 Determine whether errors should be ignored. 

677 

678 Notes 

679 ----- 

680 Some Datastores may implement this method as a silent no-op to 

681 disable Dataset deletion through standard interfaces. 

682 """ 

683 raise NotImplementedError("Must be implemented by subclass") 

684 

685 @abstractmethod 

686 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

687 """Retrieve a Dataset from an input `Datastore`, and store the result 

688 in this `Datastore`. 

689 

690 Parameters 

691 ---------- 

692 inputDatastore : `Datastore` 

693 The external `Datastore` from which to retreive the Dataset. 

694 datasetRef : `DatasetRef` 

695 Reference to the required Dataset. 

696 """ 

697 raise NotImplementedError("Must be implemented by subclass") 

698 

699 def export(self, refs: Iterable[DatasetRef], *, 

700 directory: Optional[str] = None, transfer: Optional[str] = None) -> Iterable[FileDataset]: 

701 """Export datasets for transfer to another data repository. 

702 

703 Parameters 

704 ---------- 

705 refs : iterable of `DatasetRef` 

706 Dataset references to be exported. 

707 directory : `str`, optional 

708 Path to a directory that should contain files corresponding to 

709 output datasets. Ignored if ``transfer`` is `None`. 

710 transfer : `str`, optional 

711 Mode that should be used to move datasets out of the repository. 

712 Valid options are the same as those of the ``transfer`` argument 

713 to ``ingest``, and datastores may similarly signal that a transfer 

714 mode is not supported by raising `NotImplementedError`. 

715 

716 Returns 

717 ------- 

718 dataset : iterable of `DatasetTransfer` 

719 Structs containing information about the exported datasets, in the 

720 same order as ``refs``. 

721 

722 Raises 

723 ------ 

724 NotImplementedError 

725 Raised if the given transfer mode is not supported. 

726 """ 

727 raise NotImplementedError(f"Transfer mode {transfer} not supported.") 

728 

729 @abstractmethod 

730 def validateConfiguration(self, entities: Iterable[Union[DatasetRef, DatasetType, StorageClass]], 

731 logFailures: bool = False) -> None: 

732 """Validate some of the configuration for this datastore. 

733 

734 Parameters 

735 ---------- 

736 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

737 Entities to test against this configuration. Can be differing 

738 types. 

739 logFailures : `bool`, optional 

740 If `True`, output a log message for every validation error 

741 detected. 

742 

743 Raises 

744 ------ 

745 DatastoreValidationError 

746 Raised if there is a validation problem with a configuration. 

747 

748 Notes 

749 ----- 

750 Which parts of the configuration are validated is at the discretion 

751 of each Datastore implementation. 

752 """ 

753 raise NotImplementedError("Must be implemented by subclass") 

754 

755 @abstractmethod 

756 def validateKey(self, 

757 lookupKey: LookupKey, entity: Union[DatasetRef, DatasetType, StorageClass]) -> None: 

758 """Validate a specific look up key with supplied entity. 

759 

760 Parameters 

761 ---------- 

762 lookupKey : `LookupKey` 

763 Key to use to retrieve information from the datastore 

764 configuration. 

765 entity : `DatasetRef`, `DatasetType`, or `StorageClass` 

766 Entity to compare with configuration retrieved using the 

767 specified lookup key. 

768 

769 Raises 

770 ------ 

771 DatastoreValidationError 

772 Raised if there is a problem with the combination of entity 

773 and lookup key. 

774 

775 Notes 

776 ----- 

777 Bypasses the normal selection priorities by allowing a key that 

778 would normally not be selected to be validated. 

779 """ 

780 raise NotImplementedError("Must be implemented by subclass") 

781 

782 @abstractmethod 

783 def getLookupKeys(self) -> Set[LookupKey]: 

784 """Return all the lookup keys relevant to this datastore. 

785 

786 Returns 

787 ------- 

788 keys : `set` of `LookupKey` 

789 The keys stored internally for looking up information based 

790 on `DatasetType` name or `StorageClass`. 

791 """ 

792 raise NotImplementedError("Must be implemented by subclass")