Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22""" 

23Support for generic data stores. 

24""" 

25 

26from __future__ import annotations 

27 

28__all__ = ("DatastoreConfig", "Datastore", "DatastoreValidationError") 

29 

30import contextlib 

31import logging 

32from collections import defaultdict 

33from typing import ( 

34 TYPE_CHECKING, 

35 Any, 

36 Callable, 

37 ClassVar, 

38 Dict, 

39 Iterable, 

40 Iterator, 

41 List, 

42 Mapping, 

43 Optional, 

44 Set, 

45 Tuple, 

46 Type, 

47 Union, 

48) 

49 

50from dataclasses import dataclass 

51from abc import ABCMeta, abstractmethod 

52 

53from lsst.utils import doImport 

54from .config import ConfigSubset, Config 

55from .exceptions import ValidationError, DatasetTypeNotSupportedError 

56from .constraints import Constraints 

57from .storageClass import StorageClassFactory 

58from .fileDataset import FileDataset 

59 

60if TYPE_CHECKING: 60 ↛ 61line 60 didn't jump to line 61, because the condition on line 60 was never true

61 from ..registry.interfaces import DatastoreRegistryBridgeManager 

62 from .datasets import DatasetRef, DatasetType 

63 from .configSupport import LookupKey 

64 from .storageClass import StorageClass 

65 from ._butlerUri import ButlerURI 

66 

67 

68class DatastoreConfig(ConfigSubset): 

69 component = "datastore" 

70 requiredKeys = ("cls",) 

71 defaultConfigFile = "datastore.yaml" 

72 

73 

74class DatastoreValidationError(ValidationError): 

75 """There is a problem with the Datastore configuration. 

76 """ 

77 pass 

78 

79 

80@dataclass(frozen=True) 

81class Event: 

82 __slots__ = {"name", "undoFunc", "args", "kwargs"} 

83 name: str 

84 undoFunc: Callable 

85 args: tuple 

86 kwargs: dict 

87 

88 

89class IngestPrepData: 

90 """A helper base class for `Datastore` ingest implementations. 

91 

92 Datastore implementations will generally need a custom implementation of 

93 this class. 

94 

95 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct 

96 import. 

97 

98 Parameters 

99 ---------- 

100 refs : iterable of `DatasetRef` 

101 References for the datasets that can be ingested by this datastore. 

102 """ 

103 def __init__(self, refs: Iterable[DatasetRef]): 

104 self.refs = {ref.id: ref for ref in refs} 

105 

106 

107class DatastoreTransaction: 

108 """Keeps a log of `Datastore` activity and allow rollback. 

109 

110 Parameters 

111 ---------- 

112 parent : `DatastoreTransaction`, optional 

113 The parent transaction (if any) 

114 """ 

115 Event: ClassVar[Type] = Event 

116 

117 parent: Optional['DatastoreTransaction'] 

118 """The parent transaction. (`DatastoreTransaction`, optional)""" 

119 

120 def __init__(self, parent: Optional[DatastoreTransaction] = None): 

121 self.parent = parent 

122 self._log: List[Event] = [] 

123 

124 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None: 

125 """Register event with undo function. 

126 

127 Parameters 

128 ---------- 

129 name : `str` 

130 Name of the event. 

131 undoFunc : func 

132 Function to undo this event. 

133 args : `tuple` 

134 Positional arguments to `undoFunc`. 

135 kwargs : `dict` 

136 Keyword arguments to `undoFunc`. 

137 """ 

138 self._log.append(self.Event(name, undoFunc, args, kwargs)) 

139 

140 @contextlib.contextmanager 

141 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]: 

142 """A context manager that calls `registerUndo` if the nested operation 

143 does not raise an exception. 

144 

145 This can be used to wrap individual undo-able statements within a 

146 DatastoreTransaction block. Multiple statements that can fail 

147 separately should not be part of the same `undoWith` block. 

148 

149 All arguments are forwarded directly to `registerUndo`. 

150 """ 

151 try: 

152 yield None 

153 except BaseException: 

154 raise 

155 else: 

156 self.registerUndo(name, undoFunc, *args, **kwargs) 

157 

158 def rollback(self) -> None: 

159 """Roll back all events in this transaction. 

160 """ 

161 log = logging.getLogger(__name__) 

162 while self._log: 

163 ev = self._log.pop() 

164 try: 

165 log.debug("Rolling back transaction: %s: %s(%s,%s)", ev.name, 

166 ev.undoFunc, 

167 ",".join(str(a) for a in ev.args), 

168 ",".join(f"{k}={v}" for k, v in ev.kwargs.items())) 

169 except Exception: 

170 # In case we had a problem in stringification of arguments 

171 log.warning("Rolling back transaction: %s", ev.name) 

172 try: 

173 ev.undoFunc(*ev.args, **ev.kwargs) 

174 except BaseException as e: 

175 # Deliberately swallow error that may occur in unrolling 

176 log.warning("Exception: %s caught while unrolling: %s", e, ev.name) 

177 pass 

178 

179 def commit(self) -> None: 

180 """Commit this transaction. 

181 """ 

182 if self.parent is None: 

183 # Just forget about the events, they have already happened. 

184 return 

185 else: 

186 # We may still want to events from this transaction as part of 

187 # the parent. 

188 self.parent._log.extend(self._log) 

189 

190 

191class Datastore(metaclass=ABCMeta): 

192 """Datastore interface. 

193 

194 Parameters 

195 ---------- 

196 config : `DatastoreConfig` or `str` 

197 Load configuration either from an existing config instance or by 

198 referring to a configuration file. 

199 bridgeManager : `DatastoreRegistryBridgeManager` 

200 Object that manages the interface between `Registry` and datastores. 

201 butlerRoot : `str`, optional 

202 New datastore root to use to override the configuration value. 

203 """ 

204 

205 defaultConfigFile: ClassVar[Optional[str]] = None 

206 """Path to configuration defaults. Accessed within the ``config`` resource 

207 or relative to a search path. Can be None if no defaults specified. 

208 """ 

209 

210 containerKey: ClassVar[Optional[str]] = None 

211 """Name of the key containing a list of subconfigurations that also 

212 need to be merged with defaults and will likely use different Python 

213 datastore classes (but all using DatastoreConfig). Assumed to be a 

214 list of configurations that can be represented in a DatastoreConfig 

215 and containing a "cls" definition. None indicates that no containers 

216 are expected in this Datastore.""" 

217 

218 isEphemeral: bool = False 

219 """Indicate whether this Datastore is ephemeral or not. An ephemeral 

220 datastore is one where the contents of the datastore will not exist 

221 across process restarts. This value can change per-instance.""" 

222 

223 config: DatastoreConfig 

224 """Configuration used to create Datastore.""" 

225 

226 name: str 

227 """Label associated with this Datastore.""" 

228 

229 storageClassFactory: StorageClassFactory 

230 """Factory for creating storage class instances from name.""" 

231 

232 constraints: Constraints 

233 """Constraints to apply when putting datasets into the datastore.""" 

234 

235 IngestPrepData: ClassVar = IngestPrepData 

236 """Helper base class for ingest implementations. 

237 """ 

238 

239 @classmethod 

240 @abstractmethod 

241 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

242 """Set any filesystem-dependent config options for this Datastore to 

243 be appropriate for a new empty repository with the given root. 

244 

245 Parameters 

246 ---------- 

247 root : `str` 

248 Filesystem path to the root of the data repository. 

249 config : `Config` 

250 A `Config` to update. Only the subset understood by 

251 this component will be updated. Will not expand 

252 defaults. 

253 full : `Config` 

254 A complete config with all defaults expanded that can be 

255 converted to a `DatastoreConfig`. Read-only and will not be 

256 modified by this method. 

257 Repository-specific options that should not be obtained 

258 from defaults when Butler instances are constructed 

259 should be copied from ``full`` to ``config``. 

260 overwrite : `bool`, optional 

261 If `False`, do not modify a value in ``config`` if the value 

262 already exists. Default is always to overwrite with the provided 

263 ``root``. 

264 

265 Notes 

266 ----- 

267 If a keyword is explicitly defined in the supplied ``config`` it 

268 will not be overridden by this method if ``overwrite`` is `False`. 

269 This allows explicit values set in external configs to be retained. 

270 """ 

271 raise NotImplementedError() 

272 

273 @staticmethod 

274 def fromConfig(config: Config, bridgeManager: DatastoreRegistryBridgeManager, 

275 butlerRoot: Optional[str] = None) -> 'Datastore': 

276 """Create datastore from type specified in config file. 

277 

278 Parameters 

279 ---------- 

280 config : `Config` 

281 Configuration instance. 

282 bridgeManager : `DatastoreRegistryBridgeManager` 

283 Object that manages the interface between `Registry` and 

284 datastores. 

285 butlerRoot : `str`, optional 

286 Butler root directory. 

287 """ 

288 cls = doImport(config["datastore", "cls"]) 

289 return cls(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot) 

290 

291 def __init__(self, config: Union[Config, str], 

292 bridgeManager: DatastoreRegistryBridgeManager, butlerRoot: str = None): 

293 self.config = DatastoreConfig(config) 

294 self.name = "ABCDataStore" 

295 self._transaction: Optional[DatastoreTransaction] = None 

296 

297 # All Datastores need storage classes and constraints 

298 self.storageClassFactory = StorageClassFactory() 

299 

300 # And read the constraints list 

301 constraintsConfig = self.config.get("constraints") 

302 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe) 

303 

304 def __str__(self) -> str: 

305 return self.name 

306 

307 def __repr__(self) -> str: 

308 return self.name 

309 

310 @property 

311 def names(self) -> Tuple[str, ...]: 

312 """Names associated with this datastore returned as a list. 

313 

314 Can be different to ``name`` for a chaining datastore. 

315 """ 

316 # Default implementation returns solely the name itself 

317 return (self.name, ) 

318 

319 @contextlib.contextmanager 

320 def transaction(self) -> Iterator[DatastoreTransaction]: 

321 """Context manager supporting `Datastore` transactions. 

322 

323 Transactions can be nested, and are to be used in combination with 

324 `Registry.transaction`. 

325 """ 

326 self._transaction = DatastoreTransaction(self._transaction) 

327 try: 

328 yield self._transaction 

329 except BaseException: 

330 self._transaction.rollback() 

331 raise 

332 else: 

333 self._transaction.commit() 

334 self._transaction = self._transaction.parent 

335 

336 @abstractmethod 

337 def exists(self, datasetRef: DatasetRef) -> bool: 

338 """Check if the dataset exists in the datastore. 

339 

340 Parameters 

341 ---------- 

342 datasetRef : `DatasetRef` 

343 Reference to the required dataset. 

344 

345 Returns 

346 ------- 

347 exists : `bool` 

348 `True` if the entity exists in the `Datastore`. 

349 """ 

350 raise NotImplementedError("Must be implemented by subclass") 

351 

352 @abstractmethod 

353 def get(self, datasetRef: DatasetRef, parameters: Mapping[str, Any] = None) -> Any: 

354 """Load an `InMemoryDataset` from the store. 

355 

356 Parameters 

357 ---------- 

358 datasetRef : `DatasetRef` 

359 Reference to the required Dataset. 

360 parameters : `dict` 

361 `StorageClass`-specific parameters that specify a slice of the 

362 Dataset to be loaded. 

363 

364 Returns 

365 ------- 

366 inMemoryDataset : `object` 

367 Requested Dataset or slice thereof as an InMemoryDataset. 

368 """ 

369 raise NotImplementedError("Must be implemented by subclass") 

370 

371 @abstractmethod 

372 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

373 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

374 

375 Parameters 

376 ---------- 

377 inMemoryDataset : `object` 

378 The Dataset to store. 

379 datasetRef : `DatasetRef` 

380 Reference to the associated Dataset. 

381 """ 

382 raise NotImplementedError("Must be implemented by subclass") 

383 

384 def _overrideTransferMode(self, *datasets: FileDataset, transfer: Optional[str] = None) -> Optional[str]: 

385 """Allow ingest transfer mode to be defaulted based on datasets. 

386 

387 Parameters 

388 ---------- 

389 datasets : `FileDataset` 

390 Each positional argument is a struct containing information about 

391 a file to be ingested, including its path (either absolute or 

392 relative to the datastore root, if applicable), a complete 

393 `DatasetRef` (with ``dataset_id not None``), and optionally a 

394 formatter class or its fully-qualified string name. If a formatter 

395 is not provided, this method should populate that attribute with 

396 the formatter the datastore would use for `put`. Subclasses are 

397 also permitted to modify the path attribute (typically to put it 

398 in what the datastore considers its standard form). 

399 transfer : `str`, optional 

400 How (and whether) the dataset should be added to the datastore. 

401 See `ingest` for details of transfer modes. 

402 

403 Returns 

404 ------- 

405 newTransfer : `str` 

406 Transfer mode to use. Will be identical to the supplied transfer 

407 mode unless "auto" is used. 

408 """ 

409 if transfer != "auto": 

410 return transfer 

411 raise RuntimeError(f"{transfer} is not allowed without specialization.") 

412 

413 def _prepIngest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> IngestPrepData: 

414 """Process datasets to identify which ones can be ingested into this 

415 Datastore. 

416 

417 Parameters 

418 ---------- 

419 datasets : `FileDataset` 

420 Each positional argument is a struct containing information about 

421 a file to be ingested, including its path (either absolute or 

422 relative to the datastore root, if applicable), a complete 

423 `DatasetRef` (with ``dataset_id not None``), and optionally a 

424 formatter class or its fully-qualified string name. If a formatter 

425 is not provided, this method should populate that attribute with 

426 the formatter the datastore would use for `put`. Subclasses are 

427 also permitted to modify the path attribute (typically to put it 

428 in what the datastore considers its standard form). 

429 transfer : `str`, optional 

430 How (and whether) the dataset should be added to the datastore. 

431 See `ingest` for details of transfer modes. 

432 

433 Returns 

434 ------- 

435 data : `IngestPrepData` 

436 An instance of a subclass of `IngestPrepData`, used to pass 

437 arbitrary data from `_prepIngest` to `_finishIngest`. This should 

438 include only the datasets this datastore can actually ingest; 

439 others should be silently ignored (`Datastore.ingest` will inspect 

440 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if 

441 necessary). 

442 

443 Raises 

444 ------ 

445 NotImplementedError 

446 Raised if the datastore does not support the given transfer mode 

447 (including the case where ingest is not supported at all). 

448 FileNotFoundError 

449 Raised if one of the given files does not exist. 

450 FileExistsError 

451 Raised if transfer is not `None` but the (internal) location the 

452 file would be moved to is already occupied. 

453 

454 Notes 

455 ----- 

456 This method (along with `_finishIngest`) should be implemented by 

457 subclasses to provide ingest support instead of implementing `ingest` 

458 directly. 

459 

460 `_prepIngest` should not modify the data repository or given files in 

461 any way; all changes should be deferred to `_finishIngest`. 

462 

463 When possible, exceptions should be raised in `_prepIngest` instead of 

464 `_finishIngest`. `NotImplementedError` exceptions that indicate that 

465 the transfer mode is not supported must be raised by `_prepIngest` 

466 instead of `_finishIngest`. 

467 """ 

468 raise NotImplementedError( 

469 "Datastore does not support direct file-based ingest." 

470 ) 

471 

472 def _finishIngest(self, prepData: IngestPrepData, *, transfer: Optional[str] = None) -> None: 

473 """Complete an ingest operation. 

474 

475 Parameters 

476 ---------- 

477 data : `IngestPrepData` 

478 An instance of a subclass of `IngestPrepData`. Guaranteed to be 

479 the direct result of a call to `_prepIngest` on this datastore. 

480 transfer : `str`, optional 

481 How (and whether) the dataset should be added to the datastore. 

482 See `ingest` for details of transfer modes. 

483 

484 Raises 

485 ------ 

486 FileNotFoundError 

487 Raised if one of the given files does not exist. 

488 FileExistsError 

489 Raised if transfer is not `None` but the (internal) location the 

490 file would be moved to is already occupied. 

491 

492 Notes 

493 ----- 

494 This method (along with `_prepIngest`) should be implemented by 

495 subclasses to provide ingest support instead of implementing `ingest` 

496 directly. 

497 """ 

498 raise NotImplementedError( 

499 "Datastore does not support direct file-based ingest." 

500 ) 

501 

502 def ingest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> None: 

503 """Ingest one or more files into the datastore. 

504 

505 Parameters 

506 ---------- 

507 datasets : `FileDataset` 

508 Each positional argument is a struct containing information about 

509 a file to be ingested, including its path (either absolute or 

510 relative to the datastore root, if applicable), a complete 

511 `DatasetRef` (with ``dataset_id not None``), and optionally a 

512 formatter class or its fully-qualified string name. If a formatter 

513 is not provided, the one the datastore would use for ``put`` on 

514 that dataset is assumed. 

515 transfer : `str`, optional 

516 How (and whether) the dataset should be added to the datastore. 

517 If `None` (default), the file must already be in a location 

518 appropriate for the datastore (e.g. within its root directory), 

519 and will not be modified. Other choices include "move", "copy", 

520 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

521 special transfer mode that will first try to make a hardlink and 

522 if that fails a symlink will be used instead. "relsymlink" creates 

523 a relative symlink rather than use an absolute path. 

524 Most datastores do not support all transfer modes. 

525 "auto" is a special option that will let the 

526 data store choose the most natural option for itself. 

527 

528 Raises 

529 ------ 

530 NotImplementedError 

531 Raised if the datastore does not support the given transfer mode 

532 (including the case where ingest is not supported at all). 

533 DatasetTypeNotSupportedError 

534 Raised if one or more files to be ingested have a dataset type that 

535 is not supported by the datastore. 

536 FileNotFoundError 

537 Raised if one of the given files does not exist. 

538 FileExistsError 

539 Raised if transfer is not `None` but the (internal) location the 

540 file would be moved to is already occupied. 

541 

542 Notes 

543 ----- 

544 Subclasses should implement `_prepIngest` and `_finishIngest` instead 

545 of implementing `ingest` directly. Datastores that hold and 

546 delegate to child datastores may want to call those methods as well. 

547 

548 Subclasses are encouraged to document their supported transfer modes 

549 in their class documentation. 

550 """ 

551 # Allow a datastore to select a default transfer mode 

552 transfer = self._overrideTransferMode(*datasets, transfer=transfer) 

553 prepData = self._prepIngest(*datasets, transfer=transfer) 

554 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs} 

555 if refs.keys() != prepData.refs.keys(): 

556 unsupported = refs.keys() - prepData.refs.keys() 

557 # Group unsupported refs by DatasetType for an informative 

558 # but still concise error message. 

559 byDatasetType = defaultdict(list) 

560 for datasetId in unsupported: 

561 ref = refs[datasetId] 

562 byDatasetType[ref.datasetType].append(ref) 

563 raise DatasetTypeNotSupportedError( 

564 "DatasetType(s) not supported in ingest: " 

565 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items()) 

566 ) 

567 self._finishIngest(prepData, transfer=transfer) 

568 

569 @abstractmethod 

570 def getURIs(self, datasetRef: DatasetRef, 

571 predict: bool = False) -> Tuple[Optional[ButlerURI], Dict[str, ButlerURI]]: 

572 """Return URIs associated with dataset. 

573 

574 Parameters 

575 ---------- 

576 ref : `DatasetRef` 

577 Reference to the required dataset. 

578 predict : `bool`, optional 

579 If the datastore does not know about the dataset, should it 

580 return a predicted URI or not? 

581 

582 Returns 

583 ------- 

584 primary : `ButlerURI` 

585 The URI to the primary artifact associated with this dataset. 

586 If the dataset was disassembled within the datastore this 

587 may be `None`. 

588 components : `dict` 

589 URIs to any components associated with the dataset artifact. 

590 Can be empty if there are no components. 

591 """ 

592 raise NotImplementedError() 

593 

594 @abstractmethod 

595 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ButlerURI: 

596 """URI to the Dataset. 

597 

598 Parameters 

599 ---------- 

600 datasetRef : `DatasetRef` 

601 Reference to the required Dataset. 

602 predict : `bool` 

603 If `True` attempt to predict the URI for a dataset if it does 

604 not exist in datastore. 

605 

606 Returns 

607 ------- 

608 uri : `str` 

609 URI string pointing to the Dataset within the datastore. If the 

610 Dataset does not exist in the datastore, the URI may be a guess. 

611 If the datastore does not have entities that relate well 

612 to the concept of a URI the returned URI string will be 

613 descriptive. The returned URI is not guaranteed to be obtainable. 

614 

615 Raises 

616 ------ 

617 FileNotFoundError 

618 A URI has been requested for a dataset that does not exist and 

619 guessing is not allowed. 

620 """ 

621 raise NotImplementedError("Must be implemented by subclass") 

622 

623 @abstractmethod 

624 def remove(self, datasetRef: DatasetRef) -> None: 

625 """Indicate to the Datastore that a Dataset can be removed. 

626 

627 Parameters 

628 ---------- 

629 datasetRef : `DatasetRef` 

630 Reference to the required Dataset. 

631 

632 Raises 

633 ------ 

634 FileNotFoundError 

635 When Dataset does not exist. 

636 

637 Notes 

638 ----- 

639 Some Datastores may implement this method as a silent no-op to 

640 disable Dataset deletion through standard interfaces. 

641 """ 

642 raise NotImplementedError("Must be implemented by subclass") 

643 

644 @abstractmethod 

645 def trash(self, datasetRef: DatasetRef, ignore_errors: bool = True) -> None: 

646 """Indicate to the Datastore that a Dataset can be moved to the trash. 

647 

648 Parameters 

649 ---------- 

650 datasetRef : `DatasetRef` 

651 Reference to the required Dataset. 

652 ignore_errors : `bool`, optional 

653 Determine whether errors should be ignored. 

654 

655 Raises 

656 ------ 

657 FileNotFoundError 

658 When Dataset does not exist. 

659 

660 Notes 

661 ----- 

662 Some Datastores may implement this method as a silent no-op to 

663 disable Dataset deletion through standard interfaces. 

664 """ 

665 raise NotImplementedError("Must be implemented by subclass") 

666 

667 @abstractmethod 

668 def emptyTrash(self, ignore_errors: bool = True) -> None: 

669 """Remove all datasets from the trash. 

670 

671 Parameters 

672 ---------- 

673 ignore_errors : `bool`, optional 

674 Determine whether errors should be ignored. 

675 

676 Notes 

677 ----- 

678 Some Datastores may implement this method as a silent no-op to 

679 disable Dataset deletion through standard interfaces. 

680 """ 

681 raise NotImplementedError("Must be implemented by subclass") 

682 

683 @abstractmethod 

684 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

685 """Retrieve a Dataset from an input `Datastore`, and store the result 

686 in this `Datastore`. 

687 

688 Parameters 

689 ---------- 

690 inputDatastore : `Datastore` 

691 The external `Datastore` from which to retreive the Dataset. 

692 datasetRef : `DatasetRef` 

693 Reference to the required Dataset. 

694 """ 

695 raise NotImplementedError("Must be implemented by subclass") 

696 

697 def export(self, refs: Iterable[DatasetRef], *, 

698 directory: Optional[str] = None, transfer: Optional[str] = None) -> Iterable[FileDataset]: 

699 """Export datasets for transfer to another data repository. 

700 

701 Parameters 

702 ---------- 

703 refs : iterable of `DatasetRef` 

704 Dataset references to be exported. 

705 directory : `str`, optional 

706 Path to a directory that should contain files corresponding to 

707 output datasets. Ignored if ``transfer`` is `None`. 

708 transfer : `str`, optional 

709 Mode that should be used to move datasets out of the repository. 

710 Valid options are the same as those of the ``transfer`` argument 

711 to ``ingest``, and datastores may similarly signal that a transfer 

712 mode is not supported by raising `NotImplementedError`. 

713 

714 Returns 

715 ------- 

716 dataset : iterable of `DatasetTransfer` 

717 Structs containing information about the exported datasets, in the 

718 same order as ``refs``. 

719 

720 Raises 

721 ------ 

722 NotImplementedError 

723 Raised if the given transfer mode is not supported. 

724 """ 

725 raise NotImplementedError(f"Transfer mode {transfer} not supported.") 

726 

727 @abstractmethod 

728 def validateConfiguration(self, entities: Iterable[Union[DatasetRef, DatasetType, StorageClass]], 

729 logFailures: bool = False) -> None: 

730 """Validate some of the configuration for this datastore. 

731 

732 Parameters 

733 ---------- 

734 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

735 Entities to test against this configuration. Can be differing 

736 types. 

737 logFailures : `bool`, optional 

738 If `True`, output a log message for every validation error 

739 detected. 

740 

741 Raises 

742 ------ 

743 DatastoreValidationError 

744 Raised if there is a validation problem with a configuration. 

745 

746 Notes 

747 ----- 

748 Which parts of the configuration are validated is at the discretion 

749 of each Datastore implementation. 

750 """ 

751 raise NotImplementedError("Must be implemented by subclass") 

752 

753 @abstractmethod 

754 def validateKey(self, 

755 lookupKey: LookupKey, entity: Union[DatasetRef, DatasetType, StorageClass]) -> None: 

756 """Validate a specific look up key with supplied entity. 

757 

758 Parameters 

759 ---------- 

760 lookupKey : `LookupKey` 

761 Key to use to retrieve information from the datastore 

762 configuration. 

763 entity : `DatasetRef`, `DatasetType`, or `StorageClass` 

764 Entity to compare with configuration retrieved using the 

765 specified lookup key. 

766 

767 Raises 

768 ------ 

769 DatastoreValidationError 

770 Raised if there is a problem with the combination of entity 

771 and lookup key. 

772 

773 Notes 

774 ----- 

775 Bypasses the normal selection priorities by allowing a key that 

776 would normally not be selected to be validated. 

777 """ 

778 raise NotImplementedError("Must be implemented by subclass") 

779 

780 @abstractmethod 

781 def getLookupKeys(self) -> Set[LookupKey]: 

782 """Return all the lookup keys relevant to this datastore. 

783 

784 Returns 

785 ------- 

786 keys : `set` of `LookupKey` 

787 The keys stored internally for looking up information based 

788 on `DatasetType` name or `StorageClass`. 

789 """ 

790 raise NotImplementedError("Must be implemented by subclass")