Coverage for python/lsst/daf/butler/core/datastore.py : 46%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""
23Support for generic data stores.
24"""
26from __future__ import annotations
28__all__ = ("DatastoreConfig", "Datastore", "DatastoreValidationError")
30import contextlib
31import logging
32from collections import defaultdict
33from typing import (
34 TYPE_CHECKING,
35 Any,
36 Callable,
37 ClassVar,
38 Dict,
39 Iterable,
40 Iterator,
41 List,
42 Mapping,
43 Optional,
44 Set,
45 Tuple,
46 Type,
47 Union,
48)
50from dataclasses import dataclass
51from abc import ABCMeta, abstractmethod
53from lsst.utils import doImport
54from .config import ConfigSubset, Config
55from .exceptions import ValidationError, DatasetTypeNotSupportedError
56from .constraints import Constraints
57from .storageClass import StorageClassFactory
58from .fileDataset import FileDataset
60if TYPE_CHECKING: 60 ↛ 61line 60 didn't jump to line 61, because the condition on line 60 was never true
61 from ..registry.interfaces import DatastoreRegistryBridgeManager
62 from .datasets import DatasetRef, DatasetType
63 from .configSupport import LookupKey
64 from .storageClass import StorageClass
65 from ._butlerUri import ButlerURI
68class DatastoreConfig(ConfigSubset):
69 component = "datastore"
70 requiredKeys = ("cls",)
71 defaultConfigFile = "datastore.yaml"
74class DatastoreValidationError(ValidationError):
75 """There is a problem with the Datastore configuration.
76 """
77 pass
80@dataclass(frozen=True)
81class Event:
82 __slots__ = {"name", "undoFunc", "args", "kwargs"}
83 name: str
84 undoFunc: Callable
85 args: tuple
86 kwargs: dict
89class IngestPrepData:
90 """A helper base class for `Datastore` ingest implementations.
92 Datastore implementations will generally need a custom implementation of
93 this class.
95 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct
96 import.
98 Parameters
99 ----------
100 refs : iterable of `DatasetRef`
101 References for the datasets that can be ingested by this datastore.
102 """
103 def __init__(self, refs: Iterable[DatasetRef]):
104 self.refs = {ref.id: ref for ref in refs}
107class DatastoreTransaction:
108 """Keeps a log of `Datastore` activity and allow rollback.
110 Parameters
111 ----------
112 parent : `DatastoreTransaction`, optional
113 The parent transaction (if any)
114 """
115 Event: ClassVar[Type] = Event
117 parent: Optional['DatastoreTransaction']
118 """The parent transaction. (`DatastoreTransaction`, optional)"""
120 def __init__(self, parent: Optional[DatastoreTransaction] = None):
121 self.parent = parent
122 self._log: List[Event] = []
124 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None:
125 """Register event with undo function.
127 Parameters
128 ----------
129 name : `str`
130 Name of the event.
131 undoFunc : func
132 Function to undo this event.
133 args : `tuple`
134 Positional arguments to `undoFunc`.
135 kwargs : `dict`
136 Keyword arguments to `undoFunc`.
137 """
138 self._log.append(self.Event(name, undoFunc, args, kwargs))
140 @contextlib.contextmanager
141 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]:
142 """A context manager that calls `registerUndo` if the nested operation
143 does not raise an exception.
145 This can be used to wrap individual undo-able statements within a
146 DatastoreTransaction block. Multiple statements that can fail
147 separately should not be part of the same `undoWith` block.
149 All arguments are forwarded directly to `registerUndo`.
150 """
151 try:
152 yield None
153 except BaseException:
154 raise
155 else:
156 self.registerUndo(name, undoFunc, *args, **kwargs)
158 def rollback(self) -> None:
159 """Roll back all events in this transaction.
160 """
161 log = logging.getLogger(__name__)
162 while self._log:
163 ev = self._log.pop()
164 try:
165 log.debug("Rolling back transaction: %s: %s(%s,%s)", ev.name,
166 ev.undoFunc,
167 ",".join(str(a) for a in ev.args),
168 ",".join(f"{k}={v}" for k, v in ev.kwargs.items()))
169 except Exception:
170 # In case we had a problem in stringification of arguments
171 log.warning("Rolling back transaction: %s", ev.name)
172 try:
173 ev.undoFunc(*ev.args, **ev.kwargs)
174 except BaseException as e:
175 # Deliberately swallow error that may occur in unrolling
176 log.warning("Exception: %s caught while unrolling: %s", e, ev.name)
177 pass
179 def commit(self) -> None:
180 """Commit this transaction.
181 """
182 if self.parent is None:
183 # Just forget about the events, they have already happened.
184 return
185 else:
186 # We may still want to events from this transaction as part of
187 # the parent.
188 self.parent._log.extend(self._log)
191class Datastore(metaclass=ABCMeta):
192 """Datastore interface.
194 Parameters
195 ----------
196 config : `DatastoreConfig` or `str`
197 Load configuration either from an existing config instance or by
198 referring to a configuration file.
199 bridgeManager : `DatastoreRegistryBridgeManager`
200 Object that manages the interface between `Registry` and datastores.
201 butlerRoot : `str`, optional
202 New datastore root to use to override the configuration value.
203 """
205 defaultConfigFile: ClassVar[Optional[str]] = None
206 """Path to configuration defaults. Accessed within the ``config`` resource
207 or relative to a search path. Can be None if no defaults specified.
208 """
210 containerKey: ClassVar[Optional[str]] = None
211 """Name of the key containing a list of subconfigurations that also
212 need to be merged with defaults and will likely use different Python
213 datastore classes (but all using DatastoreConfig). Assumed to be a
214 list of configurations that can be represented in a DatastoreConfig
215 and containing a "cls" definition. None indicates that no containers
216 are expected in this Datastore."""
218 isEphemeral: bool = False
219 """Indicate whether this Datastore is ephemeral or not. An ephemeral
220 datastore is one where the contents of the datastore will not exist
221 across process restarts. This value can change per-instance."""
223 config: DatastoreConfig
224 """Configuration used to create Datastore."""
226 name: str
227 """Label associated with this Datastore."""
229 storageClassFactory: StorageClassFactory
230 """Factory for creating storage class instances from name."""
232 constraints: Constraints
233 """Constraints to apply when putting datasets into the datastore."""
235 # MyPy does not like for this to be annotated as any kind of type, because
236 # it can't do static checking on type variables that can change at runtime.
237 IngestPrepData: ClassVar[Any] = IngestPrepData
238 """Helper base class for ingest implementations.
239 """
241 @classmethod
242 @abstractmethod
243 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
244 """Set any filesystem-dependent config options for this Datastore to
245 be appropriate for a new empty repository with the given root.
247 Parameters
248 ----------
249 root : `str`
250 Filesystem path to the root of the data repository.
251 config : `Config`
252 A `Config` to update. Only the subset understood by
253 this component will be updated. Will not expand
254 defaults.
255 full : `Config`
256 A complete config with all defaults expanded that can be
257 converted to a `DatastoreConfig`. Read-only and will not be
258 modified by this method.
259 Repository-specific options that should not be obtained
260 from defaults when Butler instances are constructed
261 should be copied from ``full`` to ``config``.
262 overwrite : `bool`, optional
263 If `False`, do not modify a value in ``config`` if the value
264 already exists. Default is always to overwrite with the provided
265 ``root``.
267 Notes
268 -----
269 If a keyword is explicitly defined in the supplied ``config`` it
270 will not be overridden by this method if ``overwrite`` is `False`.
271 This allows explicit values set in external configs to be retained.
272 """
273 raise NotImplementedError()
275 @staticmethod
276 def fromConfig(config: Config, bridgeManager: DatastoreRegistryBridgeManager,
277 butlerRoot: Optional[Union[str, ButlerURI]] = None) -> 'Datastore':
278 """Create datastore from type specified in config file.
280 Parameters
281 ----------
282 config : `Config`
283 Configuration instance.
284 bridgeManager : `DatastoreRegistryBridgeManager`
285 Object that manages the interface between `Registry` and
286 datastores.
287 butlerRoot : `str`, optional
288 Butler root directory.
289 """
290 cls = doImport(config["datastore", "cls"])
291 return cls(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot)
293 def __init__(self, config: Union[Config, str],
294 bridgeManager: DatastoreRegistryBridgeManager, butlerRoot: str = None):
295 self.config = DatastoreConfig(config)
296 self.name = "ABCDataStore"
297 self._transaction: Optional[DatastoreTransaction] = None
299 # All Datastores need storage classes and constraints
300 self.storageClassFactory = StorageClassFactory()
302 # And read the constraints list
303 constraintsConfig = self.config.get("constraints")
304 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe)
306 def __str__(self) -> str:
307 return self.name
309 def __repr__(self) -> str:
310 return self.name
312 @property
313 def names(self) -> Tuple[str, ...]:
314 """Names associated with this datastore returned as a list.
316 Can be different to ``name`` for a chaining datastore.
317 """
318 # Default implementation returns solely the name itself
319 return (self.name, )
321 @contextlib.contextmanager
322 def transaction(self) -> Iterator[DatastoreTransaction]:
323 """Context manager supporting `Datastore` transactions.
325 Transactions can be nested, and are to be used in combination with
326 `Registry.transaction`.
327 """
328 self._transaction = DatastoreTransaction(self._transaction)
329 try:
330 yield self._transaction
331 except BaseException:
332 self._transaction.rollback()
333 raise
334 else:
335 self._transaction.commit()
336 self._transaction = self._transaction.parent
338 @abstractmethod
339 def exists(self, datasetRef: DatasetRef) -> bool:
340 """Check if the dataset exists in the datastore.
342 Parameters
343 ----------
344 datasetRef : `DatasetRef`
345 Reference to the required dataset.
347 Returns
348 -------
349 exists : `bool`
350 `True` if the entity exists in the `Datastore`.
351 """
352 raise NotImplementedError("Must be implemented by subclass")
354 @abstractmethod
355 def get(self, datasetRef: DatasetRef, parameters: Mapping[str, Any] = None) -> Any:
356 """Load an `InMemoryDataset` from the store.
358 Parameters
359 ----------
360 datasetRef : `DatasetRef`
361 Reference to the required Dataset.
362 parameters : `dict`
363 `StorageClass`-specific parameters that specify a slice of the
364 Dataset to be loaded.
366 Returns
367 -------
368 inMemoryDataset : `object`
369 Requested Dataset or slice thereof as an InMemoryDataset.
370 """
371 raise NotImplementedError("Must be implemented by subclass")
373 @abstractmethod
374 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None:
375 """Write a `InMemoryDataset` with a given `DatasetRef` to the store.
377 Parameters
378 ----------
379 inMemoryDataset : `object`
380 The Dataset to store.
381 datasetRef : `DatasetRef`
382 Reference to the associated Dataset.
383 """
384 raise NotImplementedError("Must be implemented by subclass")
386 def _overrideTransferMode(self, *datasets: FileDataset, transfer: Optional[str] = None) -> Optional[str]:
387 """Allow ingest transfer mode to be defaulted based on datasets.
389 Parameters
390 ----------
391 datasets : `FileDataset`
392 Each positional argument is a struct containing information about
393 a file to be ingested, including its path (either absolute or
394 relative to the datastore root, if applicable), a complete
395 `DatasetRef` (with ``dataset_id not None``), and optionally a
396 formatter class or its fully-qualified string name. If a formatter
397 is not provided, this method should populate that attribute with
398 the formatter the datastore would use for `put`. Subclasses are
399 also permitted to modify the path attribute (typically to put it
400 in what the datastore considers its standard form).
401 transfer : `str`, optional
402 How (and whether) the dataset should be added to the datastore.
403 See `ingest` for details of transfer modes.
405 Returns
406 -------
407 newTransfer : `str`
408 Transfer mode to use. Will be identical to the supplied transfer
409 mode unless "auto" is used.
410 """
411 if transfer != "auto":
412 return transfer
413 raise RuntimeError(f"{transfer} is not allowed without specialization.")
415 def _prepIngest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> IngestPrepData:
416 """Process datasets to identify which ones can be ingested into this
417 Datastore.
419 Parameters
420 ----------
421 datasets : `FileDataset`
422 Each positional argument is a struct containing information about
423 a file to be ingested, including its path (either absolute or
424 relative to the datastore root, if applicable), a complete
425 `DatasetRef` (with ``dataset_id not None``), and optionally a
426 formatter class or its fully-qualified string name. If a formatter
427 is not provided, this method should populate that attribute with
428 the formatter the datastore would use for `put`. Subclasses are
429 also permitted to modify the path attribute (typically to put it
430 in what the datastore considers its standard form).
431 transfer : `str`, optional
432 How (and whether) the dataset should be added to the datastore.
433 See `ingest` for details of transfer modes.
435 Returns
436 -------
437 data : `IngestPrepData`
438 An instance of a subclass of `IngestPrepData`, used to pass
439 arbitrary data from `_prepIngest` to `_finishIngest`. This should
440 include only the datasets this datastore can actually ingest;
441 others should be silently ignored (`Datastore.ingest` will inspect
442 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if
443 necessary).
445 Raises
446 ------
447 NotImplementedError
448 Raised if the datastore does not support the given transfer mode
449 (including the case where ingest is not supported at all).
450 FileNotFoundError
451 Raised if one of the given files does not exist.
452 FileExistsError
453 Raised if transfer is not `None` but the (internal) location the
454 file would be moved to is already occupied.
456 Notes
457 -----
458 This method (along with `_finishIngest`) should be implemented by
459 subclasses to provide ingest support instead of implementing `ingest`
460 directly.
462 `_prepIngest` should not modify the data repository or given files in
463 any way; all changes should be deferred to `_finishIngest`.
465 When possible, exceptions should be raised in `_prepIngest` instead of
466 `_finishIngest`. `NotImplementedError` exceptions that indicate that
467 the transfer mode is not supported must be raised by `_prepIngest`
468 instead of `_finishIngest`.
469 """
470 raise NotImplementedError(
471 "Datastore does not support direct file-based ingest."
472 )
474 def _finishIngest(self, prepData: IngestPrepData, *, transfer: Optional[str] = None) -> None:
475 """Complete an ingest operation.
477 Parameters
478 ----------
479 data : `IngestPrepData`
480 An instance of a subclass of `IngestPrepData`. Guaranteed to be
481 the direct result of a call to `_prepIngest` on this datastore.
482 transfer : `str`, optional
483 How (and whether) the dataset should be added to the datastore.
484 See `ingest` for details of transfer modes.
486 Raises
487 ------
488 FileNotFoundError
489 Raised if one of the given files does not exist.
490 FileExistsError
491 Raised if transfer is not `None` but the (internal) location the
492 file would be moved to is already occupied.
494 Notes
495 -----
496 This method (along with `_prepIngest`) should be implemented by
497 subclasses to provide ingest support instead of implementing `ingest`
498 directly.
499 """
500 raise NotImplementedError(
501 "Datastore does not support direct file-based ingest."
502 )
504 def ingest(self, *datasets: FileDataset, transfer: Optional[str] = None) -> None:
505 """Ingest one or more files into the datastore.
507 Parameters
508 ----------
509 datasets : `FileDataset`
510 Each positional argument is a struct containing information about
511 a file to be ingested, including its path (either absolute or
512 relative to the datastore root, if applicable), a complete
513 `DatasetRef` (with ``dataset_id not None``), and optionally a
514 formatter class or its fully-qualified string name. If a formatter
515 is not provided, the one the datastore would use for ``put`` on
516 that dataset is assumed.
517 transfer : `str`, optional
518 How (and whether) the dataset should be added to the datastore.
519 If `None` (default), the file must already be in a location
520 appropriate for the datastore (e.g. within its root directory),
521 and will not be modified. Other choices include "move", "copy",
522 "link", "symlink", "relsymlink", and "hardlink". "link" is a
523 special transfer mode that will first try to make a hardlink and
524 if that fails a symlink will be used instead. "relsymlink" creates
525 a relative symlink rather than use an absolute path.
526 Most datastores do not support all transfer modes.
527 "auto" is a special option that will let the
528 data store choose the most natural option for itself.
530 Raises
531 ------
532 NotImplementedError
533 Raised if the datastore does not support the given transfer mode
534 (including the case where ingest is not supported at all).
535 DatasetTypeNotSupportedError
536 Raised if one or more files to be ingested have a dataset type that
537 is not supported by the datastore.
538 FileNotFoundError
539 Raised if one of the given files does not exist.
540 FileExistsError
541 Raised if transfer is not `None` but the (internal) location the
542 file would be moved to is already occupied.
544 Notes
545 -----
546 Subclasses should implement `_prepIngest` and `_finishIngest` instead
547 of implementing `ingest` directly. Datastores that hold and
548 delegate to child datastores may want to call those methods as well.
550 Subclasses are encouraged to document their supported transfer modes
551 in their class documentation.
552 """
553 # Allow a datastore to select a default transfer mode
554 transfer = self._overrideTransferMode(*datasets, transfer=transfer)
555 prepData = self._prepIngest(*datasets, transfer=transfer)
556 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs}
557 if refs.keys() != prepData.refs.keys():
558 unsupported = refs.keys() - prepData.refs.keys()
559 # Group unsupported refs by DatasetType for an informative
560 # but still concise error message.
561 byDatasetType = defaultdict(list)
562 for datasetId in unsupported:
563 ref = refs[datasetId]
564 byDatasetType[ref.datasetType].append(ref)
565 raise DatasetTypeNotSupportedError(
566 "DatasetType(s) not supported in ingest: "
567 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items())
568 )
569 self._finishIngest(prepData, transfer=transfer)
571 @abstractmethod
572 def getURIs(self, datasetRef: DatasetRef,
573 predict: bool = False) -> Tuple[Optional[ButlerURI], Dict[str, ButlerURI]]:
574 """Return URIs associated with dataset.
576 Parameters
577 ----------
578 ref : `DatasetRef`
579 Reference to the required dataset.
580 predict : `bool`, optional
581 If the datastore does not know about the dataset, should it
582 return a predicted URI or not?
584 Returns
585 -------
586 primary : `ButlerURI`
587 The URI to the primary artifact associated with this dataset.
588 If the dataset was disassembled within the datastore this
589 may be `None`.
590 components : `dict`
591 URIs to any components associated with the dataset artifact.
592 Can be empty if there are no components.
593 """
594 raise NotImplementedError()
596 @abstractmethod
597 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ButlerURI:
598 """URI to the Dataset.
600 Parameters
601 ----------
602 datasetRef : `DatasetRef`
603 Reference to the required Dataset.
604 predict : `bool`
605 If `True` attempt to predict the URI for a dataset if it does
606 not exist in datastore.
608 Returns
609 -------
610 uri : `str`
611 URI string pointing to the Dataset within the datastore. If the
612 Dataset does not exist in the datastore, the URI may be a guess.
613 If the datastore does not have entities that relate well
614 to the concept of a URI the returned URI string will be
615 descriptive. The returned URI is not guaranteed to be obtainable.
617 Raises
618 ------
619 FileNotFoundError
620 A URI has been requested for a dataset that does not exist and
621 guessing is not allowed.
622 """
623 raise NotImplementedError("Must be implemented by subclass")
625 @abstractmethod
626 def remove(self, datasetRef: DatasetRef) -> None:
627 """Indicate to the Datastore that a Dataset can be removed.
629 Parameters
630 ----------
631 datasetRef : `DatasetRef`
632 Reference to the required Dataset.
634 Raises
635 ------
636 FileNotFoundError
637 When Dataset does not exist.
639 Notes
640 -----
641 Some Datastores may implement this method as a silent no-op to
642 disable Dataset deletion through standard interfaces.
643 """
644 raise NotImplementedError("Must be implemented by subclass")
646 @abstractmethod
647 def trash(self, datasetRef: DatasetRef, ignore_errors: bool = True) -> None:
648 """Indicate to the Datastore that a Dataset can be moved to the trash.
650 Parameters
651 ----------
652 datasetRef : `DatasetRef`
653 Reference to the required Dataset.
654 ignore_errors : `bool`, optional
655 Determine whether errors should be ignored.
657 Raises
658 ------
659 FileNotFoundError
660 When Dataset does not exist.
662 Notes
663 -----
664 Some Datastores may implement this method as a silent no-op to
665 disable Dataset deletion through standard interfaces.
666 """
667 raise NotImplementedError("Must be implemented by subclass")
669 @abstractmethod
670 def emptyTrash(self, ignore_errors: bool = True) -> None:
671 """Remove all datasets from the trash.
673 Parameters
674 ----------
675 ignore_errors : `bool`, optional
676 Determine whether errors should be ignored.
678 Notes
679 -----
680 Some Datastores may implement this method as a silent no-op to
681 disable Dataset deletion through standard interfaces.
682 """
683 raise NotImplementedError("Must be implemented by subclass")
685 @abstractmethod
686 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:
687 """Retrieve a Dataset from an input `Datastore`, and store the result
688 in this `Datastore`.
690 Parameters
691 ----------
692 inputDatastore : `Datastore`
693 The external `Datastore` from which to retreive the Dataset.
694 datasetRef : `DatasetRef`
695 Reference to the required Dataset.
696 """
697 raise NotImplementedError("Must be implemented by subclass")
699 def export(self, refs: Iterable[DatasetRef], *,
700 directory: Optional[str] = None, transfer: Optional[str] = None) -> Iterable[FileDataset]:
701 """Export datasets for transfer to another data repository.
703 Parameters
704 ----------
705 refs : iterable of `DatasetRef`
706 Dataset references to be exported.
707 directory : `str`, optional
708 Path to a directory that should contain files corresponding to
709 output datasets. Ignored if ``transfer`` is `None`.
710 transfer : `str`, optional
711 Mode that should be used to move datasets out of the repository.
712 Valid options are the same as those of the ``transfer`` argument
713 to ``ingest``, and datastores may similarly signal that a transfer
714 mode is not supported by raising `NotImplementedError`.
716 Returns
717 -------
718 dataset : iterable of `DatasetTransfer`
719 Structs containing information about the exported datasets, in the
720 same order as ``refs``.
722 Raises
723 ------
724 NotImplementedError
725 Raised if the given transfer mode is not supported.
726 """
727 raise NotImplementedError(f"Transfer mode {transfer} not supported.")
729 @abstractmethod
730 def validateConfiguration(self, entities: Iterable[Union[DatasetRef, DatasetType, StorageClass]],
731 logFailures: bool = False) -> None:
732 """Validate some of the configuration for this datastore.
734 Parameters
735 ----------
736 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
737 Entities to test against this configuration. Can be differing
738 types.
739 logFailures : `bool`, optional
740 If `True`, output a log message for every validation error
741 detected.
743 Raises
744 ------
745 DatastoreValidationError
746 Raised if there is a validation problem with a configuration.
748 Notes
749 -----
750 Which parts of the configuration are validated is at the discretion
751 of each Datastore implementation.
752 """
753 raise NotImplementedError("Must be implemented by subclass")
755 @abstractmethod
756 def validateKey(self,
757 lookupKey: LookupKey, entity: Union[DatasetRef, DatasetType, StorageClass]) -> None:
758 """Validate a specific look up key with supplied entity.
760 Parameters
761 ----------
762 lookupKey : `LookupKey`
763 Key to use to retrieve information from the datastore
764 configuration.
765 entity : `DatasetRef`, `DatasetType`, or `StorageClass`
766 Entity to compare with configuration retrieved using the
767 specified lookup key.
769 Raises
770 ------
771 DatastoreValidationError
772 Raised if there is a problem with the combination of entity
773 and lookup key.
775 Notes
776 -----
777 Bypasses the normal selection priorities by allowing a key that
778 would normally not be selected to be validated.
779 """
780 raise NotImplementedError("Must be implemented by subclass")
782 @abstractmethod
783 def getLookupKeys(self) -> Set[LookupKey]:
784 """Return all the lookup keys relevant to this datastore.
786 Returns
787 -------
788 keys : `set` of `LookupKey`
789 The keys stored internally for looking up information based
790 on `DatasetType` name or `StorageClass`.
791 """
792 raise NotImplementedError("Must be implemented by subclass")