Coverage for python/lsst/daf/butler/datastore/_datastore.py: 64%
272 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-25 10:50 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-25 10:50 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Support for generic data stores."""
30from __future__ import annotations
32__all__ = (
33 "DatasetRefURIs",
34 "Datastore",
35 "DatastoreConfig",
36 "DatastoreOpaqueTable",
37 "DatastoreValidationError",
38 "NullDatastore",
39 "DatastoreTransaction",
40)
42import contextlib
43import dataclasses
44import logging
45import time
46from abc import ABCMeta, abstractmethod
47from collections import abc, defaultdict
48from collections.abc import Callable, Iterable, Iterator, Mapping
49from typing import TYPE_CHECKING, Any, ClassVar
51from lsst.utils import doImportType
53from .._config import Config, ConfigSubset
54from .._exceptions import DatasetTypeNotSupportedError, ValidationError
55from .._file_dataset import FileDataset
56from .._storage_class import StorageClassFactory
57from .constraints import Constraints
59if TYPE_CHECKING:
60 from lsst.resources import ResourcePath, ResourcePathExpression
62 from .. import ddl
63 from .._config_support import LookupKey
64 from .._dataset_ref import DatasetRef
65 from .._dataset_type import DatasetType
66 from .._storage_class import StorageClass
67 from ..registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
68 from .record_data import DatastoreRecordData
69 from .stored_file_info import StoredDatastoreItemInfo
71_LOG = logging.getLogger(__name__)
74class DatastoreConfig(ConfigSubset):
75 """Configuration for Datastores."""
77 component = "datastore"
78 requiredKeys = ("cls",)
79 defaultConfigFile = "datastore.yaml"
82class DatastoreValidationError(ValidationError):
83 """There is a problem with the Datastore configuration."""
85 pass
88@dataclasses.dataclass(frozen=True)
89class Event:
90 """Representation of an event that can be rolled back."""
92 __slots__ = {"name", "undoFunc", "args", "kwargs"}
93 name: str
94 undoFunc: Callable
95 args: tuple
96 kwargs: dict
99@dataclasses.dataclass(frozen=True)
100class DatastoreOpaqueTable:
101 """Definition of the opaque table which stores datastore records.
103 Table definition contains `.ddl.TableSpec` for a table and a class
104 of a record which must be a subclass of `StoredDatastoreItemInfo`.
105 """
107 __slots__ = {"table_spec", "record_class"}
108 table_spec: ddl.TableSpec
109 record_class: type[StoredDatastoreItemInfo]
112class IngestPrepData:
113 """A helper base class for `Datastore` ingest implementations.
115 Datastore implementations will generally need a custom implementation of
116 this class.
118 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct
119 import.
121 Parameters
122 ----------
123 refs : iterable of `DatasetRef`
124 References for the datasets that can be ingested by this datastore.
125 """
127 def __init__(self, refs: Iterable[DatasetRef]):
128 self.refs = {ref.id: ref for ref in refs}
131class DatastoreTransaction:
132 """Keeps a log of `Datastore` activity and allow rollback.
134 Parameters
135 ----------
136 parent : `DatastoreTransaction`, optional
137 The parent transaction (if any).
138 """
140 Event: ClassVar[type] = Event
142 parent: DatastoreTransaction | None
143 """The parent transaction. (`DatastoreTransaction`, optional)"""
145 def __init__(self, parent: DatastoreTransaction | None = None):
146 self.parent = parent
147 self._log: list[Event] = []
149 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None:
150 """Register event with undo function.
152 Parameters
153 ----------
154 name : `str`
155 Name of the event.
156 undoFunc : `~collections.abc.Callable`
157 Function to undo this event.
158 *args : `tuple`
159 Positional arguments to ``undoFunc``.
160 **kwargs
161 Keyword arguments to ``undoFunc``.
162 """
163 self._log.append(self.Event(name, undoFunc, args, kwargs))
165 @contextlib.contextmanager
166 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]:
167 """Register undo function if nested operation succeeds.
169 Calls `registerUndo`.
171 This can be used to wrap individual undo-able statements within a
172 DatastoreTransaction block. Multiple statements that can fail
173 separately should not be part of the same `undoWith` block.
175 All arguments are forwarded directly to `registerUndo`.
177 Parameters
178 ----------
179 name : `str`
180 The name to associate with this event.
181 undoFunc : `~collections.abc.Callable`
182 Function to undo this event.
183 *args : `tuple`
184 Positional arguments for ``undoFunc``.
185 **kwargs : `typing.Any`
186 Keyword arguments for ``undoFunc``.
187 """
188 try:
189 yield None
190 except BaseException:
191 raise
192 else:
193 self.registerUndo(name, undoFunc, *args, **kwargs)
195 def rollback(self) -> None:
196 """Roll back all events in this transaction."""
197 log = logging.getLogger(__name__)
198 while self._log:
199 ev = self._log.pop()
200 try:
201 log.debug(
202 "Rolling back transaction: %s: %s(%s,%s)",
203 ev.name,
204 ev.undoFunc,
205 ",".join(str(a) for a in ev.args),
206 ",".join(f"{k}={v}" for k, v in ev.kwargs.items()),
207 )
208 except Exception:
209 # In case we had a problem in stringification of arguments
210 log.warning("Rolling back transaction: %s", ev.name)
211 try:
212 ev.undoFunc(*ev.args, **ev.kwargs)
213 except BaseException as e:
214 # Deliberately swallow error that may occur in unrolling
215 log.warning("Exception: %s caught while unrolling: %s", e, ev.name)
216 pass
218 def commit(self) -> None:
219 """Commit this transaction."""
220 if self.parent is None:
221 # Just forget about the events, they have already happened.
222 return
223 else:
224 # We may still want to events from this transaction as part of
225 # the parent.
226 self.parent._log.extend(self._log)
229@dataclasses.dataclass
230class DatasetRefURIs(abc.Sequence):
231 """Represents the primary and component ResourcePath(s) associated with a
232 DatasetRef.
234 This is used in places where its members used to be represented as a tuple
235 `(primaryURI, componentURIs)`. To maintain backward compatibility this
236 inherits from Sequence and so instances can be treated as a two-item
237 tuple.
239 Parameters
240 ----------
241 primaryURI : `lsst.resources.ResourcePath` or `None`, optional
242 The URI to the primary artifact associated with this dataset. If the
243 dataset was disassembled within the datastore this may be `None`.
244 componentURIs : `dict` [`str`, `~lsst.resources.ResourcePath`] or `None`
245 The URIs to any components associated with the dataset artifact
246 indexed by component name. This can be empty if there are no
247 components.
248 """
250 def __init__(
251 self,
252 primaryURI: ResourcePath | None = None,
253 componentURIs: dict[str, ResourcePath] | None = None,
254 ):
255 self.primaryURI = primaryURI
256 self.componentURIs = componentURIs or {}
258 def __getitem__(self, index: Any) -> Any:
259 """Get primaryURI and componentURIs by index.
261 Provides support for tuple-like access.
262 """
263 if index == 0:
264 return self.primaryURI
265 elif index == 1:
266 return self.componentURIs
267 raise IndexError("list index out of range")
269 def __len__(self) -> int:
270 """Get the number of data members.
272 Provides support for tuple-like access.
273 """
274 return 2
276 def __repr__(self) -> str:
277 return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})"
280class Datastore(metaclass=ABCMeta):
281 """Datastore interface.
283 Parameters
284 ----------
285 config : `DatastoreConfig` or `str`
286 Load configuration either from an existing config instance or by
287 referring to a configuration file.
288 bridgeManager : `DatastoreRegistryBridgeManager`
289 Object that manages the interface between `Registry` and datastores.
290 """
292 defaultConfigFile: ClassVar[str | None] = None
293 """Path to configuration defaults. Accessed within the ``config`` resource
294 or relative to a search path. Can be None if no defaults specified.
295 """
297 containerKey: ClassVar[str | None] = None
298 """Name of the key containing a list of subconfigurations that also
299 need to be merged with defaults and will likely use different Python
300 datastore classes (but all using DatastoreConfig). Assumed to be a
301 list of configurations that can be represented in a DatastoreConfig
302 and containing a "cls" definition. None indicates that no containers
303 are expected in this Datastore."""
305 isEphemeral: bool = False
306 """Indicate whether this Datastore is ephemeral or not. An ephemeral
307 datastore is one where the contents of the datastore will not exist
308 across process restarts. This value can change per-instance."""
310 config: DatastoreConfig
311 """Configuration used to create Datastore."""
313 name: str
314 """Label associated with this Datastore."""
316 storageClassFactory: StorageClassFactory
317 """Factory for creating storage class instances from name."""
319 constraints: Constraints
320 """Constraints to apply when putting datasets into the datastore."""
322 # MyPy does not like for this to be annotated as any kind of type, because
323 # it can't do static checking on type variables that can change at runtime.
324 IngestPrepData: ClassVar[Any] = IngestPrepData
325 """Helper base class for ingest implementations.
326 """
328 @classmethod
329 @abstractmethod
330 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
331 """Set filesystem-dependent config options for this datastore.
333 The options will be appropriate for a new empty repository with the
334 given root.
336 Parameters
337 ----------
338 root : `str`
339 Filesystem path to the root of the data repository.
340 config : `Config`
341 A `Config` to update. Only the subset understood by
342 this component will be updated. Will not expand
343 defaults.
344 full : `Config`
345 A complete config with all defaults expanded that can be
346 converted to a `DatastoreConfig`. Read-only and will not be
347 modified by this method.
348 Repository-specific options that should not be obtained
349 from defaults when Butler instances are constructed
350 should be copied from ``full`` to ``config``.
351 overwrite : `bool`, optional
352 If `False`, do not modify a value in ``config`` if the value
353 already exists. Default is always to overwrite with the provided
354 ``root``.
356 Notes
357 -----
358 If a keyword is explicitly defined in the supplied ``config`` it
359 will not be overridden by this method if ``overwrite`` is `False`.
360 This allows explicit values set in external configs to be retained.
361 """
362 raise NotImplementedError()
364 @staticmethod
365 def fromConfig(
366 config: Config,
367 bridgeManager: DatastoreRegistryBridgeManager,
368 butlerRoot: ResourcePathExpression | None = None,
369 ) -> Datastore:
370 """Create datastore from type specified in config file.
372 Parameters
373 ----------
374 config : `Config` or `~lsst.resources.ResourcePathExpression`
375 Configuration instance.
376 bridgeManager : `DatastoreRegistryBridgeManager`
377 Object that manages the interface between `Registry` and
378 datastores.
379 butlerRoot : `str`, optional
380 Butler root directory.
381 """
382 config = DatastoreConfig(config)
383 cls = doImportType(config["cls"])
384 if not issubclass(cls, Datastore):
385 raise TypeError(f"Imported child class {config['cls']} is not a Datastore")
386 return cls._create_from_config(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot)
388 def __init__(
389 self,
390 config: DatastoreConfig,
391 bridgeManager: DatastoreRegistryBridgeManager,
392 ):
393 self.config = config
394 self.name = "ABCDataStore"
395 self._transaction: DatastoreTransaction | None = None
397 # All Datastores need storage classes and constraints
398 self.storageClassFactory = StorageClassFactory()
400 # And read the constraints list
401 constraintsConfig = self.config.get("constraints")
402 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe)
404 @classmethod
405 @abstractmethod
406 def _create_from_config(
407 cls,
408 config: DatastoreConfig,
409 bridgeManager: DatastoreRegistryBridgeManager,
410 butlerRoot: ResourcePathExpression | None,
411 ) -> Datastore:
412 """`Datastore`.``fromConfig`` calls this to instantiate Datastore
413 subclasses. This is the primary constructor for the individual
414 Datastore subclasses.
415 """
416 raise NotImplementedError()
418 @abstractmethod
419 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore:
420 """Make an independent copy of this Datastore with a different
421 `DatastoreRegistryBridgeManager` instance.
423 Parameters
424 ----------
425 bridgeManager : `DatastoreRegistryBridgeManager`
426 New `DatastoreRegistryBridgeManager` object to use when
427 instantiating managers.
429 Returns
430 -------
431 datastore : `Datastore`
432 New `Datastore` instance with the same configuration as the
433 existing instance.
434 """
435 raise NotImplementedError()
437 def __str__(self) -> str:
438 return self.name
440 def __repr__(self) -> str:
441 return self.name
443 @property
444 def names(self) -> tuple[str, ...]:
445 """Names associated with this datastore returned as a list.
447 Can be different to ``name`` for a chaining datastore.
448 """
449 # Default implementation returns solely the name itself
450 return (self.name,)
452 @property
453 def roots(self) -> dict[str, ResourcePath | None]:
454 """Return the root URIs for each named datastore.
456 Mapping from datastore name to root URI. The URI can be `None`
457 if a datastore has no concept of a root URI.
458 (`dict` [`str`, `ResourcePath` | `None`])
459 """
460 return {self.name: None}
462 @contextlib.contextmanager
463 def transaction(self) -> Iterator[DatastoreTransaction]:
464 """Context manager supporting `Datastore` transactions.
466 Transactions can be nested, and are to be used in combination with
467 `Registry.transaction`.
468 """
469 self._transaction = DatastoreTransaction(self._transaction)
470 try:
471 yield self._transaction
472 except BaseException:
473 self._transaction.rollback()
474 raise
475 else:
476 self._transaction.commit()
477 self._transaction = self._transaction.parent
479 @abstractmethod
480 def knows(self, ref: DatasetRef) -> bool:
481 """Check if the dataset is known to the datastore.
483 Does not check for existence of any artifact.
485 Parameters
486 ----------
487 ref : `DatasetRef`
488 Reference to the required dataset.
490 Returns
491 -------
492 exists : `bool`
493 `True` if the dataset is known to the datastore.
494 """
495 raise NotImplementedError()
497 def knows_these(self, refs: Iterable[DatasetRef]) -> dict[DatasetRef, bool]:
498 """Check which of the given datasets are known to this datastore.
500 This is like ``mexist()`` but does not check that the file exists.
502 Parameters
503 ----------
504 refs : iterable `DatasetRef`
505 The datasets to check.
507 Returns
508 -------
509 exists : `dict`[`DatasetRef`, `bool`]
510 Mapping of dataset to boolean indicating whether the dataset
511 is known to the datastore.
512 """
513 # Non-optimized default calls knows() repeatedly.
514 return {ref: self.knows(ref) for ref in refs}
516 def mexists(
517 self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool] | None = None
518 ) -> dict[DatasetRef, bool]:
519 """Check the existence of multiple datasets at once.
521 Parameters
522 ----------
523 refs : iterable of `DatasetRef`
524 The datasets to be checked.
525 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`]
526 Optional mapping of datastore artifact to existence. Updated by
527 this method with details of all artifacts tested. Can be `None`
528 if the caller is not interested.
530 Returns
531 -------
532 existence : `dict` of [`DatasetRef`, `bool`]
533 Mapping from dataset to boolean indicating existence.
534 """
535 existence: dict[DatasetRef, bool] = {}
536 # Non-optimized default.
537 for ref in refs:
538 existence[ref] = self.exists(ref)
539 return existence
541 @abstractmethod
542 def exists(self, datasetRef: DatasetRef) -> bool:
543 """Check if the dataset exists in the datastore.
545 Parameters
546 ----------
547 datasetRef : `DatasetRef`
548 Reference to the required dataset.
550 Returns
551 -------
552 exists : `bool`
553 `True` if the entity exists in the `Datastore`.
554 """
555 raise NotImplementedError("Must be implemented by subclass")
557 @abstractmethod
558 def get(
559 self,
560 datasetRef: DatasetRef,
561 parameters: Mapping[str, Any] | None = None,
562 storageClass: StorageClass | str | None = None,
563 ) -> Any:
564 """Load an `InMemoryDataset` from the store.
566 Parameters
567 ----------
568 datasetRef : `DatasetRef`
569 Reference to the required Dataset.
570 parameters : `dict`
571 `StorageClass`-specific parameters that specify a slice of the
572 Dataset to be loaded.
573 storageClass : `StorageClass` or `str`, optional
574 The storage class to be used to override the Python type
575 returned by this method. By default the returned type matches
576 the dataset type definition for this dataset. Specifying a
577 read `StorageClass` can force a different type to be returned.
578 This type must be compatible with the original type.
580 Returns
581 -------
582 inMemoryDataset : `object`
583 Requested Dataset or slice thereof as an InMemoryDataset.
584 """
585 raise NotImplementedError("Must be implemented by subclass")
587 def prepare_get_for_external_client(self, ref: DatasetRef) -> object:
588 """Retrieve serializable data that can be used to execute a ``get()``.
590 Parameters
591 ----------
592 ref : `DatasetRef`
593 Reference to the required dataset.
595 Returns
596 -------
597 payload : `object`
598 Serializable payload containing the information needed to perform a
599 get() operation. This payload may be sent over the wire to another
600 system to perform the get().
601 """
602 raise NotImplementedError()
604 @abstractmethod
605 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None:
606 """Write a `InMemoryDataset` with a given `DatasetRef` to the store.
608 Parameters
609 ----------
610 inMemoryDataset : `object`
611 The Dataset to store.
612 datasetRef : `DatasetRef`
613 Reference to the associated Dataset.
614 """
615 raise NotImplementedError("Must be implemented by subclass")
617 @abstractmethod
618 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]:
619 """Write a `InMemoryDataset` with a given `DatasetRef` to the store.
621 Parameters
622 ----------
623 in_memory_dataset : `object`
624 The Dataset to store.
625 ref : `DatasetRef`
626 Reference to the associated Dataset.
628 Returns
629 -------
630 datastore_refs : `~collections.abc.Mapping` [`str`, `DatasetRef`]
631 Mapping of a datastore name to dataset reference stored in that
632 datastore, reference will include datastore records. Only
633 non-ephemeral datastores will appear in this mapping.
634 """
635 raise NotImplementedError("Must be implemented by subclass")
637 def _overrideTransferMode(self, *datasets: FileDataset, transfer: str | None = None) -> str | None:
638 """Allow ingest transfer mode to be defaulted based on datasets.
640 Parameters
641 ----------
642 *datasets : `FileDataset`
643 Each positional argument is a struct containing information about
644 a file to be ingested, including its path (either absolute or
645 relative to the datastore root, if applicable), a complete
646 `DatasetRef` (with ``dataset_id not None``), and optionally a
647 formatter class or its fully-qualified string name. If a formatter
648 is not provided, this method should populate that attribute with
649 the formatter the datastore would use for `put`. Subclasses are
650 also permitted to modify the path attribute (typically to put it
651 in what the datastore considers its standard form).
652 transfer : `str`, optional
653 How (and whether) the dataset should be added to the datastore.
654 See `ingest` for details of transfer modes.
656 Returns
657 -------
658 newTransfer : `str`
659 Transfer mode to use. Will be identical to the supplied transfer
660 mode unless "auto" is used.
661 """
662 if transfer != "auto":
663 return transfer
664 raise RuntimeError(f"{transfer} is not allowed without specialization.")
666 def _prepIngest(self, *datasets: FileDataset, transfer: str | None = None) -> IngestPrepData:
667 """Process datasets to identify which ones can be ingested.
669 Parameters
670 ----------
671 *datasets : `FileDataset`
672 Each positional argument is a struct containing information about
673 a file to be ingested, including its path (either absolute or
674 relative to the datastore root, if applicable), a complete
675 `DatasetRef` (with ``dataset_id not None``), and optionally a
676 formatter class or its fully-qualified string name. If a formatter
677 is not provided, this method should populate that attribute with
678 the formatter the datastore would use for `put`. Subclasses are
679 also permitted to modify the path attribute (typically to put it
680 in what the datastore considers its standard form).
681 transfer : `str`, optional
682 How (and whether) the dataset should be added to the datastore.
683 See `ingest` for details of transfer modes.
685 Returns
686 -------
687 data : `IngestPrepData`
688 An instance of a subclass of `IngestPrepData`, used to pass
689 arbitrary data from `_prepIngest` to `_finishIngest`. This should
690 include only the datasets this datastore can actually ingest;
691 others should be silently ignored (`Datastore.ingest` will inspect
692 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if
693 necessary).
695 Raises
696 ------
697 NotImplementedError
698 Raised if the datastore does not support the given transfer mode
699 (including the case where ingest is not supported at all).
700 FileNotFoundError
701 Raised if one of the given files does not exist.
702 FileExistsError
703 Raised if transfer is not `None` but the (internal) location the
704 file would be moved to is already occupied.
706 Notes
707 -----
708 This method (along with `_finishIngest`) should be implemented by
709 subclasses to provide ingest support instead of implementing `ingest`
710 directly.
712 `_prepIngest` should not modify the data repository or given files in
713 any way; all changes should be deferred to `_finishIngest`.
715 When possible, exceptions should be raised in `_prepIngest` instead of
716 `_finishIngest`. `NotImplementedError` exceptions that indicate that
717 the transfer mode is not supported must be raised by `_prepIngest`
718 instead of `_finishIngest`.
719 """
720 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.")
722 def _finishIngest(
723 self, prepData: IngestPrepData, *, transfer: str | None = None, record_validation_info: bool = True
724 ) -> None:
725 """Complete an ingest operation.
727 Parameters
728 ----------
729 prepData : `IngestPrepData`
730 An instance of a subclass of `IngestPrepData`. Guaranteed to be
731 the direct result of a call to `_prepIngest` on this datastore.
732 transfer : `str`, optional
733 How (and whether) the dataset should be added to the datastore.
734 See `ingest` for details of transfer modes.
735 record_validation_info : `bool`, optional
736 If `True`, the default, the datastore can record validation
737 information associated with the file. If `False` the datastore
738 will not attempt to track any information such as checksums
739 or file sizes. This can be useful if such information is tracked
740 in an external system or if the file is to be compressed in place.
741 It is up to the datastore whether this parameter is relevant.
743 Raises
744 ------
745 FileNotFoundError
746 Raised if one of the given files does not exist.
747 FileExistsError
748 Raised if transfer is not `None` but the (internal) location the
749 file would be moved to is already occupied.
751 Notes
752 -----
753 This method (along with `_prepIngest`) should be implemented by
754 subclasses to provide ingest support instead of implementing `ingest`
755 directly.
756 """
757 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.")
759 def ingest(
760 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True
761 ) -> None:
762 """Ingest one or more files into the datastore.
764 Parameters
765 ----------
766 *datasets : `FileDataset`
767 Each positional argument is a struct containing information about
768 a file to be ingested, including its path (either absolute or
769 relative to the datastore root, if applicable), a complete
770 `DatasetRef` (with ``dataset_id not None``), and optionally a
771 formatter class or its fully-qualified string name. If a formatter
772 is not provided, the one the datastore would use for ``put`` on
773 that dataset is assumed.
774 transfer : `str`, optional
775 How (and whether) the dataset should be added to the datastore.
776 If `None` (default), the file must already be in a location
777 appropriate for the datastore (e.g. within its root directory),
778 and will not be modified. Other choices include "move", "copy",
779 "link", "symlink", "relsymlink", and "hardlink". "link" is a
780 special transfer mode that will first try to make a hardlink and
781 if that fails a symlink will be used instead. "relsymlink" creates
782 a relative symlink rather than use an absolute path.
783 Most datastores do not support all transfer modes.
784 "auto" is a special option that will let the
785 data store choose the most natural option for itself.
786 record_validation_info : `bool`, optional
787 If `True`, the default, the datastore can record validation
788 information associated with the file. If `False` the datastore
789 will not attempt to track any information such as checksums
790 or file sizes. This can be useful if such information is tracked
791 in an external system or if the file is to be compressed in place.
792 It is up to the datastore whether this parameter is relevant.
794 Raises
795 ------
796 NotImplementedError
797 Raised if the datastore does not support the given transfer mode
798 (including the case where ingest is not supported at all).
799 DatasetTypeNotSupportedError
800 Raised if one or more files to be ingested have a dataset type that
801 is not supported by the datastore.
802 FileNotFoundError
803 Raised if one of the given files does not exist.
804 FileExistsError
805 Raised if transfer is not `None` but the (internal) location the
806 file would be moved to is already occupied.
808 Notes
809 -----
810 Subclasses should implement `_prepIngest` and `_finishIngest` instead
811 of implementing `ingest` directly. Datastores that hold and
812 delegate to child datastores may want to call those methods as well.
814 Subclasses are encouraged to document their supported transfer modes
815 in their class documentation.
816 """
817 # Allow a datastore to select a default transfer mode
818 transfer = self._overrideTransferMode(*datasets, transfer=transfer)
819 prepData = self._prepIngest(*datasets, transfer=transfer)
820 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs}
821 if refs.keys() != prepData.refs.keys():
822 unsupported = refs.keys() - prepData.refs.keys()
823 # Group unsupported refs by DatasetType for an informative
824 # but still concise error message.
825 byDatasetType = defaultdict(list)
826 for datasetId in unsupported:
827 ref = refs[datasetId]
828 byDatasetType[ref.datasetType].append(ref)
829 raise DatasetTypeNotSupportedError(
830 "DatasetType(s) not supported in ingest: "
831 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items())
832 )
833 self._finishIngest(prepData, transfer=transfer, record_validation_info=record_validation_info)
835 def transfer_from(
836 self,
837 source_datastore: Datastore,
838 refs: Iterable[DatasetRef],
839 transfer: str = "auto",
840 artifact_existence: dict[ResourcePath, bool] | None = None,
841 dry_run: bool = False,
842 ) -> tuple[set[DatasetRef], set[DatasetRef]]:
843 """Transfer dataset artifacts from another datastore to this one.
845 Parameters
846 ----------
847 source_datastore : `Datastore`
848 The datastore from which to transfer artifacts. That datastore
849 must be compatible with this datastore receiving the artifacts.
850 refs : iterable of `DatasetRef`
851 The datasets to transfer from the source datastore.
852 transfer : `str`, optional
853 How (and whether) the dataset should be added to the datastore.
854 Choices include "move", "copy",
855 "link", "symlink", "relsymlink", and "hardlink". "link" is a
856 special transfer mode that will first try to make a hardlink and
857 if that fails a symlink will be used instead. "relsymlink" creates
858 a relative symlink rather than use an absolute path.
859 Most datastores do not support all transfer modes.
860 "auto" (the default) is a special option that will let the
861 data store choose the most natural option for itself.
862 If the source location and transfer location are identical the
863 transfer mode will be ignored.
864 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`]
865 Optional mapping of datastore artifact to existence. Updated by
866 this method with details of all artifacts tested. Can be `None`
867 if the caller is not interested.
868 dry_run : `bool`, optional
869 Process the supplied source refs without updating the target
870 datastore.
872 Returns
873 -------
874 accepted : `set` [`DatasetRef`]
875 The datasets that were transferred.
876 rejected : `set` [`DatasetRef`]
877 The datasets that were rejected due to a constraints violation.
879 Raises
880 ------
881 TypeError
882 Raised if the two datastores are not compatible.
883 """
884 if type(self) is not type(source_datastore):
885 raise TypeError(
886 f"Datastore mismatch between this datastore ({type(self)}) and the "
887 f"source datastore ({type(source_datastore)})."
888 )
890 raise NotImplementedError(f"Datastore {type(self)} must implement a transfer_from method.")
892 def getManyURIs(
893 self,
894 refs: Iterable[DatasetRef],
895 predict: bool = False,
896 allow_missing: bool = False,
897 ) -> dict[DatasetRef, DatasetRefURIs]:
898 """Return URIs associated with many datasets.
900 Parameters
901 ----------
902 refs : iterable of `DatasetIdRef`
903 References to the required datasets.
904 predict : `bool`, optional
905 If `True`, allow URIs to be returned of datasets that have not
906 been written.
907 allow_missing : `bool`
908 If `False`, and ``predict`` is `False`, will raise if a
909 `DatasetRef` does not exist.
911 Returns
912 -------
913 URIs : `dict` of [`DatasetRef`, `DatasetRefUris`]
914 A dict of primary and component URIs, indexed by the passed-in
915 refs.
917 Raises
918 ------
919 FileNotFoundError
920 A URI has been requested for a dataset that does not exist and
921 guessing is not allowed.
923 Notes
924 -----
925 In file-based datastores, getManyURIs does not check that the file is
926 really there, it's assuming it is if datastore is aware of the file
927 then it actually exists.
928 """
929 uris: dict[DatasetRef, DatasetRefURIs] = {}
930 missing_refs = []
931 for ref in refs:
932 try:
933 uris[ref] = self.getURIs(ref, predict=predict)
934 except FileNotFoundError:
935 missing_refs.append(ref)
936 if missing_refs and not allow_missing:
937 raise FileNotFoundError(
938 "Missing {} refs from datastore out of {} and predict=False.".format(
939 num_missing := len(missing_refs), num_missing + len(uris)
940 )
941 )
942 return uris
944 @abstractmethod
945 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs:
946 """Return URIs associated with dataset.
948 Parameters
949 ----------
950 datasetRef : `DatasetRef`
951 Reference to the required dataset.
952 predict : `bool`, optional
953 If the datastore does not know about the dataset, controls whether
954 it should return a predicted URI or not.
956 Returns
957 -------
958 uris : `DatasetRefURIs`
959 The URI to the primary artifact associated with this dataset (if
960 the dataset was disassembled within the datastore this may be
961 `None`), and the URIs to any components associated with the dataset
962 artifact. (can be empty if there are no components).
963 """
964 raise NotImplementedError()
966 @abstractmethod
967 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
968 """URI to the Dataset.
970 Parameters
971 ----------
972 datasetRef : `DatasetRef`
973 Reference to the required Dataset.
974 predict : `bool`
975 If `True` attempt to predict the URI for a dataset if it does
976 not exist in datastore.
978 Returns
979 -------
980 uri : `str`
981 URI string pointing to the Dataset within the datastore. If the
982 Dataset does not exist in the datastore, the URI may be a guess.
983 If the datastore does not have entities that relate well
984 to the concept of a URI the returned URI string will be
985 descriptive. The returned URI is not guaranteed to be obtainable.
987 Raises
988 ------
989 FileNotFoundError
990 A URI has been requested for a dataset that does not exist and
991 guessing is not allowed.
992 """
993 raise NotImplementedError("Must be implemented by subclass")
995 @abstractmethod
996 def retrieveArtifacts(
997 self,
998 refs: Iterable[DatasetRef],
999 destination: ResourcePath,
1000 transfer: str = "auto",
1001 preserve_path: bool = True,
1002 overwrite: bool = False,
1003 ) -> list[ResourcePath]:
1004 """Retrieve the artifacts associated with the supplied refs.
1006 Parameters
1007 ----------
1008 refs : iterable of `DatasetRef`
1009 The datasets for which artifacts are to be retrieved.
1010 A single ref can result in multiple artifacts. The refs must
1011 be resolved.
1012 destination : `lsst.resources.ResourcePath`
1013 Location to write the artifacts.
1014 transfer : `str`, optional
1015 Method to use to transfer the artifacts. Must be one of the options
1016 supported by `lsst.resources.ResourcePath.transfer_from()`.
1017 "move" is not allowed.
1018 preserve_path : `bool`, optional
1019 If `True` the full path of the artifact within the datastore
1020 is preserved. If `False` the final file component of the path
1021 is used.
1022 overwrite : `bool`, optional
1023 If `True` allow transfers to overwrite existing files at the
1024 destination.
1026 Returns
1027 -------
1028 targets : `list` of `lsst.resources.ResourcePath`
1029 URIs of file artifacts in destination location. Order is not
1030 preserved.
1032 Notes
1033 -----
1034 For non-file datastores the artifacts written to the destination
1035 may not match the representation inside the datastore. For example
1036 a hierarchichal data structure in a NoSQL database may well be stored
1037 as a JSON file.
1038 """
1039 raise NotImplementedError()
1041 @abstractmethod
1042 def remove(self, datasetRef: DatasetRef) -> None:
1043 """Indicate to the Datastore that a Dataset can be removed.
1045 Parameters
1046 ----------
1047 datasetRef : `DatasetRef`
1048 Reference to the required Dataset.
1050 Raises
1051 ------
1052 FileNotFoundError
1053 When Dataset does not exist.
1055 Notes
1056 -----
1057 Some Datastores may implement this method as a silent no-op to
1058 disable Dataset deletion through standard interfaces.
1059 """
1060 raise NotImplementedError("Must be implemented by subclass")
1062 @abstractmethod
1063 def forget(self, refs: Iterable[DatasetRef]) -> None:
1064 """Indicate to the Datastore that it should remove all records of the
1065 given datasets, without actually deleting them.
1067 Parameters
1068 ----------
1069 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1070 References to the datasets being forgotten.
1072 Notes
1073 -----
1074 Asking a datastore to forget a `DatasetRef` it does not hold should be
1075 a silent no-op, not an error.
1076 """
1077 raise NotImplementedError("Must be implemented by subclass")
1079 @abstractmethod
1080 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
1081 """Indicate to the Datastore that a Dataset can be moved to the trash.
1083 Parameters
1084 ----------
1085 ref : `DatasetRef` or iterable thereof
1086 Reference(s) to the required Dataset.
1087 ignore_errors : `bool`, optional
1088 Determine whether errors should be ignored. When multiple
1089 refs are being trashed there will be no per-ref check.
1091 Raises
1092 ------
1093 FileNotFoundError
1094 When Dataset does not exist and errors are not ignored. Only
1095 checked if a single ref is supplied (and not in a list).
1097 Notes
1098 -----
1099 Some Datastores may implement this method as a silent no-op to
1100 disable Dataset deletion through standard interfaces.
1101 """
1102 raise NotImplementedError("Must be implemented by subclass")
1104 @abstractmethod
1105 def emptyTrash(self, ignore_errors: bool = True) -> None:
1106 """Remove all datasets from the trash.
1108 Parameters
1109 ----------
1110 ignore_errors : `bool`, optional
1111 Determine whether errors should be ignored.
1113 Notes
1114 -----
1115 Some Datastores may implement this method as a silent no-op to
1116 disable Dataset deletion through standard interfaces.
1117 """
1118 raise NotImplementedError("Must be implemented by subclass")
1120 @abstractmethod
1121 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:
1122 """Transfer a dataset from another datastore to this datastore.
1124 Parameters
1125 ----------
1126 inputDatastore : `Datastore`
1127 The external `Datastore` from which to retrieve the Dataset.
1128 datasetRef : `DatasetRef`
1129 Reference to the required Dataset.
1130 """
1131 raise NotImplementedError("Must be implemented by subclass")
1133 def export(
1134 self,
1135 refs: Iterable[DatasetRef],
1136 *,
1137 directory: ResourcePathExpression | None = None,
1138 transfer: str | None = "auto",
1139 ) -> Iterable[FileDataset]:
1140 """Export datasets for transfer to another data repository.
1142 Parameters
1143 ----------
1144 refs : iterable of `DatasetRef`
1145 Dataset references to be exported.
1146 directory : `str`, optional
1147 Path to a directory that should contain files corresponding to
1148 output datasets. Ignored if ``transfer`` is explicitly `None`.
1149 transfer : `str`, optional
1150 Mode that should be used to move datasets out of the repository.
1151 Valid options are the same as those of the ``transfer`` argument
1152 to ``ingest``, and datastores may similarly signal that a transfer
1153 mode is not supported by raising `NotImplementedError`. If "auto"
1154 is given and no ``directory`` is specified, `None` will be
1155 implied.
1157 Returns
1158 -------
1159 dataset : iterable of `DatasetTransfer`
1160 Structs containing information about the exported datasets, in the
1161 same order as ``refs``.
1163 Raises
1164 ------
1165 NotImplementedError
1166 Raised if the given transfer mode is not supported.
1167 """
1168 raise NotImplementedError(f"Transfer mode {transfer} not supported.")
1170 @abstractmethod
1171 def validateConfiguration(
1172 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False
1173 ) -> None:
1174 """Validate some of the configuration for this datastore.
1176 Parameters
1177 ----------
1178 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
1179 Entities to test against this configuration. Can be differing
1180 types.
1181 logFailures : `bool`, optional
1182 If `True`, output a log message for every validation error
1183 detected.
1185 Raises
1186 ------
1187 DatastoreValidationError
1188 Raised if there is a validation problem with a configuration.
1190 Notes
1191 -----
1192 Which parts of the configuration are validated is at the discretion
1193 of each Datastore implementation.
1194 """
1195 raise NotImplementedError("Must be implemented by subclass")
1197 @abstractmethod
1198 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None:
1199 """Validate a specific look up key with supplied entity.
1201 Parameters
1202 ----------
1203 lookupKey : `LookupKey`
1204 Key to use to retrieve information from the datastore
1205 configuration.
1206 entity : `DatasetRef`, `DatasetType`, or `StorageClass`
1207 Entity to compare with configuration retrieved using the
1208 specified lookup key.
1210 Raises
1211 ------
1212 DatastoreValidationError
1213 Raised if there is a problem with the combination of entity
1214 and lookup key.
1216 Notes
1217 -----
1218 Bypasses the normal selection priorities by allowing a key that
1219 would normally not be selected to be validated.
1220 """
1221 raise NotImplementedError("Must be implemented by subclass")
1223 @abstractmethod
1224 def getLookupKeys(self) -> set[LookupKey]:
1225 """Return all the lookup keys relevant to this datastore.
1227 Returns
1228 -------
1229 keys : `set` of `LookupKey`
1230 The keys stored internally for looking up information based
1231 on `DatasetType` name or `StorageClass`.
1232 """
1233 raise NotImplementedError("Must be implemented by subclass")
1235 def needs_expanded_data_ids(
1236 self,
1237 transfer: str | None,
1238 entity: DatasetRef | DatasetType | StorageClass | None = None,
1239 ) -> bool:
1240 """Test whether this datastore needs expanded data IDs to ingest.
1242 Parameters
1243 ----------
1244 transfer : `str` or `None`
1245 Transfer mode for ingest.
1246 entity : `DatasetRef` or `DatasetType` or `StorageClass` or `None`, \
1247 optional
1248 Object representing what will be ingested. If not provided (or not
1249 specific enough), `True` may be returned even if expanded data
1250 IDs aren't necessary.
1252 Returns
1253 -------
1254 needed : `bool`
1255 If `True`, expanded data IDs may be needed. `False` only if
1256 expansion definitely isn't necessary.
1257 """
1258 return True
1260 @abstractmethod
1261 def import_records(
1262 self,
1263 data: Mapping[str, DatastoreRecordData],
1264 ) -> None:
1265 """Import datastore location and record data from an in-memory data
1266 structure.
1268 Parameters
1269 ----------
1270 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
1271 Datastore records indexed by datastore name. May contain data for
1272 other `Datastore` instances (generally because they are chained to
1273 this one), which should be ignored.
1275 Notes
1276 -----
1277 Implementations should generally not check that any external resources
1278 (e.g. files) referred to by these records actually exist, for
1279 performance reasons; we expect higher-level code to guarantee that they
1280 do.
1282 Implementations are responsible for calling
1283 `DatastoreRegistryBridge.insert` on all datasets in ``data.locations``
1284 where the key is in `names`, as well as loading any opaque table data.
1286 Implementations may assume that datasets are either fully present or
1287 not at all (single-component exports are not permitted).
1288 """
1289 raise NotImplementedError()
1291 @abstractmethod
1292 def export_records(
1293 self,
1294 refs: Iterable[DatasetIdRef],
1295 ) -> Mapping[str, DatastoreRecordData]:
1296 """Export datastore records and locations to an in-memory data
1297 structure.
1299 Parameters
1300 ----------
1301 refs : `~collections.abc.Iterable` [ `DatasetIdRef` ]
1302 Datasets to save. This may include datasets not known to this
1303 datastore, which should be ignored. May not include component
1304 datasets.
1306 Returns
1307 -------
1308 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
1309 Exported datastore records indexed by datastore name.
1310 """
1311 raise NotImplementedError()
1313 def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
1314 """Specify a method that can be used by datastore to retrieve
1315 registry-defined dataset type.
1317 Parameters
1318 ----------
1319 method : `~collections.abc.Callable` | `None`
1320 Method that takes a name of the dataset type and returns a
1321 corresponding `DatasetType` instance as defined in Registry. If
1322 dataset type name is not known to registry `None` is returned.
1324 Notes
1325 -----
1326 This method is only needed for a Datastore supporting a "trusted" mode
1327 when it does not have an access to datastore records and needs to
1328 guess dataset location based on its stored dataset type.
1329 """
1330 pass
1332 @abstractmethod
1333 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
1334 """Make definitions of the opaque tables used by this Datastore.
1336 Returns
1337 -------
1338 tables : `~collections.abc.Mapping` [ `str`, `.ddl.TableSpec` ]
1339 Mapping of opaque table names to their definitions. This can be an
1340 empty mapping if Datastore does not use opaque tables to keep
1341 datastore records.
1342 """
1343 raise NotImplementedError()
1346class NullDatastore(Datastore):
1347 """A datastore that implements the `Datastore` API but always fails when
1348 it accepts any request.
1350 Parameters
1351 ----------
1352 config : `Config` or `~lsst.resources.ResourcePathExpression` or `None`
1353 Ignored.
1354 bridgeManager : `DatastoreRegistryBridgeManager` or `None`
1355 Ignored.
1356 butlerRoot : `~lsst.resources.ResourcePathExpression` or `None`
1357 Ignored.
1358 """
1360 @classmethod
1361 def _create_from_config(
1362 cls,
1363 config: Config,
1364 bridgeManager: DatastoreRegistryBridgeManager,
1365 butlerRoot: ResourcePathExpression | None = None,
1366 ) -> NullDatastore:
1367 return NullDatastore(config, bridgeManager, butlerRoot)
1369 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore:
1370 return self
1372 @classmethod
1373 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
1374 # Nothing to do. This is not a real Datastore.
1375 pass
1377 def __init__(
1378 self,
1379 config: Config | ResourcePathExpression | None,
1380 bridgeManager: DatastoreRegistryBridgeManager | None,
1381 butlerRoot: ResourcePathExpression | None = None,
1382 ):
1383 # Name ourselves with the timestamp the datastore
1384 # was created.
1385 self.name = f"{type(self).__name__}@{time.time()}"
1386 _LOG.debug("Creating datastore %s", self.name)
1388 return
1390 def knows(self, ref: DatasetRef) -> bool:
1391 return False
1393 def exists(self, datasetRef: DatasetRef) -> bool:
1394 return False
1396 def get(
1397 self,
1398 datasetRef: DatasetRef,
1399 parameters: Mapping[str, Any] | None = None,
1400 storageClass: StorageClass | str | None = None,
1401 ) -> Any:
1402 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1404 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None:
1405 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1407 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]:
1408 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1410 def ingest(
1411 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True
1412 ) -> None:
1413 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1415 def transfer_from(
1416 self,
1417 source_datastore: Datastore,
1418 refs: Iterable[DatasetRef],
1419 transfer: str = "auto",
1420 artifact_existence: dict[ResourcePath, bool] | None = None,
1421 dry_run: bool = False,
1422 ) -> tuple[set[DatasetRef], set[DatasetRef]]:
1423 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1425 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs:
1426 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1428 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
1429 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1431 def retrieveArtifacts(
1432 self,
1433 refs: Iterable[DatasetRef],
1434 destination: ResourcePath,
1435 transfer: str = "auto",
1436 preserve_path: bool = True,
1437 overwrite: bool = False,
1438 ) -> list[ResourcePath]:
1439 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1441 def remove(self, datasetRef: DatasetRef) -> None:
1442 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1444 def forget(self, refs: Iterable[DatasetRef]) -> None:
1445 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1447 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
1448 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1450 def emptyTrash(self, ignore_errors: bool = True) -> None:
1451 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1453 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:
1454 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1456 def export(
1457 self,
1458 refs: Iterable[DatasetRef],
1459 *,
1460 directory: ResourcePathExpression | None = None,
1461 transfer: str | None = "auto",
1462 ) -> Iterable[FileDataset]:
1463 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1465 def validateConfiguration(
1466 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False
1467 ) -> None:
1468 # No configuration so always validates.
1469 pass
1471 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None:
1472 pass
1474 def getLookupKeys(self) -> set[LookupKey]:
1475 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1477 def import_records(
1478 self,
1479 data: Mapping[str, DatastoreRecordData],
1480 ) -> None:
1481 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1483 def export_records(
1484 self,
1485 refs: Iterable[DatasetIdRef],
1486 ) -> Mapping[str, DatastoreRecordData]:
1487 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1489 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
1490 return {}