Coverage for python/lsst/daf/butler/datastore/_datastore.py: 64%
275 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-04 02:55 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-04 02:55 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Support for generic data stores."""
30from __future__ import annotations
32__all__ = (
33 "DatasetRefURIs",
34 "Datastore",
35 "DatastoreConfig",
36 "DatastoreOpaqueTable",
37 "DatastoreValidationError",
38 "NullDatastore",
39 "DatastoreTransaction",
40)
42import contextlib
43import dataclasses
44import logging
45import time
46from abc import ABCMeta, abstractmethod
47from collections import abc, defaultdict
48from collections.abc import Callable, Collection, Iterable, Iterator, Mapping
49from typing import TYPE_CHECKING, Any, ClassVar
51from lsst.utils import doImportType
53from .._config import Config, ConfigSubset
54from .._exceptions import DatasetTypeNotSupportedError, ValidationError
55from .._file_dataset import FileDataset
56from .._storage_class import StorageClassFactory
57from .constraints import Constraints
59if TYPE_CHECKING:
60 from lsst.resources import ResourcePath, ResourcePathExpression
62 from .. import ddl
63 from .._config_support import LookupKey
64 from .._dataset_ref import DatasetRef
65 from .._dataset_type import DatasetType
66 from .._storage_class import StorageClass
67 from ..registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
68 from .record_data import DatastoreRecordData
69 from .stored_file_info import StoredDatastoreItemInfo
71_LOG = logging.getLogger(__name__)
74class DatastoreConfig(ConfigSubset):
75 """Configuration for Datastores."""
77 component = "datastore"
78 requiredKeys = ("cls",)
79 defaultConfigFile = "datastore.yaml"
82class DatastoreValidationError(ValidationError):
83 """There is a problem with the Datastore configuration."""
85 pass
88@dataclasses.dataclass(frozen=True)
89class Event:
90 """Representation of an event that can be rolled back."""
92 __slots__ = {"name", "undoFunc", "args", "kwargs"}
93 name: str
94 undoFunc: Callable
95 args: tuple
96 kwargs: dict
99@dataclasses.dataclass(frozen=True)
100class DatastoreOpaqueTable:
101 """Definition of the opaque table which stores datastore records.
103 Table definition contains `.ddl.TableSpec` for a table and a class
104 of a record which must be a subclass of `StoredDatastoreItemInfo`.
105 """
107 __slots__ = {"table_spec", "record_class"}
108 table_spec: ddl.TableSpec
109 record_class: type[StoredDatastoreItemInfo]
112class IngestPrepData:
113 """A helper base class for `Datastore` ingest implementations.
115 Datastore implementations will generally need a custom implementation of
116 this class.
118 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct
119 import.
121 Parameters
122 ----------
123 refs : iterable of `DatasetRef`
124 References for the datasets that can be ingested by this datastore.
125 """
127 def __init__(self, refs: Iterable[DatasetRef]):
128 self.refs = {ref.id: ref for ref in refs}
131class DatastoreTransaction:
132 """Keeps a log of `Datastore` activity and allow rollback.
134 Parameters
135 ----------
136 parent : `DatastoreTransaction`, optional
137 The parent transaction (if any).
138 """
140 Event: ClassVar[type] = Event
142 parent: DatastoreTransaction | None
143 """The parent transaction. (`DatastoreTransaction`, optional)"""
145 def __init__(self, parent: DatastoreTransaction | None = None):
146 self.parent = parent
147 self._log: list[Event] = []
149 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None:
150 """Register event with undo function.
152 Parameters
153 ----------
154 name : `str`
155 Name of the event.
156 undoFunc : `~collections.abc.Callable`
157 Function to undo this event.
158 *args : `tuple`
159 Positional arguments to ``undoFunc``.
160 **kwargs
161 Keyword arguments to ``undoFunc``.
162 """
163 self._log.append(self.Event(name, undoFunc, args, kwargs))
165 @contextlib.contextmanager
166 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]:
167 """Register undo function if nested operation succeeds.
169 Calls `registerUndo`.
171 This can be used to wrap individual undo-able statements within a
172 DatastoreTransaction block. Multiple statements that can fail
173 separately should not be part of the same `undoWith` block.
175 All arguments are forwarded directly to `registerUndo`.
177 Parameters
178 ----------
179 name : `str`
180 The name to associate with this event.
181 undoFunc : `~collections.abc.Callable`
182 Function to undo this event.
183 *args : `tuple`
184 Positional arguments for ``undoFunc``.
185 **kwargs : `typing.Any`
186 Keyword arguments for ``undoFunc``.
187 """
188 try:
189 yield None
190 except BaseException:
191 raise
192 else:
193 self.registerUndo(name, undoFunc, *args, **kwargs)
195 def rollback(self) -> None:
196 """Roll back all events in this transaction."""
197 log = logging.getLogger(__name__)
198 while self._log:
199 ev = self._log.pop()
200 try:
201 log.debug(
202 "Rolling back transaction: %s: %s(%s,%s)",
203 ev.name,
204 ev.undoFunc,
205 ",".join(str(a) for a in ev.args),
206 ",".join(f"{k}={v}" for k, v in ev.kwargs.items()),
207 )
208 except Exception:
209 # In case we had a problem in stringification of arguments
210 log.warning("Rolling back transaction: %s", ev.name)
211 try:
212 ev.undoFunc(*ev.args, **ev.kwargs)
213 except BaseException as e:
214 # Deliberately swallow error that may occur in unrolling
215 log.warning("Exception: %s caught while unrolling: %s", e, ev.name)
216 pass
218 def commit(self) -> None:
219 """Commit this transaction."""
220 if self.parent is None:
221 # Just forget about the events, they have already happened.
222 return
223 else:
224 # We may still want to events from this transaction as part of
225 # the parent.
226 self.parent._log.extend(self._log)
229@dataclasses.dataclass
230class DatasetRefURIs(abc.Sequence):
231 """Represents the primary and component ResourcePath(s) associated with a
232 DatasetRef.
234 This is used in places where its members used to be represented as a tuple
235 `(primaryURI, componentURIs)`. To maintain backward compatibility this
236 inherits from Sequence and so instances can be treated as a two-item
237 tuple.
239 Parameters
240 ----------
241 primaryURI : `lsst.resources.ResourcePath` or `None`, optional
242 The URI to the primary artifact associated with this dataset. If the
243 dataset was disassembled within the datastore this may be `None`.
244 componentURIs : `dict` [`str`, `~lsst.resources.ResourcePath`] or `None`
245 The URIs to any components associated with the dataset artifact
246 indexed by component name. This can be empty if there are no
247 components.
248 """
250 def __init__(
251 self,
252 primaryURI: ResourcePath | None = None,
253 componentURIs: dict[str, ResourcePath] | None = None,
254 ):
255 self.primaryURI = primaryURI
256 self.componentURIs = componentURIs or {}
258 def __getitem__(self, index: Any) -> Any:
259 """Get primaryURI and componentURIs by index.
261 Provides support for tuple-like access.
262 """
263 if index == 0:
264 return self.primaryURI
265 elif index == 1:
266 return self.componentURIs
267 raise IndexError("list index out of range")
269 def __len__(self) -> int:
270 """Get the number of data members.
272 Provides support for tuple-like access.
273 """
274 return 2
276 def __repr__(self) -> str:
277 return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})"
280class Datastore(metaclass=ABCMeta):
281 """Datastore interface.
283 Parameters
284 ----------
285 config : `DatastoreConfig` or `str`
286 Load configuration either from an existing config instance or by
287 referring to a configuration file.
288 bridgeManager : `DatastoreRegistryBridgeManager`
289 Object that manages the interface between `Registry` and datastores.
290 """
292 defaultConfigFile: ClassVar[str | None] = None
293 """Path to configuration defaults. Accessed within the ``config`` resource
294 or relative to a search path. Can be None if no defaults specified.
295 """
297 containerKey: ClassVar[str | None] = None
298 """Name of the key containing a list of subconfigurations that also
299 need to be merged with defaults and will likely use different Python
300 datastore classes (but all using DatastoreConfig). Assumed to be a
301 list of configurations that can be represented in a DatastoreConfig
302 and containing a "cls" definition. None indicates that no containers
303 are expected in this Datastore."""
305 isEphemeral: bool = False
306 """Indicate whether this Datastore is ephemeral or not. An ephemeral
307 datastore is one where the contents of the datastore will not exist
308 across process restarts. This value can change per-instance."""
310 config: DatastoreConfig
311 """Configuration used to create Datastore."""
313 name: str
314 """Label associated with this Datastore."""
316 storageClassFactory: StorageClassFactory
317 """Factory for creating storage class instances from name."""
319 constraints: Constraints
320 """Constraints to apply when putting datasets into the datastore."""
322 # MyPy does not like for this to be annotated as any kind of type, because
323 # it can't do static checking on type variables that can change at runtime.
324 IngestPrepData: ClassVar[Any] = IngestPrepData
325 """Helper base class for ingest implementations.
326 """
328 @classmethod
329 @abstractmethod
330 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
331 """Set filesystem-dependent config options for this datastore.
333 The options will be appropriate for a new empty repository with the
334 given root.
336 Parameters
337 ----------
338 root : `str`
339 Filesystem path to the root of the data repository.
340 config : `Config`
341 A `Config` to update. Only the subset understood by
342 this component will be updated. Will not expand
343 defaults.
344 full : `Config`
345 A complete config with all defaults expanded that can be
346 converted to a `DatastoreConfig`. Read-only and will not be
347 modified by this method.
348 Repository-specific options that should not be obtained
349 from defaults when Butler instances are constructed
350 should be copied from ``full`` to ``config``.
351 overwrite : `bool`, optional
352 If `False`, do not modify a value in ``config`` if the value
353 already exists. Default is always to overwrite with the provided
354 ``root``.
356 Notes
357 -----
358 If a keyword is explicitly defined in the supplied ``config`` it
359 will not be overridden by this method if ``overwrite`` is `False`.
360 This allows explicit values set in external configs to be retained.
361 """
362 raise NotImplementedError()
364 @staticmethod
365 def fromConfig(
366 config: Config,
367 bridgeManager: DatastoreRegistryBridgeManager,
368 butlerRoot: ResourcePathExpression | None = None,
369 ) -> Datastore:
370 """Create datastore from type specified in config file.
372 Parameters
373 ----------
374 config : `Config` or `~lsst.resources.ResourcePathExpression`
375 Configuration instance.
376 bridgeManager : `DatastoreRegistryBridgeManager`
377 Object that manages the interface between `Registry` and
378 datastores.
379 butlerRoot : `str`, optional
380 Butler root directory.
381 """
382 config = DatastoreConfig(config)
383 cls = doImportType(config["cls"])
384 if not issubclass(cls, Datastore):
385 raise TypeError(f"Imported child class {config['cls']} is not a Datastore")
386 return cls._create_from_config(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot)
388 def __init__(
389 self,
390 config: DatastoreConfig,
391 bridgeManager: DatastoreRegistryBridgeManager,
392 ):
393 self.config = config
394 self.name = "ABCDataStore"
395 self._transaction: DatastoreTransaction | None = None
397 # All Datastores need storage classes and constraints
398 self.storageClassFactory = StorageClassFactory()
400 # And read the constraints list
401 constraintsConfig = self.config.get("constraints")
402 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe)
404 @classmethod
405 @abstractmethod
406 def _create_from_config(
407 cls,
408 config: DatastoreConfig,
409 bridgeManager: DatastoreRegistryBridgeManager,
410 butlerRoot: ResourcePathExpression | None,
411 ) -> Datastore:
412 """`Datastore`.``fromConfig`` calls this to instantiate Datastore
413 subclasses. This is the primary constructor for the individual
414 Datastore subclasses.
415 """
416 raise NotImplementedError()
418 @abstractmethod
419 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore:
420 """Make an independent copy of this Datastore with a different
421 `DatastoreRegistryBridgeManager` instance.
423 Parameters
424 ----------
425 bridgeManager : `DatastoreRegistryBridgeManager`
426 New `DatastoreRegistryBridgeManager` object to use when
427 instantiating managers.
429 Returns
430 -------
431 datastore : `Datastore`
432 New `Datastore` instance with the same configuration as the
433 existing instance.
434 """
435 raise NotImplementedError()
437 def __str__(self) -> str:
438 return self.name
440 def __repr__(self) -> str:
441 return self.name
443 @property
444 def names(self) -> tuple[str, ...]:
445 """Names associated with this datastore returned as a list.
447 Can be different to ``name`` for a chaining datastore.
448 """
449 # Default implementation returns solely the name itself
450 return (self.name,)
452 @property
453 def roots(self) -> dict[str, ResourcePath | None]:
454 """Return the root URIs for each named datastore.
456 Mapping from datastore name to root URI. The URI can be `None`
457 if a datastore has no concept of a root URI.
458 (`dict` [`str`, `ResourcePath` | `None`])
459 """
460 return {self.name: None}
462 @contextlib.contextmanager
463 def transaction(self) -> Iterator[DatastoreTransaction]:
464 """Context manager supporting `Datastore` transactions.
466 Transactions can be nested, and are to be used in combination with
467 `Registry.transaction`.
468 """
469 self._transaction = DatastoreTransaction(self._transaction)
470 try:
471 yield self._transaction
472 except BaseException:
473 self._transaction.rollback()
474 raise
475 else:
476 self._transaction.commit()
477 self._transaction = self._transaction.parent
479 def _set_trust_mode(self, mode: bool) -> None:
480 """Set the trust mode for this datastore.
482 Parameters
483 ----------
484 mode : `bool`
485 If `True`, get requests will be attempted even if the datastore
486 does not know about the dataset.
488 Notes
489 -----
490 This is a private method to indicate that trust mode might be a
491 transitory property that we do not want to make fully public. For now
492 only a `~lsst.daf.butler.datastores.FileDatastore` understands this
493 concept. By default this method does nothing.
494 """
495 return
497 @abstractmethod
498 def knows(self, ref: DatasetRef) -> bool:
499 """Check if the dataset is known to the datastore.
501 Does not check for existence of any artifact.
503 Parameters
504 ----------
505 ref : `DatasetRef`
506 Reference to the required dataset.
508 Returns
509 -------
510 exists : `bool`
511 `True` if the dataset is known to the datastore.
512 """
513 raise NotImplementedError()
515 def knows_these(self, refs: Iterable[DatasetRef]) -> dict[DatasetRef, bool]:
516 """Check which of the given datasets are known to this datastore.
518 This is like ``mexist()`` but does not check that the file exists.
520 Parameters
521 ----------
522 refs : iterable `DatasetRef`
523 The datasets to check.
525 Returns
526 -------
527 exists : `dict`[`DatasetRef`, `bool`]
528 Mapping of dataset to boolean indicating whether the dataset
529 is known to the datastore.
530 """
531 # Non-optimized default calls knows() repeatedly.
532 return {ref: self.knows(ref) for ref in refs}
534 def mexists(
535 self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool] | None = None
536 ) -> dict[DatasetRef, bool]:
537 """Check the existence of multiple datasets at once.
539 Parameters
540 ----------
541 refs : iterable of `DatasetRef`
542 The datasets to be checked.
543 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`]
544 Optional mapping of datastore artifact to existence. Updated by
545 this method with details of all artifacts tested. Can be `None`
546 if the caller is not interested.
548 Returns
549 -------
550 existence : `dict` of [`DatasetRef`, `bool`]
551 Mapping from dataset to boolean indicating existence.
552 """
553 existence: dict[DatasetRef, bool] = {}
554 # Non-optimized default.
555 for ref in refs:
556 existence[ref] = self.exists(ref)
557 return existence
559 @abstractmethod
560 def exists(self, datasetRef: DatasetRef) -> bool:
561 """Check if the dataset exists in the datastore.
563 Parameters
564 ----------
565 datasetRef : `DatasetRef`
566 Reference to the required dataset.
568 Returns
569 -------
570 exists : `bool`
571 `True` if the entity exists in the `Datastore`.
572 """
573 raise NotImplementedError("Must be implemented by subclass")
575 @abstractmethod
576 def get(
577 self,
578 datasetRef: DatasetRef,
579 parameters: Mapping[str, Any] | None = None,
580 storageClass: StorageClass | str | None = None,
581 ) -> Any:
582 """Load an `InMemoryDataset` from the store.
584 Parameters
585 ----------
586 datasetRef : `DatasetRef`
587 Reference to the required Dataset.
588 parameters : `dict`
589 `StorageClass`-specific parameters that specify a slice of the
590 Dataset to be loaded.
591 storageClass : `StorageClass` or `str`, optional
592 The storage class to be used to override the Python type
593 returned by this method. By default the returned type matches
594 the dataset type definition for this dataset. Specifying a
595 read `StorageClass` can force a different type to be returned.
596 This type must be compatible with the original type.
598 Returns
599 -------
600 inMemoryDataset : `object`
601 Requested Dataset or slice thereof as an InMemoryDataset.
602 """
603 raise NotImplementedError("Must be implemented by subclass")
605 def prepare_get_for_external_client(self, ref: DatasetRef) -> object | None:
606 """Retrieve serializable data that can be used to execute a ``get()``.
608 Parameters
609 ----------
610 ref : `DatasetRef`
611 Reference to the required dataset.
613 Returns
614 -------
615 payload : `object` | `None`
616 Serializable payload containing the information needed to perform a
617 get() operation. This payload may be sent over the wire to another
618 system to perform the get(). Returns `None` if the dataset is not
619 known to this datastore.
620 """
621 raise NotImplementedError()
623 @abstractmethod
624 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None:
625 """Write a `InMemoryDataset` with a given `DatasetRef` to the store.
627 Parameters
628 ----------
629 inMemoryDataset : `object`
630 The Dataset to store.
631 datasetRef : `DatasetRef`
632 Reference to the associated Dataset.
633 """
634 raise NotImplementedError("Must be implemented by subclass")
636 @abstractmethod
637 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]:
638 """Write a `InMemoryDataset` with a given `DatasetRef` to the store.
640 Parameters
641 ----------
642 in_memory_dataset : `object`
643 The Dataset to store.
644 ref : `DatasetRef`
645 Reference to the associated Dataset.
647 Returns
648 -------
649 datastore_refs : `~collections.abc.Mapping` [`str`, `DatasetRef`]
650 Mapping of a datastore name to dataset reference stored in that
651 datastore, reference will include datastore records. Only
652 non-ephemeral datastores will appear in this mapping.
653 """
654 raise NotImplementedError("Must be implemented by subclass")
656 def _overrideTransferMode(self, *datasets: FileDataset, transfer: str | None = None) -> str | None:
657 """Allow ingest transfer mode to be defaulted based on datasets.
659 Parameters
660 ----------
661 *datasets : `FileDataset`
662 Each positional argument is a struct containing information about
663 a file to be ingested, including its path (either absolute or
664 relative to the datastore root, if applicable), a complete
665 `DatasetRef` (with ``dataset_id not None``), and optionally a
666 formatter class or its fully-qualified string name. If a formatter
667 is not provided, this method should populate that attribute with
668 the formatter the datastore would use for `put`. Subclasses are
669 also permitted to modify the path attribute (typically to put it
670 in what the datastore considers its standard form).
671 transfer : `str`, optional
672 How (and whether) the dataset should be added to the datastore.
673 See `ingest` for details of transfer modes.
675 Returns
676 -------
677 newTransfer : `str`
678 Transfer mode to use. Will be identical to the supplied transfer
679 mode unless "auto" is used.
680 """
681 if transfer != "auto":
682 return transfer
683 raise RuntimeError(f"{transfer} is not allowed without specialization.")
685 def _prepIngest(self, *datasets: FileDataset, transfer: str | None = None) -> IngestPrepData:
686 """Process datasets to identify which ones can be ingested.
688 Parameters
689 ----------
690 *datasets : `FileDataset`
691 Each positional argument is a struct containing information about
692 a file to be ingested, including its path (either absolute or
693 relative to the datastore root, if applicable), a complete
694 `DatasetRef` (with ``dataset_id not None``), and optionally a
695 formatter class or its fully-qualified string name. If a formatter
696 is not provided, this method should populate that attribute with
697 the formatter the datastore would use for `put`. Subclasses are
698 also permitted to modify the path attribute (typically to put it
699 in what the datastore considers its standard form).
700 transfer : `str`, optional
701 How (and whether) the dataset should be added to the datastore.
702 See `ingest` for details of transfer modes.
704 Returns
705 -------
706 data : `IngestPrepData`
707 An instance of a subclass of `IngestPrepData`, used to pass
708 arbitrary data from `_prepIngest` to `_finishIngest`. This should
709 include only the datasets this datastore can actually ingest;
710 others should be silently ignored (`Datastore.ingest` will inspect
711 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if
712 necessary).
714 Raises
715 ------
716 NotImplementedError
717 Raised if the datastore does not support the given transfer mode
718 (including the case where ingest is not supported at all).
719 FileNotFoundError
720 Raised if one of the given files does not exist.
721 FileExistsError
722 Raised if transfer is not `None` but the (internal) location the
723 file would be moved to is already occupied.
725 Notes
726 -----
727 This method (along with `_finishIngest`) should be implemented by
728 subclasses to provide ingest support instead of implementing `ingest`
729 directly.
731 `_prepIngest` should not modify the data repository or given files in
732 any way; all changes should be deferred to `_finishIngest`.
734 When possible, exceptions should be raised in `_prepIngest` instead of
735 `_finishIngest`. `NotImplementedError` exceptions that indicate that
736 the transfer mode is not supported must be raised by `_prepIngest`
737 instead of `_finishIngest`.
738 """
739 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.")
741 def _finishIngest(
742 self, prepData: IngestPrepData, *, transfer: str | None = None, record_validation_info: bool = True
743 ) -> None:
744 """Complete an ingest operation.
746 Parameters
747 ----------
748 prepData : `IngestPrepData`
749 An instance of a subclass of `IngestPrepData`. Guaranteed to be
750 the direct result of a call to `_prepIngest` on this datastore.
751 transfer : `str`, optional
752 How (and whether) the dataset should be added to the datastore.
753 See `ingest` for details of transfer modes.
754 record_validation_info : `bool`, optional
755 If `True`, the default, the datastore can record validation
756 information associated with the file. If `False` the datastore
757 will not attempt to track any information such as checksums
758 or file sizes. This can be useful if such information is tracked
759 in an external system or if the file is to be compressed in place.
760 It is up to the datastore whether this parameter is relevant.
762 Raises
763 ------
764 FileNotFoundError
765 Raised if one of the given files does not exist.
766 FileExistsError
767 Raised if transfer is not `None` but the (internal) location the
768 file would be moved to is already occupied.
770 Notes
771 -----
772 This method (along with `_prepIngest`) should be implemented by
773 subclasses to provide ingest support instead of implementing `ingest`
774 directly.
775 """
776 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.")
778 def ingest(
779 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True
780 ) -> None:
781 """Ingest one or more files into the datastore.
783 Parameters
784 ----------
785 *datasets : `FileDataset`
786 Each positional argument is a struct containing information about
787 a file to be ingested, including its path (either absolute or
788 relative to the datastore root, if applicable), a complete
789 `DatasetRef` (with ``dataset_id not None``), and optionally a
790 formatter class or its fully-qualified string name. If a formatter
791 is not provided, the one the datastore would use for ``put`` on
792 that dataset is assumed.
793 transfer : `str`, optional
794 How (and whether) the dataset should be added to the datastore.
795 If `None` (default), the file must already be in a location
796 appropriate for the datastore (e.g. within its root directory),
797 and will not be modified. Other choices include "move", "copy",
798 "link", "symlink", "relsymlink", and "hardlink". "link" is a
799 special transfer mode that will first try to make a hardlink and
800 if that fails a symlink will be used instead. "relsymlink" creates
801 a relative symlink rather than use an absolute path.
802 Most datastores do not support all transfer modes.
803 "auto" is a special option that will let the
804 data store choose the most natural option for itself.
805 record_validation_info : `bool`, optional
806 If `True`, the default, the datastore can record validation
807 information associated with the file. If `False` the datastore
808 will not attempt to track any information such as checksums
809 or file sizes. This can be useful if such information is tracked
810 in an external system or if the file is to be compressed in place.
811 It is up to the datastore whether this parameter is relevant.
813 Raises
814 ------
815 NotImplementedError
816 Raised if the datastore does not support the given transfer mode
817 (including the case where ingest is not supported at all).
818 DatasetTypeNotSupportedError
819 Raised if one or more files to be ingested have a dataset type that
820 is not supported by the datastore.
821 FileNotFoundError
822 Raised if one of the given files does not exist.
823 FileExistsError
824 Raised if transfer is not `None` but the (internal) location the
825 file would be moved to is already occupied.
827 Notes
828 -----
829 Subclasses should implement `_prepIngest` and `_finishIngest` instead
830 of implementing `ingest` directly. Datastores that hold and
831 delegate to child datastores may want to call those methods as well.
833 Subclasses are encouraged to document their supported transfer modes
834 in their class documentation.
835 """
836 # Allow a datastore to select a default transfer mode
837 transfer = self._overrideTransferMode(*datasets, transfer=transfer)
838 prepData = self._prepIngest(*datasets, transfer=transfer)
839 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs}
840 if refs.keys() != prepData.refs.keys():
841 unsupported = refs.keys() - prepData.refs.keys()
842 # Group unsupported refs by DatasetType for an informative
843 # but still concise error message.
844 byDatasetType = defaultdict(list)
845 for datasetId in unsupported:
846 ref = refs[datasetId]
847 byDatasetType[ref.datasetType].append(ref)
848 raise DatasetTypeNotSupportedError(
849 "DatasetType(s) not supported in ingest: "
850 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items())
851 )
852 self._finishIngest(prepData, transfer=transfer, record_validation_info=record_validation_info)
854 def transfer_from(
855 self,
856 source_datastore: Datastore,
857 refs: Collection[DatasetRef],
858 transfer: str = "auto",
859 artifact_existence: dict[ResourcePath, bool] | None = None,
860 dry_run: bool = False,
861 ) -> tuple[set[DatasetRef], set[DatasetRef]]:
862 """Transfer dataset artifacts from another datastore to this one.
864 Parameters
865 ----------
866 source_datastore : `Datastore`
867 The datastore from which to transfer artifacts. That datastore
868 must be compatible with this datastore receiving the artifacts.
869 refs : `~collections.abc.Collection` of `DatasetRef`
870 The datasets to transfer from the source datastore.
871 transfer : `str`, optional
872 How (and whether) the dataset should be added to the datastore.
873 Choices include "move", "copy",
874 "link", "symlink", "relsymlink", and "hardlink". "link" is a
875 special transfer mode that will first try to make a hardlink and
876 if that fails a symlink will be used instead. "relsymlink" creates
877 a relative symlink rather than use an absolute path.
878 Most datastores do not support all transfer modes.
879 "auto" (the default) is a special option that will let the
880 data store choose the most natural option for itself.
881 If the source location and transfer location are identical the
882 transfer mode will be ignored.
883 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`]
884 Optional mapping of datastore artifact to existence. Updated by
885 this method with details of all artifacts tested. Can be `None`
886 if the caller is not interested.
887 dry_run : `bool`, optional
888 Process the supplied source refs without updating the target
889 datastore.
891 Returns
892 -------
893 accepted : `set` [`DatasetRef`]
894 The datasets that were transferred.
895 rejected : `set` [`DatasetRef`]
896 The datasets that were rejected due to a constraints violation.
898 Raises
899 ------
900 TypeError
901 Raised if the two datastores are not compatible.
902 """
903 if type(self) is not type(source_datastore):
904 raise TypeError(
905 f"Datastore mismatch between this datastore ({type(self)}) and the "
906 f"source datastore ({type(source_datastore)})."
907 )
909 raise NotImplementedError(f"Datastore {type(self)} must implement a transfer_from method.")
911 def getManyURIs(
912 self,
913 refs: Iterable[DatasetRef],
914 predict: bool = False,
915 allow_missing: bool = False,
916 ) -> dict[DatasetRef, DatasetRefURIs]:
917 """Return URIs associated with many datasets.
919 Parameters
920 ----------
921 refs : iterable of `DatasetIdRef`
922 References to the required datasets.
923 predict : `bool`, optional
924 If `True`, allow URIs to be returned of datasets that have not
925 been written.
926 allow_missing : `bool`
927 If `False`, and ``predict`` is `False`, will raise if a
928 `DatasetRef` does not exist.
930 Returns
931 -------
932 URIs : `dict` of [`DatasetRef`, `DatasetRefUris`]
933 A dict of primary and component URIs, indexed by the passed-in
934 refs.
936 Raises
937 ------
938 FileNotFoundError
939 A URI has been requested for a dataset that does not exist and
940 guessing is not allowed.
942 Notes
943 -----
944 In file-based datastores, getManyURIs does not check that the file is
945 really there, it's assuming it is if datastore is aware of the file
946 then it actually exists.
947 """
948 uris: dict[DatasetRef, DatasetRefURIs] = {}
949 missing_refs = []
950 for ref in refs:
951 try:
952 uris[ref] = self.getURIs(ref, predict=predict)
953 except FileNotFoundError:
954 missing_refs.append(ref)
955 if missing_refs and not allow_missing:
956 num_missing = len(missing_refs)
957 raise FileNotFoundError(
958 f"Missing {num_missing} refs from datastore out of "
959 f"{num_missing + len(uris)} and predict=False."
960 )
961 return uris
963 @abstractmethod
964 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs:
965 """Return URIs associated with dataset.
967 Parameters
968 ----------
969 datasetRef : `DatasetRef`
970 Reference to the required dataset.
971 predict : `bool`, optional
972 If the datastore does not know about the dataset, controls whether
973 it should return a predicted URI or not.
975 Returns
976 -------
977 uris : `DatasetRefURIs`
978 The URI to the primary artifact associated with this dataset (if
979 the dataset was disassembled within the datastore this may be
980 `None`), and the URIs to any components associated with the dataset
981 artifact. (can be empty if there are no components).
982 """
983 raise NotImplementedError()
985 @abstractmethod
986 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
987 """URI to the Dataset.
989 Parameters
990 ----------
991 datasetRef : `DatasetRef`
992 Reference to the required Dataset.
993 predict : `bool`
994 If `True` attempt to predict the URI for a dataset if it does
995 not exist in datastore.
997 Returns
998 -------
999 uri : `str`
1000 URI string pointing to the Dataset within the datastore. If the
1001 Dataset does not exist in the datastore, the URI may be a guess.
1002 If the datastore does not have entities that relate well
1003 to the concept of a URI the returned URI string will be
1004 descriptive. The returned URI is not guaranteed to be obtainable.
1006 Raises
1007 ------
1008 FileNotFoundError
1009 A URI has been requested for a dataset that does not exist and
1010 guessing is not allowed.
1011 """
1012 raise NotImplementedError("Must be implemented by subclass")
1014 @abstractmethod
1015 def retrieveArtifacts(
1016 self,
1017 refs: Iterable[DatasetRef],
1018 destination: ResourcePath,
1019 transfer: str = "auto",
1020 preserve_path: bool = True,
1021 overwrite: bool = False,
1022 ) -> list[ResourcePath]:
1023 """Retrieve the artifacts associated with the supplied refs.
1025 Parameters
1026 ----------
1027 refs : iterable of `DatasetRef`
1028 The datasets for which artifacts are to be retrieved.
1029 A single ref can result in multiple artifacts. The refs must
1030 be resolved.
1031 destination : `lsst.resources.ResourcePath`
1032 Location to write the artifacts.
1033 transfer : `str`, optional
1034 Method to use to transfer the artifacts. Must be one of the options
1035 supported by `lsst.resources.ResourcePath.transfer_from()`.
1036 "move" is not allowed.
1037 preserve_path : `bool`, optional
1038 If `True` the full path of the artifact within the datastore
1039 is preserved. If `False` the final file component of the path
1040 is used.
1041 overwrite : `bool`, optional
1042 If `True` allow transfers to overwrite existing files at the
1043 destination.
1045 Returns
1046 -------
1047 targets : `list` of `lsst.resources.ResourcePath`
1048 URIs of file artifacts in destination location. Order is not
1049 preserved.
1051 Notes
1052 -----
1053 For non-file datastores the artifacts written to the destination
1054 may not match the representation inside the datastore. For example
1055 a hierarchichal data structure in a NoSQL database may well be stored
1056 as a JSON file.
1057 """
1058 raise NotImplementedError()
1060 @abstractmethod
1061 def remove(self, datasetRef: DatasetRef) -> None:
1062 """Indicate to the Datastore that a Dataset can be removed.
1064 Parameters
1065 ----------
1066 datasetRef : `DatasetRef`
1067 Reference to the required Dataset.
1069 Raises
1070 ------
1071 FileNotFoundError
1072 When Dataset does not exist.
1074 Notes
1075 -----
1076 Some Datastores may implement this method as a silent no-op to
1077 disable Dataset deletion through standard interfaces.
1078 """
1079 raise NotImplementedError("Must be implemented by subclass")
1081 @abstractmethod
1082 def forget(self, refs: Iterable[DatasetRef]) -> None:
1083 """Indicate to the Datastore that it should remove all records of the
1084 given datasets, without actually deleting them.
1086 Parameters
1087 ----------
1088 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1089 References to the datasets being forgotten.
1091 Notes
1092 -----
1093 Asking a datastore to forget a `DatasetRef` it does not hold should be
1094 a silent no-op, not an error.
1095 """
1096 raise NotImplementedError("Must be implemented by subclass")
1098 @abstractmethod
1099 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
1100 """Indicate to the Datastore that a Dataset can be moved to the trash.
1102 Parameters
1103 ----------
1104 ref : `DatasetRef` or iterable thereof
1105 Reference(s) to the required Dataset.
1106 ignore_errors : `bool`, optional
1107 Determine whether errors should be ignored. When multiple
1108 refs are being trashed there will be no per-ref check.
1110 Raises
1111 ------
1112 FileNotFoundError
1113 When Dataset does not exist and errors are not ignored. Only
1114 checked if a single ref is supplied (and not in a list).
1116 Notes
1117 -----
1118 Some Datastores may implement this method as a silent no-op to
1119 disable Dataset deletion through standard interfaces.
1120 """
1121 raise NotImplementedError("Must be implemented by subclass")
1123 @abstractmethod
1124 def emptyTrash(self, ignore_errors: bool = True) -> None:
1125 """Remove all datasets from the trash.
1127 Parameters
1128 ----------
1129 ignore_errors : `bool`, optional
1130 Determine whether errors should be ignored.
1132 Notes
1133 -----
1134 Some Datastores may implement this method as a silent no-op to
1135 disable Dataset deletion through standard interfaces.
1136 """
1137 raise NotImplementedError("Must be implemented by subclass")
1139 @abstractmethod
1140 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:
1141 """Transfer a dataset from another datastore to this datastore.
1143 Parameters
1144 ----------
1145 inputDatastore : `Datastore`
1146 The external `Datastore` from which to retrieve the Dataset.
1147 datasetRef : `DatasetRef`
1148 Reference to the required Dataset.
1149 """
1150 raise NotImplementedError("Must be implemented by subclass")
1152 def export(
1153 self,
1154 refs: Iterable[DatasetRef],
1155 *,
1156 directory: ResourcePathExpression | None = None,
1157 transfer: str | None = "auto",
1158 ) -> Iterable[FileDataset]:
1159 """Export datasets for transfer to another data repository.
1161 Parameters
1162 ----------
1163 refs : iterable of `DatasetRef`
1164 Dataset references to be exported.
1165 directory : `str`, optional
1166 Path to a directory that should contain files corresponding to
1167 output datasets. Ignored if ``transfer`` is explicitly `None`.
1168 transfer : `str`, optional
1169 Mode that should be used to move datasets out of the repository.
1170 Valid options are the same as those of the ``transfer`` argument
1171 to ``ingest``, and datastores may similarly signal that a transfer
1172 mode is not supported by raising `NotImplementedError`. If "auto"
1173 is given and no ``directory`` is specified, `None` will be
1174 implied.
1176 Returns
1177 -------
1178 dataset : iterable of `DatasetTransfer`
1179 Structs containing information about the exported datasets, in the
1180 same order as ``refs``.
1182 Raises
1183 ------
1184 NotImplementedError
1185 Raised if the given transfer mode is not supported.
1186 """
1187 raise NotImplementedError(f"Transfer mode {transfer} not supported.")
1189 @abstractmethod
1190 def validateConfiguration(
1191 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False
1192 ) -> None:
1193 """Validate some of the configuration for this datastore.
1195 Parameters
1196 ----------
1197 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
1198 Entities to test against this configuration. Can be differing
1199 types.
1200 logFailures : `bool`, optional
1201 If `True`, output a log message for every validation error
1202 detected.
1204 Raises
1205 ------
1206 DatastoreValidationError
1207 Raised if there is a validation problem with a configuration.
1209 Notes
1210 -----
1211 Which parts of the configuration are validated is at the discretion
1212 of each Datastore implementation.
1213 """
1214 raise NotImplementedError("Must be implemented by subclass")
1216 @abstractmethod
1217 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None:
1218 """Validate a specific look up key with supplied entity.
1220 Parameters
1221 ----------
1222 lookupKey : `LookupKey`
1223 Key to use to retrieve information from the datastore
1224 configuration.
1225 entity : `DatasetRef`, `DatasetType`, or `StorageClass`
1226 Entity to compare with configuration retrieved using the
1227 specified lookup key.
1229 Raises
1230 ------
1231 DatastoreValidationError
1232 Raised if there is a problem with the combination of entity
1233 and lookup key.
1235 Notes
1236 -----
1237 Bypasses the normal selection priorities by allowing a key that
1238 would normally not be selected to be validated.
1239 """
1240 raise NotImplementedError("Must be implemented by subclass")
1242 @abstractmethod
1243 def getLookupKeys(self) -> set[LookupKey]:
1244 """Return all the lookup keys relevant to this datastore.
1246 Returns
1247 -------
1248 keys : `set` of `LookupKey`
1249 The keys stored internally for looking up information based
1250 on `DatasetType` name or `StorageClass`.
1251 """
1252 raise NotImplementedError("Must be implemented by subclass")
1254 def needs_expanded_data_ids(
1255 self,
1256 transfer: str | None,
1257 entity: DatasetRef | DatasetType | StorageClass | None = None,
1258 ) -> bool:
1259 """Test whether this datastore needs expanded data IDs to ingest.
1261 Parameters
1262 ----------
1263 transfer : `str` or `None`
1264 Transfer mode for ingest.
1265 entity : `DatasetRef` or `DatasetType` or `StorageClass` or `None`, \
1266 optional
1267 Object representing what will be ingested. If not provided (or not
1268 specific enough), `True` may be returned even if expanded data
1269 IDs aren't necessary.
1271 Returns
1272 -------
1273 needed : `bool`
1274 If `True`, expanded data IDs may be needed. `False` only if
1275 expansion definitely isn't necessary.
1276 """
1277 return True
1279 @abstractmethod
1280 def import_records(
1281 self,
1282 data: Mapping[str, DatastoreRecordData],
1283 ) -> None:
1284 """Import datastore location and record data from an in-memory data
1285 structure.
1287 Parameters
1288 ----------
1289 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
1290 Datastore records indexed by datastore name. May contain data for
1291 other `Datastore` instances (generally because they are chained to
1292 this one), which should be ignored.
1294 Notes
1295 -----
1296 Implementations should generally not check that any external resources
1297 (e.g. files) referred to by these records actually exist, for
1298 performance reasons; we expect higher-level code to guarantee that they
1299 do.
1301 Implementations are responsible for calling
1302 `DatastoreRegistryBridge.insert` on all datasets in ``data.locations``
1303 where the key is in `names`, as well as loading any opaque table data.
1305 Implementations may assume that datasets are either fully present or
1306 not at all (single-component exports are not permitted).
1307 """
1308 raise NotImplementedError()
1310 @abstractmethod
1311 def export_records(
1312 self,
1313 refs: Iterable[DatasetIdRef],
1314 ) -> Mapping[str, DatastoreRecordData]:
1315 """Export datastore records and locations to an in-memory data
1316 structure.
1318 Parameters
1319 ----------
1320 refs : `~collections.abc.Iterable` [ `DatasetIdRef` ]
1321 Datasets to save. This may include datasets not known to this
1322 datastore, which should be ignored. May not include component
1323 datasets.
1325 Returns
1326 -------
1327 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
1328 Exported datastore records indexed by datastore name.
1329 """
1330 raise NotImplementedError()
1332 def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
1333 """Specify a method that can be used by datastore to retrieve
1334 registry-defined dataset type.
1336 Parameters
1337 ----------
1338 method : `~collections.abc.Callable` | `None`
1339 Method that takes a name of the dataset type and returns a
1340 corresponding `DatasetType` instance as defined in Registry. If
1341 dataset type name is not known to registry `None` is returned.
1343 Notes
1344 -----
1345 This method is only needed for a Datastore supporting a "trusted" mode
1346 when it does not have an access to datastore records and needs to
1347 guess dataset location based on its stored dataset type.
1348 """
1349 pass
1351 @abstractmethod
1352 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
1353 """Make definitions of the opaque tables used by this Datastore.
1355 Returns
1356 -------
1357 tables : `~collections.abc.Mapping` [ `str`, `.ddl.TableSpec` ]
1358 Mapping of opaque table names to their definitions. This can be an
1359 empty mapping if Datastore does not use opaque tables to keep
1360 datastore records.
1361 """
1362 raise NotImplementedError()
1365class NullDatastore(Datastore):
1366 """A datastore that implements the `Datastore` API but always fails when
1367 it accepts any request.
1369 Parameters
1370 ----------
1371 config : `Config` or `~lsst.resources.ResourcePathExpression` or `None`
1372 Ignored.
1373 bridgeManager : `DatastoreRegistryBridgeManager` or `None`
1374 Ignored.
1375 butlerRoot : `~lsst.resources.ResourcePathExpression` or `None`
1376 Ignored.
1377 """
1379 @classmethod
1380 def _create_from_config(
1381 cls,
1382 config: Config,
1383 bridgeManager: DatastoreRegistryBridgeManager,
1384 butlerRoot: ResourcePathExpression | None = None,
1385 ) -> NullDatastore:
1386 return NullDatastore(config, bridgeManager, butlerRoot)
1388 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore:
1389 return self
1391 @classmethod
1392 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
1393 # Nothing to do. This is not a real Datastore.
1394 pass
1396 def __init__(
1397 self,
1398 config: Config | ResourcePathExpression | None,
1399 bridgeManager: DatastoreRegistryBridgeManager | None,
1400 butlerRoot: ResourcePathExpression | None = None,
1401 ):
1402 # Name ourselves with the timestamp the datastore
1403 # was created.
1404 self.name = f"{type(self).__name__}@{time.time()}"
1405 _LOG.debug("Creating datastore %s", self.name)
1407 return
1409 def knows(self, ref: DatasetRef) -> bool:
1410 return False
1412 def exists(self, datasetRef: DatasetRef) -> bool:
1413 return False
1415 def get(
1416 self,
1417 datasetRef: DatasetRef,
1418 parameters: Mapping[str, Any] | None = None,
1419 storageClass: StorageClass | str | None = None,
1420 ) -> Any:
1421 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1423 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None:
1424 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1426 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]:
1427 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1429 def ingest(
1430 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True
1431 ) -> None:
1432 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1434 def transfer_from(
1435 self,
1436 source_datastore: Datastore,
1437 refs: Iterable[DatasetRef],
1438 transfer: str = "auto",
1439 artifact_existence: dict[ResourcePath, bool] | None = None,
1440 dry_run: bool = False,
1441 ) -> tuple[set[DatasetRef], set[DatasetRef]]:
1442 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1444 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs:
1445 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1447 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
1448 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
1450 def retrieveArtifacts(
1451 self,
1452 refs: Iterable[DatasetRef],
1453 destination: ResourcePath,
1454 transfer: str = "auto",
1455 preserve_path: bool = True,
1456 overwrite: bool = False,
1457 ) -> list[ResourcePath]:
1458 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1460 def remove(self, datasetRef: DatasetRef) -> None:
1461 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1463 def forget(self, refs: Iterable[DatasetRef]) -> None:
1464 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1466 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
1467 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1469 def emptyTrash(self, ignore_errors: bool = True) -> None:
1470 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1472 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:
1473 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1475 def export(
1476 self,
1477 refs: Iterable[DatasetRef],
1478 *,
1479 directory: ResourcePathExpression | None = None,
1480 transfer: str | None = "auto",
1481 ) -> Iterable[FileDataset]:
1482 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1484 def validateConfiguration(
1485 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False
1486 ) -> None:
1487 # No configuration so always validates.
1488 pass
1490 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None:
1491 pass
1493 def getLookupKeys(self) -> set[LookupKey]:
1494 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1496 def import_records(
1497 self,
1498 data: Mapping[str, DatastoreRecordData],
1499 ) -> None:
1500 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1502 def export_records(
1503 self,
1504 refs: Iterable[DatasetIdRef],
1505 ) -> Mapping[str, DatastoreRecordData]:
1506 raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
1508 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
1509 return {}