Coverage for python/lsst/daf/butler/datastore/generic_base.py: 32%
38 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-05 11:07 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-05 11:07 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Generic datastore code useful for most datastores."""
30from __future__ import annotations
32__all__ = ("GenericBaseDatastore",)
34import logging
35from collections.abc import Mapping
36from typing import TYPE_CHECKING, Any, Generic, TypeVar
38from .._exceptions import DatasetTypeNotSupportedError
39from ..datastore._datastore import Datastore
40from .stored_file_info import StoredDatastoreItemInfo
42if TYPE_CHECKING:
43 from .._dataset_ref import DatasetRef
44 from .._storage_class import StorageClass
46log = logging.getLogger(__name__)
48_InfoType = TypeVar("_InfoType", bound=StoredDatastoreItemInfo)
51class GenericBaseDatastore(Datastore, Generic[_InfoType]):
52 """Methods useful for most implementations of a `Datastore`.
54 Should always be sub-classed since key abstract methods are missing.
55 """
57 def _post_process_get(
58 self,
59 inMemoryDataset: object,
60 readStorageClass: StorageClass,
61 assemblerParams: Mapping[str, Any] | None = None,
62 isComponent: bool = False,
63 ) -> object:
64 """Given the Python object read from the datastore, manipulate
65 it based on the supplied parameters and ensure the Python
66 type is correct.
68 Parameters
69 ----------
70 inMemoryDataset : `object`
71 Dataset to check.
72 readStorageClass: `StorageClass`
73 The `StorageClass` used to obtain the assembler and to
74 check the python type.
75 assemblerParams : `dict`, optional
76 Parameters to pass to the assembler. Can be `None`.
77 isComponent : `bool`, optional
78 If this is a component, allow the inMemoryDataset to be `None`.
80 Returns
81 -------
82 dataset : `object`
83 In-memory dataset, potentially converted to expected type.
84 """
85 # Process any left over parameters
86 if assemblerParams:
87 inMemoryDataset = readStorageClass.delegate().handleParameters(inMemoryDataset, assemblerParams)
89 # Validate the returned data type matches the expected data type
90 pytype = readStorageClass.pytype
92 allowedTypes = []
93 if pytype:
94 allowedTypes.append(pytype)
96 # Special case components to allow them to be None
97 if isComponent:
98 allowedTypes.append(type(None))
100 if allowedTypes and not isinstance(inMemoryDataset, tuple(allowedTypes)):
101 inMemoryDataset = readStorageClass.coerce_type(inMemoryDataset)
103 return inMemoryDataset
105 def _validate_put_parameters(self, inMemoryDataset: object, ref: DatasetRef) -> None:
106 """Validate the supplied arguments for put.
108 Parameters
109 ----------
110 inMemoryDataset : `object`
111 The dataset to store.
112 ref : `DatasetRef`
113 Reference to the associated Dataset.
114 """
115 storageClass = ref.datasetType.storageClass
117 # Sanity check
118 if not isinstance(inMemoryDataset, storageClass.pytype):
119 raise TypeError(
120 f"Inconsistency between supplied object ({type(inMemoryDataset)}) "
121 f"and storage class type ({storageClass.pytype})"
122 )
124 # Confirm that we can accept this dataset
125 if not self.constraints.isAcceptable(ref):
126 # Raise rather than use boolean return value.
127 raise DatasetTypeNotSupportedError(
128 f"Dataset {ref} has been rejected by this datastore via configuration."
129 )
131 return
133 def remove(self, ref: DatasetRef) -> None:
134 """Indicate to the Datastore that a dataset can be removed.
136 .. warning::
138 This method deletes the artifact associated with this
139 dataset and can not be reversed.
141 Parameters
142 ----------
143 ref : `DatasetRef`
144 Reference to the required Dataset.
146 Raises
147 ------
148 FileNotFoundError
149 Attempt to remove a dataset that does not exist.
151 Notes
152 -----
153 This method is used for immediate removal of a dataset and is
154 generally reserved for internal testing of datastore APIs.
155 It is implemented by calling `trash()` and then immediately calling
156 `emptyTrash()`. This call is meant to be immediate so errors
157 encountered during removal are not ignored.
158 """
159 self.trash(ref, ignore_errors=False)
160 self.emptyTrash(ignore_errors=False)
162 def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None:
163 """Retrieve a dataset from an input `Datastore`,
164 and store the result in this `Datastore`.
166 Parameters
167 ----------
168 inputDatastore : `Datastore`
169 The external `Datastore` from which to retreive the Dataset.
170 ref : `DatasetRef`
171 Reference to the required dataset in the input data store.
172 """
173 assert inputDatastore is not self # unless we want it for renames?
174 inMemoryDataset = inputDatastore.get(ref)
175 return self.put(inMemoryDataset, ref)