Coverage for python/lsst/daf/butler/datastores/genericDatastore.py : 87%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Generic datastore code useful for most datastores."""
24__all__ = ("GenericBaseDatastore", )
26import logging
27from abc import abstractmethod
29from lsst.daf.butler import Datastore, DatasetTypeNotSupportedError
31log = logging.getLogger(__name__)
34class GenericBaseDatastore(Datastore):
35 """Methods useful for most implementations of a `Datastore`.
37 Should always be sub-classed since key abstract methods are missing.
38 """
40 @abstractmethod
41 def addStoredItemInfo(self, refs, infos):
42 """Record internal storage information associated with one or more
43 datasets.
45 Parameters
46 ----------
47 refs : sequence of `DatasetRef`
48 The datasets that have been stored.
49 infos : sequence of `StoredDatastoreItemInfo`
50 Metadata associated with the stored datasets.
51 """
52 raise NotImplementedError()
54 @abstractmethod
55 def getStoredItemInfo(self, ref):
56 """Retrieve information associated with file stored in this
57 `Datastore`.
59 Parameters
60 ----------
61 ref : `DatasetRef`
62 The dataset that is to be queried.
64 Returns
65 -------
66 info : `StoredFilenfo`
67 Stored information about this file and its formatter.
69 Raises
70 ------
71 KeyError
72 Dataset with that id can not be found.
73 """
74 raise NotImplementedError()
76 @abstractmethod
77 def removeStoredItemInfo(self, ref):
78 """Remove information about the file associated with this dataset.
80 Parameters
81 ----------
82 ref : `DatasetRef`
83 The dataset that has been removed.
84 """
85 raise NotImplementedError()
87 def _register_datasets(self, refsAndInfos):
88 """Update registry to indicate that one or more datasets have been
89 stored.
91 Parameters
92 ----------
93 refsAndInfos : sequence `tuple` [`DatasetRef`, `StoredDatasetItemInfo`]
94 Datasets to register and the internal datastore metadata associated
95 with them.
96 """
97 expandedRefs = []
98 expandedItemInfos = []
100 for ref, itemInfo in refsAndInfos:
101 # Need the main dataset and the components
102 expandedRefs.extend(ref.flatten([ref]))
104 # Need one for the main ref and then one for each component
105 expandedItemInfos.extend([itemInfo] * (len(ref.components) + 1))
107 self.registry.insertDatasetLocations(self.name, expandedRefs)
108 self.addStoredItemInfo(expandedRefs, expandedItemInfos)
110 def _move_to_trash_in_registry(self, ref):
111 """Tell registry that this dataset and associated components
112 are to be trashed.
114 Parameters
115 ----------
116 ref : `DatasetRef`
117 Dataset to mark for removal from registry.
119 Notes
120 -----
121 Dataset is not removed from internal stored item info table.
122 """
124 # Note that a ref can point to component dataset refs that
125 # have been deleted already from registry but are still in
126 # the python object. moveDatasetLocationToTrash will deal with that.
127 self.registry.moveDatasetLocationToTrash(self.name, list(ref.flatten([ref])))
129 def _post_process_get(self, inMemoryDataset, readStorageClass, assemblerParams=None,
130 isComponent=False):
131 """Given the Python object read from the datastore, manipulate
132 it based on the supplied parameters and ensure the Python
133 type is correct.
135 Parameters
136 ----------
137 inMemoryDataset : `object`
138 Dataset to check.
139 readStorageClass: `StorageClass`
140 The `StorageClass` used to obtain the assembler and to
141 check the python type.
142 assemblerParams : `dict`, optional
143 Parameters to pass to the assembler. Can be `None`.
144 isComponent : `bool`, optional
145 If this is a component, allow the inMemoryDataset to be `None`.
146 """
147 # Process any left over parameters
148 if assemblerParams:
149 inMemoryDataset = readStorageClass.assembler().handleParameters(inMemoryDataset, assemblerParams)
151 # Validate the returned data type matches the expected data type
152 pytype = readStorageClass.pytype
154 allowedTypes = []
155 if pytype: 155 ↛ 159line 155 didn't jump to line 159, because the condition on line 155 was never false
156 allowedTypes.append(pytype)
158 # Special case components to allow them to be None
159 if isComponent:
160 allowedTypes.append(type(None))
162 if allowedTypes and not isinstance(inMemoryDataset, tuple(allowedTypes)): 162 ↛ 163line 162 didn't jump to line 163, because the condition on line 162 was never true
163 raise TypeError("Got Python type {} from datastore but expected {}".format(type(inMemoryDataset),
164 pytype))
166 return inMemoryDataset
168 def _validate_put_parameters(self, inMemoryDataset, ref):
169 """Validate the supplied arguments for put.
171 Parameters
172 ----------
173 inMemoryDataset : `object`
174 The dataset to store.
175 ref : `DatasetRef`
176 Reference to the associated Dataset.
177 """
178 storageClass = ref.datasetType.storageClass
180 # Sanity check
181 if not isinstance(inMemoryDataset, storageClass.pytype): 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true
182 raise TypeError("Inconsistency between supplied object ({}) "
183 "and storage class type ({})".format(type(inMemoryDataset),
184 storageClass.pytype))
186 # Confirm that we can accept this dataset
187 if not self.constraints.isAcceptable(ref):
188 # Raise rather than use boolean return value.
189 raise DatasetTypeNotSupportedError(f"Dataset {ref} has been rejected by this datastore via"
190 " configuration.")
192 return
194 def remove(self, ref):
195 """Indicate to the Datastore that a dataset can be removed.
197 .. warning::
199 This method deletes the artifact associated with this
200 dataset and can not be reversed.
202 Parameters
203 ----------
204 ref : `DatasetRef`
205 Reference to the required Dataset.
207 Raises
208 ------
209 FileNotFoundError
210 Attempt to remove a dataset that does not exist.
212 Notes
213 -----
214 This method is used for immediate removal of a dataset and is
215 generally reserved for internal testing of datastore APIs.
216 It is implemented by calling `trash()` and then immediately calling
217 `emptyTrash()`. This call is meant to be immediate so errors
218 encountered during removal are not ignored.
219 """
220 self.trash(ref, ignore_errors=False)
221 self.emptyTrash(ignore_errors=False)
223 def transfer(self, inputDatastore, ref):
224 """Retrieve a dataset from an input `Datastore`,
225 and store the result in this `Datastore`.
227 Parameters
228 ----------
229 inputDatastore : `Datastore`
230 The external `Datastore` from which to retreive the Dataset.
231 ref : `DatasetRef`
232 Reference to the required dataset in the input data store.
234 """
235 assert inputDatastore is not self # unless we want it for renames?
236 inMemoryDataset = inputDatastore.get(ref)
237 return self.put(inMemoryDataset, ref)