Coverage for python/lsst/daf/butler/datastore/generic_base.py: 32%

38 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Generic datastore code useful for most datastores.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("GenericBaseDatastore",) 

33 

34import logging 

35from collections.abc import Mapping 

36from typing import TYPE_CHECKING, Any, Generic, TypeVar 

37 

38from .._exceptions import DatasetTypeNotSupportedError 

39from ..datastore._datastore import Datastore 

40from .stored_file_info import StoredDatastoreItemInfo 

41 

42if TYPE_CHECKING: 

43 from .._dataset_ref import DatasetRef 

44 from .._storage_class import StorageClass 

45 

46log = logging.getLogger(__name__) 

47 

48_InfoType = TypeVar("_InfoType", bound=StoredDatastoreItemInfo) 

49 

50 

51class GenericBaseDatastore(Datastore, Generic[_InfoType]): 

52 """Methods useful for most implementations of a `Datastore`. 

53 

54 Should always be sub-classed since key abstract methods are missing. 

55 """ 

56 

57 def _post_process_get( 

58 self, 

59 inMemoryDataset: object, 

60 readStorageClass: StorageClass, 

61 assemblerParams: Mapping[str, Any] | None = None, 

62 isComponent: bool = False, 

63 ) -> object: 

64 """Given the Python object read from the datastore, manipulate 

65 it based on the supplied parameters and ensure the Python 

66 type is correct. 

67 

68 Parameters 

69 ---------- 

70 inMemoryDataset : `object` 

71 Dataset to check. 

72 readStorageClass: `StorageClass` 

73 The `StorageClass` used to obtain the assembler and to 

74 check the python type. 

75 assemblerParams : `dict`, optional 

76 Parameters to pass to the assembler. Can be `None`. 

77 isComponent : `bool`, optional 

78 If this is a component, allow the inMemoryDataset to be `None`. 

79 

80 Returns 

81 ------- 

82 dataset : `object` 

83 In-memory dataset, potentially converted to expected type. 

84 """ 

85 # Process any left over parameters 

86 if assemblerParams: 

87 inMemoryDataset = readStorageClass.delegate().handleParameters(inMemoryDataset, assemblerParams) 

88 

89 # Validate the returned data type matches the expected data type 

90 pytype = readStorageClass.pytype 

91 

92 allowedTypes = [] 

93 if pytype: 

94 allowedTypes.append(pytype) 

95 

96 # Special case components to allow them to be None 

97 if isComponent: 

98 allowedTypes.append(type(None)) 

99 

100 if allowedTypes and not isinstance(inMemoryDataset, tuple(allowedTypes)): 

101 inMemoryDataset = readStorageClass.coerce_type(inMemoryDataset) 

102 

103 return inMemoryDataset 

104 

105 def _validate_put_parameters(self, inMemoryDataset: object, ref: DatasetRef) -> None: 

106 """Validate the supplied arguments for put. 

107 

108 Parameters 

109 ---------- 

110 inMemoryDataset : `object` 

111 The dataset to store. 

112 ref : `DatasetRef` 

113 Reference to the associated Dataset. 

114 """ 

115 storageClass = ref.datasetType.storageClass 

116 

117 # Sanity check 

118 if not isinstance(inMemoryDataset, storageClass.pytype): 

119 raise TypeError( 

120 f"Inconsistency between supplied object ({type(inMemoryDataset)}) " 

121 f"and storage class type ({storageClass.pytype})" 

122 ) 

123 

124 # Confirm that we can accept this dataset 

125 if not self.constraints.isAcceptable(ref): 

126 # Raise rather than use boolean return value. 

127 raise DatasetTypeNotSupportedError( 

128 f"Dataset {ref} has been rejected by this datastore via configuration." 

129 ) 

130 

131 return 

132 

133 def remove(self, ref: DatasetRef) -> None: 

134 """Indicate to the Datastore that a dataset can be removed. 

135 

136 .. warning:: 

137 

138 This method deletes the artifact associated with this 

139 dataset and can not be reversed. 

140 

141 Parameters 

142 ---------- 

143 ref : `DatasetRef` 

144 Reference to the required Dataset. 

145 

146 Raises 

147 ------ 

148 FileNotFoundError 

149 Attempt to remove a dataset that does not exist. 

150 

151 Notes 

152 ----- 

153 This method is used for immediate removal of a dataset and is 

154 generally reserved for internal testing of datastore APIs. 

155 It is implemented by calling `trash()` and then immediately calling 

156 `emptyTrash()`. This call is meant to be immediate so errors 

157 encountered during removal are not ignored. 

158 """ 

159 self.trash(ref, ignore_errors=False) 

160 self.emptyTrash(ignore_errors=False) 

161 

162 def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None: 

163 """Retrieve a dataset from an input `Datastore`, 

164 and store the result in this `Datastore`. 

165 

166 Parameters 

167 ---------- 

168 inputDatastore : `Datastore` 

169 The external `Datastore` from which to retreive the Dataset. 

170 ref : `DatasetRef` 

171 Reference to the required dataset in the input data store. 

172 """ 

173 assert inputDatastore is not self # unless we want it for renames? 

174 inMemoryDataset = inputDatastore.get(ref) 

175 return self.put(inMemoryDataset, ref)