Coverage for python/lsst/daf/butler/core/storedFileInfo.py: 46%

60 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 19:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("StoredFileInfo", "StoredDatastoreItemInfo") 

25 

26import inspect 

27from dataclasses import dataclass 

28from typing import Optional, Any, Dict, TYPE_CHECKING, Type 

29 

30from ._butlerUri import ButlerURI 

31from .location import Location, LocationFactory 

32from .formatter import Formatter, FormatterParameter 

33from .storageClass import StorageClass, StorageClassFactory 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from .datasets import DatasetRef 

37 

38# String to use when a Python None is encountered 

39NULLSTR = "__NULL_STRING__" 

40 

41 

42class StoredDatastoreItemInfo: 

43 """Internal information associated with a stored dataset in a `Datastore`. 

44 

45 This is an empty base class. Datastore implementations are expected to 

46 write their own subclasses. 

47 """ 

48 

49 __slots__ = () 

50 

51 def file_location(self, factory: LocationFactory) -> Location: 

52 """Return the location of artifact. 

53 

54 Parameters 

55 ---------- 

56 factory : `LocationFactory` 

57 Factory relevant to the datastore represented by this item. 

58 

59 Returns 

60 ------- 

61 location : `Location` 

62 The location of the item within this datastore. 

63 """ 

64 raise NotImplementedError("The base class does not know how to locate an item in a datastore.") 

65 

66 @classmethod 

67 def from_record(cls: Type[StoredDatastoreItemInfo], record: Dict[str, Any]) -> StoredDatastoreItemInfo: 

68 """Create instance from database record. 

69 

70 Parameters 

71 ---------- 

72 record : `dict` 

73 The record associated with this item. 

74 

75 Returns 

76 ------- 

77 info : instance of the relevant type. 

78 The newly-constructed item corresponding to the record. 

79 """ 

80 raise NotImplementedError() 

81 

82 

83@dataclass(frozen=True) 

84class StoredFileInfo(StoredDatastoreItemInfo): 

85 """Datastore-private metadata associated with a Datastore file.""" 

86 

87 __slots__ = {"formatter", "path", "storageClass", "component", 

88 "checksum", "file_size"} 

89 

90 storageClassFactory = StorageClassFactory() 

91 

92 def __init__(self, formatter: FormatterParameter, 

93 path: str, 

94 storageClass: StorageClass, 

95 component: Optional[str], 

96 checksum: Optional[str], 

97 file_size: int): 

98 

99 # Use these shenanigans to allow us to use a frozen dataclass 

100 object.__setattr__(self, "path", path) 

101 object.__setattr__(self, "storageClass", storageClass) 

102 object.__setattr__(self, "component", component) 

103 object.__setattr__(self, "checksum", checksum) 

104 object.__setattr__(self, "file_size", file_size) 

105 

106 if isinstance(formatter, str): 

107 # We trust that this string refers to a Formatter 

108 formatterStr = formatter 

109 elif isinstance(formatter, Formatter) or \ 

110 (inspect.isclass(formatter) and issubclass(formatter, Formatter)): 

111 formatterStr = formatter.name() 

112 else: 

113 raise TypeError(f"Supplied formatter '{formatter}' is not a Formatter") 

114 object.__setattr__(self, "formatter", formatterStr) 

115 

116 formatter: str 

117 """Fully-qualified name of Formatter. If a Formatter class or instance 

118 is given the name will be extracted.""" 

119 

120 path: str 

121 """Path to dataset within Datastore.""" 

122 

123 storageClass: StorageClass 

124 """StorageClass associated with Dataset.""" 

125 

126 component: Optional[str] 

127 """Component associated with this file. Can be None if the file does 

128 not refer to a component of a composite.""" 

129 

130 checksum: Optional[str] 

131 """Checksum of the serialized dataset.""" 

132 

133 file_size: int 

134 """Size of the serialized dataset in bytes.""" 

135 

136 def to_record(self, ref: DatasetRef) -> Dict[str, Any]: 

137 """Convert the supplied ref to a database record.""" 

138 component = ref.datasetType.component() 

139 if component is None and self.component is not None: 

140 component = self.component 

141 if component is None: 

142 # Use empty string since we want this to be part of the 

143 # primary key. 

144 component = NULLSTR 

145 

146 return dict(dataset_id=ref.id, formatter=self.formatter, path=self.path, 

147 storage_class=self.storageClass.name, component=component, 

148 checksum=self.checksum, file_size=self.file_size) 

149 

150 def file_location(self, factory: LocationFactory) -> Location: 

151 """Return the location of artifact. 

152 

153 Parameters 

154 ---------- 

155 factory : `LocationFactory` 

156 Factory relevant to the datastore represented by this item. 

157 

158 Returns 

159 ------- 

160 location : `Location` 

161 The location of the item within this datastore. 

162 """ 

163 uriInStore = ButlerURI(self.path, forceAbsolute=False) 

164 if uriInStore.isabs(): 

165 location = Location(None, uriInStore) 

166 else: 

167 location = factory.fromPath(uriInStore) 

168 return location 

169 

170 @classmethod 

171 def from_record(cls: Type[StoredFileInfo], record: Dict[str, Any]) -> StoredFileInfo: 

172 """Create instance from database record. 

173 

174 Parameters 

175 ---------- 

176 record : `dict` 

177 The record associated with this item. 

178 

179 Returns 

180 ------- 

181 info : `StoredFileInfo` 

182 The newly-constructed item corresponding to the record. 

183 """ 

184 # Convert name of StorageClass to instance 

185 storageClass = cls.storageClassFactory.getStorageClass(record["storage_class"]) 

186 component = record["component"] if (record["component"] 

187 and record["component"] != NULLSTR) else None 

188 

189 info = StoredFileInfo(formatter=record["formatter"], 

190 path=record["path"], 

191 storageClass=storageClass, 

192 component=component, 

193 checksum=record["checksum"], 

194 file_size=record["file_size"]) 

195 return info