Coverage for python/lsst/daf/butler/core/storedFileInfo.py: 47%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

60 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("StoredFileInfo", "StoredDatastoreItemInfo") 

25 

26import inspect 

27from dataclasses import dataclass 

28from typing import TYPE_CHECKING, Any, Dict, Optional, Type 

29 

30from ._butlerUri import ButlerURI 

31from .formatter import Formatter, FormatterParameter 

32from .location import Location, LocationFactory 

33from .storageClass import StorageClass, StorageClassFactory 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from .datasets import DatasetRef 

37 

38# String to use when a Python None is encountered 

39NULLSTR = "__NULL_STRING__" 

40 

41 

42class StoredDatastoreItemInfo: 

43 """Internal information associated with a stored dataset in a `Datastore`. 

44 

45 This is an empty base class. Datastore implementations are expected to 

46 write their own subclasses. 

47 """ 

48 

49 __slots__ = () 

50 

51 def file_location(self, factory: LocationFactory) -> Location: 

52 """Return the location of artifact. 

53 

54 Parameters 

55 ---------- 

56 factory : `LocationFactory` 

57 Factory relevant to the datastore represented by this item. 

58 

59 Returns 

60 ------- 

61 location : `Location` 

62 The location of the item within this datastore. 

63 """ 

64 raise NotImplementedError("The base class does not know how to locate an item in a datastore.") 

65 

66 @classmethod 

67 def from_record(cls: Type[StoredDatastoreItemInfo], record: Dict[str, Any]) -> StoredDatastoreItemInfo: 

68 """Create instance from database record. 

69 

70 Parameters 

71 ---------- 

72 record : `dict` 

73 The record associated with this item. 

74 

75 Returns 

76 ------- 

77 info : instance of the relevant type. 

78 The newly-constructed item corresponding to the record. 

79 """ 

80 raise NotImplementedError() 

81 

82 

83@dataclass(frozen=True) 

84class StoredFileInfo(StoredDatastoreItemInfo): 

85 """Datastore-private metadata associated with a Datastore file.""" 

86 

87 __slots__ = {"formatter", "path", "storageClass", "component", "checksum", "file_size"} 

88 

89 storageClassFactory = StorageClassFactory() 

90 

91 def __init__( 

92 self, 

93 formatter: FormatterParameter, 

94 path: str, 

95 storageClass: StorageClass, 

96 component: Optional[str], 

97 checksum: Optional[str], 

98 file_size: int, 

99 ): 

100 

101 # Use these shenanigans to allow us to use a frozen dataclass 

102 object.__setattr__(self, "path", path) 

103 object.__setattr__(self, "storageClass", storageClass) 

104 object.__setattr__(self, "component", component) 

105 object.__setattr__(self, "checksum", checksum) 

106 object.__setattr__(self, "file_size", file_size) 

107 

108 if isinstance(formatter, str): 

109 # We trust that this string refers to a Formatter 

110 formatterStr = formatter 

111 elif isinstance(formatter, Formatter) or ( 

112 inspect.isclass(formatter) and issubclass(formatter, Formatter) 

113 ): 

114 formatterStr = formatter.name() 

115 else: 

116 raise TypeError(f"Supplied formatter '{formatter}' is not a Formatter") 

117 object.__setattr__(self, "formatter", formatterStr) 

118 

119 formatter: str 

120 """Fully-qualified name of Formatter. If a Formatter class or instance 

121 is given the name will be extracted.""" 

122 

123 path: str 

124 """Path to dataset within Datastore.""" 

125 

126 storageClass: StorageClass 

127 """StorageClass associated with Dataset.""" 

128 

129 component: Optional[str] 

130 """Component associated with this file. Can be None if the file does 

131 not refer to a component of a composite.""" 

132 

133 checksum: Optional[str] 

134 """Checksum of the serialized dataset.""" 

135 

136 file_size: int 

137 """Size of the serialized dataset in bytes.""" 

138 

139 def to_record(self, ref: DatasetRef) -> Dict[str, Any]: 

140 """Convert the supplied ref to a database record.""" 

141 component = ref.datasetType.component() 

142 if component is None and self.component is not None: 

143 component = self.component 

144 if component is None: 

145 # Use empty string since we want this to be part of the 

146 # primary key. 

147 component = NULLSTR 

148 

149 return dict( 

150 dataset_id=ref.id, 

151 formatter=self.formatter, 

152 path=self.path, 

153 storage_class=self.storageClass.name, 

154 component=component, 

155 checksum=self.checksum, 

156 file_size=self.file_size, 

157 ) 

158 

159 def file_location(self, factory: LocationFactory) -> Location: 

160 """Return the location of artifact. 

161 

162 Parameters 

163 ---------- 

164 factory : `LocationFactory` 

165 Factory relevant to the datastore represented by this item. 

166 

167 Returns 

168 ------- 

169 location : `Location` 

170 The location of the item within this datastore. 

171 """ 

172 uriInStore = ButlerURI(self.path, forceAbsolute=False) 

173 if uriInStore.isabs(): 

174 location = Location(None, uriInStore) 

175 else: 

176 location = factory.fromPath(uriInStore) 

177 return location 

178 

179 @classmethod 

180 def from_record(cls: Type[StoredFileInfo], record: Dict[str, Any]) -> StoredFileInfo: 

181 """Create instance from database record. 

182 

183 Parameters 

184 ---------- 

185 record : `dict` 

186 The record associated with this item. 

187 

188 Returns 

189 ------- 

190 info : `StoredFileInfo` 

191 The newly-constructed item corresponding to the record. 

192 """ 

193 # Convert name of StorageClass to instance 

194 storageClass = cls.storageClassFactory.getStorageClass(record["storage_class"]) 

195 component = record["component"] if (record["component"] and record["component"] != NULLSTR) else None 

196 

197 info = StoredFileInfo( 

198 formatter=record["formatter"], 

199 path=record["path"], 

200 storageClass=storageClass, 

201 component=component, 

202 checksum=record["checksum"], 

203 file_size=record["file_size"], 

204 ) 

205 return info