Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("FormatterTest", "DoNothingFormatter", "LenientYamlFormatter", "MetricsExampleFormatter") 

25 

26from typing import ( 

27 TYPE_CHECKING, 

28 Any, 

29 Mapping, 

30 Optional, 

31) 

32 

33import yaml 

34 

35from ..core import Formatter 

36from ..formatters.yaml import YamlFormatter 

37 

38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 from ..core import Location 

40 

41 

42class DoNothingFormatter(Formatter): 

43 """A test formatter that does not need to format anything and has 

44 parameters.""" 

45 

46 def read(self, component: Optional[str] = None) -> Any: 

47 raise NotImplementedError("Type does not support reading") 

48 

49 def write(self, inMemoryDataset: Any) -> str: 

50 raise NotImplementedError("Type does not support writing") 

51 

52 

53class FormatterTest(Formatter): 

54 """A test formatter that does not need to format anything.""" 

55 

56 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"}) 

57 

58 def read(self, component: Optional[str] = None) -> Any: 

59 raise NotImplementedError("Type does not support reading") 

60 

61 def write(self, inMemoryDataset: Any) -> str: 

62 raise NotImplementedError("Type does not support writing") 

63 

64 @staticmethod 

65 def validateWriteRecipes(recipes: Optional[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: 

66 if not recipes: 

67 return recipes 

68 for recipeName in recipes: 

69 if "mode" not in recipes[recipeName]: 

70 raise RuntimeError("'mode' is a required write recipe parameter") 

71 return recipes 

72 

73 

74class LenientYamlFormatter(YamlFormatter): 

75 """A test formatter that allows any file extension but always reads and 

76 writes YAML.""" 

77 extension = ".yaml" 

78 

79 @classmethod 

80 def validateExtension(cls, location: Location) -> None: 

81 return 

82 

83 

84class MetricsExampleFormatter(Formatter): 

85 """A specialist test formatter for metrics that supports components 

86 directly without assembler.""" 

87 

88 extension = ".yaml" 

89 """Always write YAML""" 

90 

91 def read(self, component=None): 

92 """Read data from a file. 

93 

94 Parameters 

95 ---------- 

96 component : `str`, optional 

97 Component to read from the file. Only used if the `StorageClass` 

98 for reading differed from the `StorageClass` used to write the 

99 file. 

100 

101 Returns 

102 ------- 

103 inMemoryDataset : `object` 

104 The requested data as a Python object. The type of object 

105 is controlled by the specific formatter. 

106 

107 Raises 

108 ------ 

109 ValueError 

110 Component requested but this file does not seem to be a concrete 

111 composite. 

112 KeyError 

113 Raised when parameters passed with fileDescriptor are not 

114 supported. 

115 """ 

116 

117 # This formatter can not read a subset from disk because it 

118 # uses yaml. 

119 path = self.fileDescriptor.location.path 

120 with open(path, "r") as fd: 

121 data = yaml.load(fd, Loader=yaml.SafeLoader) 

122 

123 # We can slice up front if required 

124 parameters = self.fileDescriptor.parameters 

125 if "data" in data and parameters and "slice" in parameters: 

126 data["data"] = data["data"][parameters["slice"]] 

127 

128 pytype = self.fileDescriptor.storageClass.pytype 

129 inMemoryDataset = pytype(**data) 

130 

131 if not component: 

132 return inMemoryDataset 

133 

134 if component == "summary": 

135 return inMemoryDataset.summary 

136 elif component == "output": 

137 return inMemoryDataset.output 

138 elif component == "data": 

139 return inMemoryDataset.data 

140 elif component == "counter": 

141 return len(inMemoryDataset.data) 

142 raise ValueError(f"Unsupported component: {component}") 

143 

144 def write(self, inMemoryDataset: Any) -> str: 

145 """Write a Dataset. 

146 

147 Parameters 

148 ---------- 

149 inMemoryDataset : `object` 

150 The Dataset to store. 

151 

152 Returns 

153 ------- 

154 path : `str` 

155 The path to where the Dataset was stored within the datastore. 

156 """ 

157 fileDescriptor = self.fileDescriptor 

158 

159 # Update the location with the formatter-preferred file extension 

160 fileDescriptor.location.updateExtension(self.extension) 

161 

162 with open(fileDescriptor.location.path, "w") as fd: 

163 yaml.dump(inMemoryDataset._asdict(), fd) 

164 return fileDescriptor.location.pathInStore 

165 

166 

167class MetricsExampleDataFormatter(Formatter): 

168 """A specialist test formatter for the data component of a MetricsExample. 

169 

170 This is needed if the MetricsExample is dissassembled and we want to 

171 support the read-only component. 

172 """ 

173 

174 unsupportedParameters = None 

175 """Let the assembler handle slice""" 

176 

177 extension = ".yaml" 

178 """Always write YAML""" 

179 

180 def read(self, component=None): 

181 """Read data from a file. 

182 

183 Parameters 

184 ---------- 

185 component : `str`, optional 

186 Component to read from the file. Only used if the `StorageClass` 

187 for reading differed from the `StorageClass` used to write the 

188 file. 

189 

190 Returns 

191 ------- 

192 inMemoryDataset : `object` 

193 The requested data as a Python object. The type of object 

194 is controlled by the specific formatter. 

195 

196 Raises 

197 ------ 

198 ValueError 

199 Component requested but this file does not seem to be a concrete 

200 composite. 

201 KeyError 

202 Raised when parameters passed with fileDescriptor are not 

203 supported. 

204 """ 

205 

206 # This formatter can not read a subset from disk because it 

207 # uses yaml. 

208 path = self.fileDescriptor.location.path 

209 with open(path, "r") as fd: 

210 data = yaml.load(fd, Loader=yaml.SafeLoader) 

211 

212 # We can slice up front if required 

213 parameters = self.fileDescriptor.parameters 

214 if parameters and "slice" in parameters: 

215 data = data[parameters["slice"]] 

216 

217 # This should be a native list 

218 inMemoryDataset = data 

219 

220 if not component: 

221 return inMemoryDataset 

222 

223 if component == "counter": 

224 return len(inMemoryDataset) 

225 raise ValueError(f"Unsupported component: {component}") 

226 

227 def write(self, inMemoryDataset: Any) -> str: 

228 """Write a Dataset. 

229 

230 Parameters 

231 ---------- 

232 inMemoryDataset : `object` 

233 The Dataset to store. 

234 

235 Returns 

236 ------- 

237 path : `str` 

238 The path to where the Dataset was stored within the datastore. 

239 """ 

240 fileDescriptor = self.fileDescriptor 

241 

242 # Update the location with the formatter-preferred file extension 

243 fileDescriptor.location.updateExtension(self.extension) 

244 

245 with open(fileDescriptor.location.path, "w") as fd: 

246 yaml.dump(inMemoryDataset, fd) 

247 return fileDescriptor.location.pathInStore