Coverage for python/lsst/daf/butler/tests/testFormatters.py: 30%

89 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-28 10:10 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "FormatterTest", 

26 "DoNothingFormatter", 

27 "LenientYamlFormatter", 

28 "MetricsExampleFormatter", 

29 "MultipleExtensionsFormatter", 

30 "SingleExtensionFormatter", 

31) 

32 

33import json 

34from collections.abc import Mapping 

35from typing import TYPE_CHECKING, Any 

36 

37import yaml 

38 

39from ..core import Formatter 

40from ..formatters.yaml import YamlFormatter 

41 

42if TYPE_CHECKING: 

43 from ..core import Location 

44 

45 

46class DoNothingFormatter(Formatter): 

47 """A test formatter that does not need to format anything and has 

48 parameters. 

49 """ 

50 

51 def read(self, component: str | None = None) -> Any: 

52 raise NotImplementedError("Type does not support reading") 

53 

54 def write(self, inMemoryDataset: Any) -> None: 

55 raise NotImplementedError("Type does not support writing") 

56 

57 

58class FormatterTest(Formatter): 

59 """A test formatter that does not need to format anything.""" 

60 

61 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"}) 

62 

63 def read(self, component: str | None = None) -> Any: 

64 raise NotImplementedError("Type does not support reading") 

65 

66 def write(self, inMemoryDataset: Any) -> None: 

67 raise NotImplementedError("Type does not support writing") 

68 

69 @staticmethod 

70 def validateWriteRecipes(recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None: 

71 if not recipes: 

72 return recipes 

73 for recipeName in recipes: 

74 if "mode" not in recipes[recipeName]: 

75 raise RuntimeError("'mode' is a required write recipe parameter") 

76 return recipes 

77 

78 

79class SingleExtensionFormatter(DoNothingFormatter): 

80 """A do nothing formatter that has a single extension registered.""" 

81 

82 extension = ".fits" 

83 

84 

85class MultipleExtensionsFormatter(SingleExtensionFormatter): 

86 """A formatter that has multiple extensions registered.""" 

87 

88 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"}) 

89 

90 

91class LenientYamlFormatter(YamlFormatter): 

92 """A test formatter that allows any file extension but always reads and 

93 writes YAML. 

94 """ 

95 

96 extension = ".yaml" 

97 

98 @classmethod 

99 def validateExtension(cls, location: Location) -> None: 

100 return 

101 

102 

103class MetricsExampleFormatter(Formatter): 

104 """A specialist test formatter for metrics that supports components 

105 directly without assembler delegate. 

106 """ 

107 

108 supportedExtensions = frozenset({".yaml", ".json"}) 

109 

110 @property 

111 def extension(self) -> str: 

112 """Always write yaml by default.""" 

113 return ".yaml" 

114 

115 def read(self, component: str | None = None) -> Any: 

116 """Read data from a file. 

117 

118 Parameters 

119 ---------- 

120 component : `str`, optional 

121 Component to read from the file. Only used if the `StorageClass` 

122 for reading differed from the `StorageClass` used to write the 

123 file. 

124 

125 Returns 

126 ------- 

127 inMemoryDataset : `object` 

128 The requested data as a Python object. The type of object 

129 is controlled by the specific formatter. 

130 

131 Raises 

132 ------ 

133 ValueError 

134 Component requested but this file does not seem to be a concrete 

135 composite. 

136 KeyError 

137 Raised when parameters passed with fileDescriptor are not 

138 supported. 

139 """ 

140 # This formatter can not read a subset from disk because it 

141 # uses yaml or json. 

142 path = self.fileDescriptor.location.path 

143 

144 with open(path) as fd: 

145 if path.endswith(".yaml"): 

146 data = yaml.load(fd, Loader=yaml.SafeLoader) 

147 elif path.endswith(".json"): 

148 data = json.load(fd) 

149 else: 

150 raise RuntimeError(f"Unsupported file extension found in path '{path}'") 

151 

152 # We can slice up front if required 

153 parameters = self.fileDescriptor.parameters 

154 if "data" in data and parameters and "slice" in parameters: 

155 data["data"] = data["data"][parameters["slice"]] 

156 

157 pytype = self.fileDescriptor.storageClass.pytype 

158 inMemoryDataset = pytype(**data) 

159 

160 if not component: 

161 return inMemoryDataset 

162 

163 if component == "summary": 

164 return inMemoryDataset.summary 

165 elif component == "output": 

166 return inMemoryDataset.output 

167 elif component == "data": 

168 return inMemoryDataset.data 

169 elif component == "counter": 

170 return len(inMemoryDataset.data) 

171 raise ValueError(f"Unsupported component: {component}") 

172 

173 def write(self, inMemoryDataset: Any) -> None: 

174 """Write a Dataset. 

175 

176 Parameters 

177 ---------- 

178 inMemoryDataset : `object` 

179 The Dataset to store. 

180 

181 Returns 

182 ------- 

183 path : `str` 

184 The path to where the Dataset was stored within the datastore. 

185 """ 

186 fileDescriptor = self.fileDescriptor 

187 

188 # Update the location with the formatter-preferred file extension 

189 fileDescriptor.location.updateExtension(self.extension) 

190 

191 with open(fileDescriptor.location.path, "w") as fd: 

192 yaml.dump(inMemoryDataset._asdict(), fd) 

193 

194 

195class MetricsExampleDataFormatter(Formatter): 

196 """A specialist test formatter for the data component of a MetricsExample. 

197 

198 This is needed if the MetricsExample is disassembled and we want to 

199 support the derived component. 

200 """ 

201 

202 unsupportedParameters = None 

203 """Let the assembler delegate handle slice""" 

204 

205 extension = ".yaml" 

206 """Always write YAML""" 

207 

208 def read(self, component: str | None = None) -> Any: 

209 """Read data from a file. 

210 

211 Parameters 

212 ---------- 

213 component : `str`, optional 

214 Component to read from the file. Only used if the `StorageClass` 

215 for reading differed from the `StorageClass` used to write the 

216 file. 

217 

218 Returns 

219 ------- 

220 inMemoryDataset : `object` 

221 The requested data as a Python object. The type of object 

222 is controlled by the specific formatter. 

223 

224 Raises 

225 ------ 

226 ValueError 

227 Component requested but this file does not seem to be a concrete 

228 composite. 

229 KeyError 

230 Raised when parameters passed with fileDescriptor are not 

231 supported. 

232 """ 

233 # This formatter can not read a subset from disk because it 

234 # uses yaml. 

235 path = self.fileDescriptor.location.path 

236 with open(path) as fd: 

237 data = yaml.load(fd, Loader=yaml.SafeLoader) 

238 

239 # We can slice up front if required 

240 parameters = self.fileDescriptor.parameters 

241 if parameters and "slice" in parameters: 

242 data = data[parameters["slice"]] 

243 

244 # This should be a native list 

245 inMemoryDataset = data 

246 

247 if not component: 

248 return inMemoryDataset 

249 

250 if component == "counter": 

251 return len(inMemoryDataset) 

252 raise ValueError(f"Unsupported component: {component}") 

253 

254 def write(self, inMemoryDataset: Any) -> None: 

255 """Write a Dataset. 

256 

257 Parameters 

258 ---------- 

259 inMemoryDataset : `object` 

260 The Dataset to store. 

261 

262 Returns 

263 ------- 

264 path : `str` 

265 The path to where the Dataset was stored within the datastore. 

266 """ 

267 fileDescriptor = self.fileDescriptor 

268 

269 # Update the location with the formatter-preferred file extension 

270 fileDescriptor.location.updateExtension(self.extension) 

271 

272 with open(fileDescriptor.location.path, "w") as fd: 

273 yaml.dump(inMemoryDataset, fd)