Coverage for python/lsst/daf/butler/tests/testFormatters.py: 30%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "FormatterTest", 

26 "DoNothingFormatter", 

27 "LenientYamlFormatter", 

28 "MetricsExampleFormatter", 

29 "MultipleExtensionsFormatter", 

30 "SingleExtensionFormatter", 

31) 

32 

33import json 

34from collections.abc import Mapping 

35from typing import TYPE_CHECKING, Any 

36 

37import yaml 

38 

39from ..core import Formatter 

40from ..formatters.yaml import YamlFormatter 

41 

42if TYPE_CHECKING: 

43 from ..core import Location 

44 

45 

46class DoNothingFormatter(Formatter): 

47 """A test formatter that does not need to format anything and has 

48 parameters.""" 

49 

50 def read(self, component: str | None = None) -> Any: 

51 raise NotImplementedError("Type does not support reading") 

52 

53 def write(self, inMemoryDataset: Any) -> None: 

54 raise NotImplementedError("Type does not support writing") 

55 

56 

57class FormatterTest(Formatter): 

58 """A test formatter that does not need to format anything.""" 

59 

60 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"}) 

61 

62 def read(self, component: str | None = None) -> Any: 

63 raise NotImplementedError("Type does not support reading") 

64 

65 def write(self, inMemoryDataset: Any) -> None: 

66 raise NotImplementedError("Type does not support writing") 

67 

68 @staticmethod 

69 def validateWriteRecipes(recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None: 

70 if not recipes: 

71 return recipes 

72 for recipeName in recipes: 

73 if "mode" not in recipes[recipeName]: 

74 raise RuntimeError("'mode' is a required write recipe parameter") 

75 return recipes 

76 

77 

78class SingleExtensionFormatter(DoNothingFormatter): 

79 """A do nothing formatter that has a single extension registered.""" 

80 

81 extension = ".fits" 

82 

83 

84class MultipleExtensionsFormatter(SingleExtensionFormatter): 

85 """A formatter that has multiple extensions registered.""" 

86 

87 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"}) 

88 

89 

90class LenientYamlFormatter(YamlFormatter): 

91 """A test formatter that allows any file extension but always reads and 

92 writes YAML.""" 

93 

94 extension = ".yaml" 

95 

96 @classmethod 

97 def validateExtension(cls, location: Location) -> None: 

98 return 

99 

100 

101class MetricsExampleFormatter(Formatter): 

102 """A specialist test formatter for metrics that supports components 

103 directly without assembler delegate.""" 

104 

105 supportedExtensions = frozenset({".yaml", ".json"}) 

106 

107 @property 

108 def extension(self) -> str: 

109 """Always write yaml by default.""" 

110 return ".yaml" 

111 

112 def read(self, component: str | None = None) -> Any: 

113 """Read data from a file. 

114 

115 Parameters 

116 ---------- 

117 component : `str`, optional 

118 Component to read from the file. Only used if the `StorageClass` 

119 for reading differed from the `StorageClass` used to write the 

120 file. 

121 

122 Returns 

123 ------- 

124 inMemoryDataset : `object` 

125 The requested data as a Python object. The type of object 

126 is controlled by the specific formatter. 

127 

128 Raises 

129 ------ 

130 ValueError 

131 Component requested but this file does not seem to be a concrete 

132 composite. 

133 KeyError 

134 Raised when parameters passed with fileDescriptor are not 

135 supported. 

136 """ 

137 

138 # This formatter can not read a subset from disk because it 

139 # uses yaml or json. 

140 path = self.fileDescriptor.location.path 

141 

142 with open(path, "r") as fd: 

143 if path.endswith(".yaml"): 

144 data = yaml.load(fd, Loader=yaml.SafeLoader) 

145 elif path.endswith(".json"): 

146 data = json.load(fd) 

147 else: 

148 raise RuntimeError(f"Unsupported file extension found in path '{path}'") 

149 

150 # We can slice up front if required 

151 parameters = self.fileDescriptor.parameters 

152 if "data" in data and parameters and "slice" in parameters: 

153 data["data"] = data["data"][parameters["slice"]] 

154 

155 pytype = self.fileDescriptor.storageClass.pytype 

156 inMemoryDataset = pytype(**data) 

157 

158 if not component: 

159 return inMemoryDataset 

160 

161 if component == "summary": 

162 return inMemoryDataset.summary 

163 elif component == "output": 

164 return inMemoryDataset.output 

165 elif component == "data": 

166 return inMemoryDataset.data 

167 elif component == "counter": 

168 return len(inMemoryDataset.data) 

169 raise ValueError(f"Unsupported component: {component}") 

170 

171 def write(self, inMemoryDataset: Any) -> None: 

172 """Write a Dataset. 

173 

174 Parameters 

175 ---------- 

176 inMemoryDataset : `object` 

177 The Dataset to store. 

178 

179 Returns 

180 ------- 

181 path : `str` 

182 The path to where the Dataset was stored within the datastore. 

183 """ 

184 fileDescriptor = self.fileDescriptor 

185 

186 # Update the location with the formatter-preferred file extension 

187 fileDescriptor.location.updateExtension(self.extension) 

188 

189 with open(fileDescriptor.location.path, "w") as fd: 

190 yaml.dump(inMemoryDataset._asdict(), fd) 

191 

192 

193class MetricsExampleDataFormatter(Formatter): 

194 """A specialist test formatter for the data component of a MetricsExample. 

195 

196 This is needed if the MetricsExample is disassembled and we want to 

197 support the derived component. 

198 """ 

199 

200 unsupportedParameters = None 

201 """Let the assembler delegate handle slice""" 

202 

203 extension = ".yaml" 

204 """Always write YAML""" 

205 

206 def read(self, component: str | None = None) -> Any: 

207 """Read data from a file. 

208 

209 Parameters 

210 ---------- 

211 component : `str`, optional 

212 Component to read from the file. Only used if the `StorageClass` 

213 for reading differed from the `StorageClass` used to write the 

214 file. 

215 

216 Returns 

217 ------- 

218 inMemoryDataset : `object` 

219 The requested data as a Python object. The type of object 

220 is controlled by the specific formatter. 

221 

222 Raises 

223 ------ 

224 ValueError 

225 Component requested but this file does not seem to be a concrete 

226 composite. 

227 KeyError 

228 Raised when parameters passed with fileDescriptor are not 

229 supported. 

230 """ 

231 

232 # This formatter can not read a subset from disk because it 

233 # uses yaml. 

234 path = self.fileDescriptor.location.path 

235 with open(path, "r") as fd: 

236 data = yaml.load(fd, Loader=yaml.SafeLoader) 

237 

238 # We can slice up front if required 

239 parameters = self.fileDescriptor.parameters 

240 if parameters and "slice" in parameters: 

241 data = data[parameters["slice"]] 

242 

243 # This should be a native list 

244 inMemoryDataset = data 

245 

246 if not component: 

247 return inMemoryDataset 

248 

249 if component == "counter": 

250 return len(inMemoryDataset) 

251 raise ValueError(f"Unsupported component: {component}") 

252 

253 def write(self, inMemoryDataset: Any) -> None: 

254 """Write a Dataset. 

255 

256 Parameters 

257 ---------- 

258 inMemoryDataset : `object` 

259 The Dataset to store. 

260 

261 Returns 

262 ------- 

263 path : `str` 

264 The path to where the Dataset was stored within the datastore. 

265 """ 

266 fileDescriptor = self.fileDescriptor 

267 

268 # Update the location with the formatter-preferred file extension 

269 fileDescriptor.location.updateExtension(self.extension) 

270 

271 with open(fileDescriptor.location.path, "w") as fd: 

272 yaml.dump(inMemoryDataset, fd)