Coverage for python/lsst/daf/butler/tests/testFormatters.py: 34%

89 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-30 09:54 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "FormatterTest", 

32 "DoNothingFormatter", 

33 "LenientYamlFormatter", 

34 "MetricsExampleFormatter", 

35 "MultipleExtensionsFormatter", 

36 "SingleExtensionFormatter", 

37) 

38 

39import json 

40from collections.abc import Mapping 

41from typing import TYPE_CHECKING, Any 

42 

43import yaml 

44 

45from .._formatter import Formatter 

46from ..formatters.yaml import YamlFormatter 

47 

48if TYPE_CHECKING: 

49 from .._location import Location 

50 

51 

52class DoNothingFormatter(Formatter): 

53 """A test formatter that does not need to format anything and has 

54 parameters. 

55 """ 

56 

57 def read(self, component: str | None = None) -> Any: 

58 raise NotImplementedError("Type does not support reading") 

59 

60 def write(self, inMemoryDataset: Any) -> None: 

61 raise NotImplementedError("Type does not support writing") 

62 

63 

64class FormatterTest(Formatter): 

65 """A test formatter that does not need to format anything.""" 

66 

67 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"}) 

68 

69 def read(self, component: str | None = None) -> Any: 

70 raise NotImplementedError("Type does not support reading") 

71 

72 def write(self, inMemoryDataset: Any) -> None: 

73 raise NotImplementedError("Type does not support writing") 

74 

75 @staticmethod 

76 def validateWriteRecipes(recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None: 

77 if not recipes: 

78 return recipes 

79 for recipeName in recipes: 

80 if "mode" not in recipes[recipeName]: 

81 raise RuntimeError("'mode' is a required write recipe parameter") 

82 return recipes 

83 

84 

85class SingleExtensionFormatter(DoNothingFormatter): 

86 """A do nothing formatter that has a single extension registered.""" 

87 

88 extension = ".fits" 

89 

90 

91class MultipleExtensionsFormatter(SingleExtensionFormatter): 

92 """A formatter that has multiple extensions registered.""" 

93 

94 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"}) 

95 

96 

97class LenientYamlFormatter(YamlFormatter): 

98 """A test formatter that allows any file extension but always reads and 

99 writes YAML. 

100 """ 

101 

102 extension = ".yaml" 

103 

104 @classmethod 

105 def validateExtension(cls, location: Location) -> None: 

106 return 

107 

108 

109class MetricsExampleFormatter(Formatter): 

110 """A specialist test formatter for metrics that supports components 

111 directly without assembler delegate. 

112 """ 

113 

114 supportedExtensions = frozenset({".yaml", ".json"}) 

115 

116 @property 

117 def extension(self) -> str: 

118 """Always write yaml by default.""" 

119 return ".yaml" 

120 

121 def read(self, component: str | None = None) -> Any: 

122 """Read data from a file. 

123 

124 Parameters 

125 ---------- 

126 component : `str`, optional 

127 Component to read from the file. Only used if the `StorageClass` 

128 for reading differed from the `StorageClass` used to write the 

129 file. 

130 

131 Returns 

132 ------- 

133 inMemoryDataset : `object` 

134 The requested data as a Python object. The type of object 

135 is controlled by the specific formatter. 

136 

137 Raises 

138 ------ 

139 ValueError 

140 Component requested but this file does not seem to be a concrete 

141 composite. 

142 KeyError 

143 Raised when parameters passed with fileDescriptor are not 

144 supported. 

145 """ 

146 # This formatter can not read a subset from disk because it 

147 # uses yaml or json. 

148 path = self.fileDescriptor.location.path 

149 

150 with open(path) as fd: 

151 if path.endswith(".yaml"): 

152 data = yaml.load(fd, Loader=yaml.SafeLoader) 

153 elif path.endswith(".json"): 

154 data = json.load(fd) 

155 else: 

156 raise RuntimeError(f"Unsupported file extension found in path '{path}'") 

157 

158 # We can slice up front if required 

159 parameters = self.fileDescriptor.parameters 

160 if "data" in data and parameters and "slice" in parameters: 

161 data["data"] = data["data"][parameters["slice"]] 

162 

163 pytype = self.fileDescriptor.storageClass.pytype 

164 inMemoryDataset = pytype(**data) 

165 

166 if not component: 

167 return inMemoryDataset 

168 

169 if component == "summary": 

170 return inMemoryDataset.summary 

171 elif component == "output": 

172 return inMemoryDataset.output 

173 elif component == "data": 

174 return inMemoryDataset.data 

175 elif component == "counter": 

176 return len(inMemoryDataset.data) 

177 raise ValueError(f"Unsupported component: {component}") 

178 

179 def write(self, inMemoryDataset: Any) -> None: 

180 """Write a Dataset. 

181 

182 Parameters 

183 ---------- 

184 inMemoryDataset : `object` 

185 The Dataset to store. 

186 

187 Returns 

188 ------- 

189 path : `str` 

190 The path to where the Dataset was stored within the datastore. 

191 """ 

192 fileDescriptor = self.fileDescriptor 

193 

194 # Update the location with the formatter-preferred file extension 

195 fileDescriptor.location.updateExtension(self.extension) 

196 

197 with open(fileDescriptor.location.path, "w") as fd: 

198 yaml.dump(inMemoryDataset._asdict(), fd) 

199 

200 

201class MetricsExampleDataFormatter(Formatter): 

202 """A specialist test formatter for the data component of a MetricsExample. 

203 

204 This is needed if the MetricsExample is disassembled and we want to 

205 support the derived component. 

206 """ 

207 

208 unsupportedParameters = None 

209 """Let the assembler delegate handle slice""" 

210 

211 extension = ".yaml" 

212 """Always write YAML""" 

213 

214 def read(self, component: str | None = None) -> Any: 

215 """Read data from a file. 

216 

217 Parameters 

218 ---------- 

219 component : `str`, optional 

220 Component to read from the file. Only used if the `StorageClass` 

221 for reading differed from the `StorageClass` used to write the 

222 file. 

223 

224 Returns 

225 ------- 

226 inMemoryDataset : `object` 

227 The requested data as a Python object. The type of object 

228 is controlled by the specific formatter. 

229 

230 Raises 

231 ------ 

232 ValueError 

233 Component requested but this file does not seem to be a concrete 

234 composite. 

235 KeyError 

236 Raised when parameters passed with fileDescriptor are not 

237 supported. 

238 """ 

239 # This formatter can not read a subset from disk because it 

240 # uses yaml. 

241 path = self.fileDescriptor.location.path 

242 with open(path) as fd: 

243 data = yaml.load(fd, Loader=yaml.SafeLoader) 

244 

245 # We can slice up front if required 

246 parameters = self.fileDescriptor.parameters 

247 if parameters and "slice" in parameters: 

248 data = data[parameters["slice"]] 

249 

250 # This should be a native list 

251 inMemoryDataset = data 

252 

253 if not component: 

254 return inMemoryDataset 

255 

256 if component == "counter": 

257 return len(inMemoryDataset) 

258 raise ValueError(f"Unsupported component: {component}") 

259 

260 def write(self, inMemoryDataset: Any) -> None: 

261 """Write a Dataset. 

262 

263 Parameters 

264 ---------- 

265 inMemoryDataset : `object` 

266 The Dataset to store. 

267 

268 Returns 

269 ------- 

270 path : `str` 

271 The path to where the Dataset was stored within the datastore. 

272 """ 

273 fileDescriptor = self.fileDescriptor 

274 

275 # Update the location with the formatter-preferred file extension 

276 fileDescriptor.location.updateExtension(self.extension) 

277 

278 with open(fileDescriptor.location.path, "w") as fd: 

279 yaml.dump(inMemoryDataset, fd)