Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("FormatterTest", "DoNothingFormatter", "LenientYamlFormatter", "MetricsExampleFormatter", 

25 "MultipleExtensionsFormatter", "SingleExtensionFormatter") 

26 

27from typing import ( 

28 TYPE_CHECKING, 

29 Any, 

30 Mapping, 

31 Optional, 

32) 

33 

34import yaml 

35 

36from ..core import Formatter 

37from ..formatters.yaml import YamlFormatter 

38 

39if TYPE_CHECKING: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true

40 from ..core import Location 

41 

42 

43class DoNothingFormatter(Formatter): 

44 """A test formatter that does not need to format anything and has 

45 parameters.""" 

46 

47 def read(self, component: Optional[str] = None) -> Any: 

48 raise NotImplementedError("Type does not support reading") 

49 

50 def write(self, inMemoryDataset: Any) -> str: 

51 raise NotImplementedError("Type does not support writing") 

52 

53 

54class FormatterTest(Formatter): 

55 """A test formatter that does not need to format anything.""" 

56 

57 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"}) 

58 

59 def read(self, component: Optional[str] = None) -> Any: 

60 raise NotImplementedError("Type does not support reading") 

61 

62 def write(self, inMemoryDataset: Any) -> str: 

63 raise NotImplementedError("Type does not support writing") 

64 

65 @staticmethod 

66 def validateWriteRecipes(recipes: Optional[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: 

67 if not recipes: 

68 return recipes 

69 for recipeName in recipes: 

70 if "mode" not in recipes[recipeName]: 

71 raise RuntimeError("'mode' is a required write recipe parameter") 

72 return recipes 

73 

74 

75class SingleExtensionFormatter(DoNothingFormatter): 

76 """A do nothing formatter that has a single extension registered.""" 

77 extension = ".fits" 

78 

79 

80class MultipleExtensionsFormatter(SingleExtensionFormatter): 

81 """A formatter that has multiple extensions registered.""" 

82 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"}) 

83 

84 

85class LenientYamlFormatter(YamlFormatter): 

86 """A test formatter that allows any file extension but always reads and 

87 writes YAML.""" 

88 extension = ".yaml" 

89 

90 @classmethod 

91 def validateExtension(cls, location: Location) -> None: 

92 return 

93 

94 

95class MetricsExampleFormatter(Formatter): 

96 """A specialist test formatter for metrics that supports components 

97 directly without assembler.""" 

98 

99 extension = ".yaml" 

100 """Always write YAML""" 

101 

102 def read(self, component=None): 

103 """Read data from a file. 

104 

105 Parameters 

106 ---------- 

107 component : `str`, optional 

108 Component to read from the file. Only used if the `StorageClass` 

109 for reading differed from the `StorageClass` used to write the 

110 file. 

111 

112 Returns 

113 ------- 

114 inMemoryDataset : `object` 

115 The requested data as a Python object. The type of object 

116 is controlled by the specific formatter. 

117 

118 Raises 

119 ------ 

120 ValueError 

121 Component requested but this file does not seem to be a concrete 

122 composite. 

123 KeyError 

124 Raised when parameters passed with fileDescriptor are not 

125 supported. 

126 """ 

127 

128 # This formatter can not read a subset from disk because it 

129 # uses yaml. 

130 path = self.fileDescriptor.location.path 

131 with open(path, "r") as fd: 

132 data = yaml.load(fd, Loader=yaml.SafeLoader) 

133 

134 # We can slice up front if required 

135 parameters = self.fileDescriptor.parameters 

136 if "data" in data and parameters and "slice" in parameters: 

137 data["data"] = data["data"][parameters["slice"]] 

138 

139 pytype = self.fileDescriptor.storageClass.pytype 

140 inMemoryDataset = pytype(**data) 

141 

142 if not component: 

143 return inMemoryDataset 

144 

145 if component == "summary": 

146 return inMemoryDataset.summary 

147 elif component == "output": 

148 return inMemoryDataset.output 

149 elif component == "data": 

150 return inMemoryDataset.data 

151 elif component == "counter": 

152 return len(inMemoryDataset.data) 

153 raise ValueError(f"Unsupported component: {component}") 

154 

155 def write(self, inMemoryDataset: Any) -> str: 

156 """Write a Dataset. 

157 

158 Parameters 

159 ---------- 

160 inMemoryDataset : `object` 

161 The Dataset to store. 

162 

163 Returns 

164 ------- 

165 path : `str` 

166 The path to where the Dataset was stored within the datastore. 

167 """ 

168 fileDescriptor = self.fileDescriptor 

169 

170 # Update the location with the formatter-preferred file extension 

171 fileDescriptor.location.updateExtension(self.extension) 

172 

173 with open(fileDescriptor.location.path, "w") as fd: 

174 yaml.dump(inMemoryDataset._asdict(), fd) 

175 return fileDescriptor.location.pathInStore 

176 

177 

178class MetricsExampleDataFormatter(Formatter): 

179 """A specialist test formatter for the data component of a MetricsExample. 

180 

181 This is needed if the MetricsExample is dissassembled and we want to 

182 support the read-only component. 

183 """ 

184 

185 unsupportedParameters = None 

186 """Let the assembler handle slice""" 

187 

188 extension = ".yaml" 

189 """Always write YAML""" 

190 

191 def read(self, component=None): 

192 """Read data from a file. 

193 

194 Parameters 

195 ---------- 

196 component : `str`, optional 

197 Component to read from the file. Only used if the `StorageClass` 

198 for reading differed from the `StorageClass` used to write the 

199 file. 

200 

201 Returns 

202 ------- 

203 inMemoryDataset : `object` 

204 The requested data as a Python object. The type of object 

205 is controlled by the specific formatter. 

206 

207 Raises 

208 ------ 

209 ValueError 

210 Component requested but this file does not seem to be a concrete 

211 composite. 

212 KeyError 

213 Raised when parameters passed with fileDescriptor are not 

214 supported. 

215 """ 

216 

217 # This formatter can not read a subset from disk because it 

218 # uses yaml. 

219 path = self.fileDescriptor.location.path 

220 with open(path, "r") as fd: 

221 data = yaml.load(fd, Loader=yaml.SafeLoader) 

222 

223 # We can slice up front if required 

224 parameters = self.fileDescriptor.parameters 

225 if parameters and "slice" in parameters: 

226 data = data[parameters["slice"]] 

227 

228 # This should be a native list 

229 inMemoryDataset = data 

230 

231 if not component: 

232 return inMemoryDataset 

233 

234 if component == "counter": 

235 return len(inMemoryDataset) 

236 raise ValueError(f"Unsupported component: {component}") 

237 

238 def write(self, inMemoryDataset: Any) -> str: 

239 """Write a Dataset. 

240 

241 Parameters 

242 ---------- 

243 inMemoryDataset : `object` 

244 The Dataset to store. 

245 

246 Returns 

247 ------- 

248 path : `str` 

249 The path to where the Dataset was stored within the datastore. 

250 """ 

251 fileDescriptor = self.fileDescriptor 

252 

253 # Update the location with the formatter-preferred file extension 

254 fileDescriptor.location.updateExtension(self.extension) 

255 

256 with open(fileDescriptor.location.path, "w") as fd: 

257 yaml.dump(inMemoryDataset, fd) 

258 return fileDescriptor.location.pathInStore