Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22""" 

23Python classes that can be used to test datastores without requiring 

24large external dependencies on python classes such as afw or serialization 

25formats such as FITS or HDF5. 

26""" 

27 

28__all__ = ("ListAssembler", "MetricsAssembler", "MetricsExample", "registerMetricsExample") 

29 

30 

31import copy 

32from lsst.daf.butler import CompositeAssembler, StorageClass 

33 

34 

35def registerMetricsExample(butler): 

36 """Modify a repository to support reading and writing 

37 `MetricsExample` objects. 

38 

39 This method allows `MetricsExample` to be used with test repositories 

40 in any package without needing to provide a custom configuration there. 

41 

42 Parameters 

43 ---------- 

44 butler : `lsst.daf.butler.Butler` 

45 The repository that needs to support `MetricsExample`. 

46 

47 Notes 

48 ----- 

49 This method enables the following storage classes: 

50 

51 ``StructuredData`` 

52 A `MetricsExample` whose ``summary``, ``output``, and ``data`` members 

53 can be retrieved as dataset components. 

54 ``StructuredDataNoComponents`` 

55 A monolithic write of a `MetricsExample`. 

56 """ 

57 yamlDict = _addFullStorageClass( 

58 butler, 

59 "StructuredDataDictYaml", 

60 "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter", 

61 pytype=dict, 

62 ) 

63 

64 yamlList = _addFullStorageClass( 

65 butler, 

66 "StructuredDataListYaml", 

67 "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter", 

68 pytype=list, 

69 parameters={"slice"}, 

70 assembler="lsst.daf.butler.tests.ListAssembler" 

71 ) 

72 

73 _addFullStorageClass( 

74 butler, 

75 "StructuredDataNoComponents", 

76 "lsst.daf.butler.formatters.pickleFormatter.PickleFormatter", 

77 pytype=MetricsExample, 

78 parameters={"slice"}, 

79 assembler="lsst.daf.butler.tests.MetricsAssembler" 

80 ) 

81 

82 _addFullStorageClass( 

83 butler, 

84 "StructuredData", 

85 "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter", 

86 pytype=MetricsExample, 

87 components={"summary": yamlDict, 

88 "output": yamlDict, 

89 "data": yamlList, 

90 }, 

91 assembler="lsst.daf.butler.tests.MetricsAssembler" 

92 ) 

93 

94 

95def _addFullStorageClass(butler, name, formatter, *args, **kwargs): 

96 """Create a storage class-formatter pair in a repository if it does not 

97 already exist. 

98 

99 Parameters 

100 ---------- 

101 butler : `lsst.daf.butler.Butler` 

102 The repository that needs to contain the class. 

103 name : `str` 

104 The name to use for the class. 

105 formatter : `str` 

106 The formatter to use with the storage class. Ignored if ``butler`` 

107 does not use formatters. 

108 *args 

109 **kwargs 

110 Arguments, other than ``name``, to the `~lsst.daf.butler.StorageClass` 

111 constructor. 

112 

113 Returns 

114 ------- 

115 class : `lsst.daf.butler.StorageClass` 

116 The newly created storage class, or the class of the same name 

117 previously found in the repository. 

118 """ 

119 storageRegistry = butler.datastore.storageClassFactory 

120 

121 storage = StorageClass(name, *args, **kwargs) 

122 try: 

123 storageRegistry.registerStorageClass(storage) 

124 except ValueError: 

125 storage = storageRegistry.getStorageClass(name) 

126 

127 for registry in _getAllFormatterRegistries(butler.datastore): 

128 registry.registerFormatter(storage, formatter) 

129 

130 return storage 

131 

132 

133def _getAllFormatterRegistries(datastore): 

134 """Return all formatter registries used by a datastore. 

135 

136 Parameters 

137 ---------- 

138 datastore : `lsst.daf.butler.Datastore` 

139 A datastore containing zero or more formatter registries. 

140 

141 Returns 

142 ------- 

143 registries : `list` [`lsst.daf.butler.FormatterRegistry`] 

144 A possibly empty list of all formatter registries used 

145 by ``datastore``. 

146 """ 

147 try: 

148 datastores = datastore.datastores 

149 except AttributeError: 

150 datastores = [datastore] 

151 

152 registries = [] 

153 for datastore in datastores: 

154 try: 

155 # Not all datastores have a formatterFactory 

156 formatterRegistry = datastore.formatterFactory 

157 except AttributeError: 

158 pass # no formatter needed 

159 else: 

160 registries.append(formatterRegistry) 

161 return registries 

162 

163 

164class MetricsExample: 

165 """Smorgasboard of information that might be the result of some 

166 processing. 

167 

168 Parameters 

169 ---------- 

170 summary : `dict` 

171 Simple dictionary mapping key performance metrics to a scalar 

172 result. 

173 output : `dict` 

174 Structured nested data. 

175 data : `list`, optional 

176 Arbitrary array data. 

177 """ 

178 

179 def __init__(self, summary=None, output=None, data=None): 

180 self.summary = summary 

181 self.output = output 

182 self.data = data 

183 

184 def __eq__(self, other): 

185 return self.summary == other.summary and self.output == other.output and self.data == other.data 

186 

187 def exportAsDict(self): 

188 """Convert object contents to a single python dict.""" 

189 exportDict = {"summary": self.summary, 

190 "output": self.output} 

191 if self.data is not None: 

192 exportDict["data"] = list(self.data) 

193 return exportDict 

194 

195 def _asdict(self): 

196 """Convert object contents to a single Python dict. 

197 

198 This interface is used for JSON serialization. 

199 

200 Returns 

201 ------- 

202 exportDict : `dict` 

203 Object contents in the form of a dict with keys corresponding 

204 to object attributes. 

205 """ 

206 return self.exportAsDict() 

207 

208 @classmethod 

209 def makeFromDict(cls, exportDict): 

210 """Create a new object from a dict that is compatible with that 

211 created by `exportAsDict`. 

212 

213 Parameters 

214 ---------- 

215 exportDict : `dict` 

216 `dict` with keys "summary", "output", and (optionally) "data". 

217 

218 Returns 

219 ------- 

220 newobject : `MetricsExample` 

221 New `MetricsExample` object. 

222 """ 

223 data = None 

224 if "data" in exportDict: 

225 data = exportDict["data"] 

226 return cls(exportDict["summary"], exportDict["output"], data) 

227 

228 

229class ListAssembler(CompositeAssembler): 

230 """Parameter handler for list parameters""" 

231 

232 def handleParameters(self, inMemoryDataset, parameters=None): 

233 """Modify the in-memory dataset using the supplied parameters, 

234 returning a possibly new object. 

235 

236 Parameters 

237 ---------- 

238 inMemoryDataset : `object` 

239 Object to modify based on the parameters. 

240 parameters : `dict` 

241 Parameters to apply. Values are specific to the parameter. 

242 Supported parameters are defined in the associated 

243 `StorageClass`. If no relevant parameters are specified the 

244 inMemoryDataset will be return unchanged. 

245 

246 Returns 

247 ------- 

248 inMemoryDataset : `object` 

249 Updated form of supplied in-memory dataset, after parameters 

250 have been used. 

251 """ 

252 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

253 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

254 if use: 

255 inMemoryDataset = inMemoryDataset[use["slice"]] 

256 return inMemoryDataset 

257 

258 

259class MetricsAssembler(CompositeAssembler): 

260 """Parameter handler for parameters using Metrics""" 

261 

262 def handleParameters(self, inMemoryDataset, parameters=None): 

263 """Modify the in-memory dataset using the supplied parameters, 

264 returning a possibly new object. 

265 

266 Parameters 

267 ---------- 

268 inMemoryDataset : `object` 

269 Object to modify based on the parameters. 

270 parameters : `dict` 

271 Parameters to apply. Values are specific to the parameter. 

272 Supported parameters are defined in the associated 

273 `StorageClass`. If no relevant parameters are specified the 

274 inMemoryDataset will be return unchanged. 

275 

276 Returns 

277 ------- 

278 inMemoryDataset : `object` 

279 Updated form of supplied in-memory dataset, after parameters 

280 have been used. 

281 """ 

282 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

283 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

284 if use: 

285 inMemoryDataset.data = inMemoryDataset.data[use["slice"]] 

286 return inMemoryDataset