Coverage for python/lsst/daf/butler/tests/_examplePythonTypes.py: 37%

109 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 09:20 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22""" 

23Python classes that can be used to test datastores without requiring 

24large external dependencies on python classes such as afw or serialization 

25formats such as FITS or HDF5. 

26""" 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "ListDelegate", 

32 "MetricsDelegate", 

33 "MetricsExample", 

34 "registerMetricsExample", 

35 "MetricsExampleModel", 

36 "MetricsExampleDataclass", 

37) 

38 

39 

40import copy 

41import dataclasses 

42import types 

43from collections.abc import Mapping 

44from typing import TYPE_CHECKING, Any 

45 

46from lsst.daf.butler import StorageClass, StorageClassDelegate 

47from pydantic import BaseModel 

48 

49if TYPE_CHECKING: 

50 from lsst.daf.butler import Butler, Datastore, FormatterFactory 

51 

52 

53def registerMetricsExample(butler: Butler) -> None: 

54 """Modify a repository to support reading and writing 

55 `MetricsExample` objects. 

56 

57 This method allows `MetricsExample` to be used with test repositories 

58 in any package without needing to provide a custom configuration there. 

59 

60 Parameters 

61 ---------- 

62 butler : `lsst.daf.butler.Butler` 

63 The repository that needs to support `MetricsExample`. 

64 

65 Notes 

66 ----- 

67 This method enables the following storage classes: 

68 

69 ``StructuredData`` 

70 A `MetricsExample` whose ``summary``, ``output``, and ``data`` members 

71 can be retrieved as dataset components. 

72 ``StructuredDataNoComponents`` 

73 A monolithic write of a `MetricsExample`. 

74 """ 

75 yamlDict = _addFullStorageClass( 

76 butler, 

77 "StructuredDataDictYaml", 

78 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

79 pytype=dict, 

80 ) 

81 

82 yamlList = _addFullStorageClass( 

83 butler, 

84 "StructuredDataListYaml", 

85 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

86 pytype=list, 

87 parameters={"slice"}, 

88 delegate="lsst.daf.butler.tests.ListDelegate", 

89 ) 

90 

91 _addFullStorageClass( 

92 butler, 

93 "StructuredDataNoComponents", 

94 "lsst.daf.butler.formatters.pickle.PickleFormatter", 

95 pytype=MetricsExample, 

96 parameters={"slice"}, 

97 delegate="lsst.daf.butler.tests.MetricsDelegate", 

98 ) 

99 

100 _addFullStorageClass( 

101 butler, 

102 "StructuredData", 

103 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

104 pytype=MetricsExample, 

105 components={ 

106 "summary": yamlDict, 

107 "output": yamlDict, 

108 "data": yamlList, 

109 }, 

110 delegate="lsst.daf.butler.tests.MetricsDelegate", 

111 ) 

112 

113 

114def _addFullStorageClass( 

115 butler: Butler, name: str, formatter: str, *args: Any, **kwargs: Any 

116) -> StorageClass: 

117 """Create a storage class-formatter pair in a repository if it does not 

118 already exist. 

119 

120 Parameters 

121 ---------- 

122 butler : `lsst.daf.butler.Butler` 

123 The repository that needs to contain the class. 

124 name : `str` 

125 The name to use for the class. 

126 formatter : `str` 

127 The formatter to use with the storage class. Ignored if ``butler`` 

128 does not use formatters. 

129 *args 

130 **kwargs 

131 Arguments, other than ``name``, to the `~lsst.daf.butler.StorageClass` 

132 constructor. 

133 

134 Returns 

135 ------- 

136 class : `lsst.daf.butler.StorageClass` 

137 The newly created storage class, or the class of the same name 

138 previously found in the repository. 

139 """ 

140 storageRegistry = butler._datastore.storageClassFactory 

141 

142 storage = StorageClass(name, *args, **kwargs) 

143 try: 

144 storageRegistry.registerStorageClass(storage) 

145 except ValueError: 

146 storage = storageRegistry.getStorageClass(name) 

147 

148 for registry in _getAllFormatterRegistries(butler._datastore): 

149 registry.registerFormatter(storage, formatter) 

150 

151 return storage 

152 

153 

154def _getAllFormatterRegistries(datastore: Datastore) -> list[FormatterFactory]: 

155 """Return all formatter registries used by a datastore. 

156 

157 Parameters 

158 ---------- 

159 datastore : `lsst.daf.butler.Datastore` 

160 A datastore containing zero or more formatter registries. 

161 

162 Returns 

163 ------- 

164 registries : `list` [`lsst.daf.butler.FormatterFactory`] 

165 A possibly empty list of all formatter registries used 

166 by ``datastore``. 

167 """ 

168 try: 

169 datastores = datastore.datastores # type: ignore[attr-defined] 

170 except AttributeError: 

171 datastores = [datastore] 

172 

173 registries = [] 

174 for datastore in datastores: 

175 try: 

176 # Not all datastores have a formatterFactory 

177 formatterRegistry = datastore.formatterFactory # type: ignore[attr-defined] 

178 except AttributeError: 

179 pass # no formatter needed 

180 else: 

181 registries.append(formatterRegistry) 

182 return registries 

183 

184 

185class MetricsExample: 

186 """Smorgasboard of information that might be the result of some 

187 processing. 

188 

189 Parameters 

190 ---------- 

191 summary : `dict` 

192 Simple dictionary mapping key performance metrics to a scalar 

193 result. 

194 output : `dict` 

195 Structured nested data. 

196 data : `list`, optional 

197 Arbitrary array data. 

198 """ 

199 

200 def __init__( 

201 self, 

202 summary: dict[str, Any] | None = None, 

203 output: dict[str, Any] | None = None, 

204 data: list[Any] | None = None, 

205 ) -> None: 

206 self.summary = summary 

207 self.output = output 

208 self.data = data 

209 

210 def __eq__(self, other: Any) -> bool: 

211 try: 

212 return self.summary == other.summary and self.output == other.output and self.data == other.data 

213 except AttributeError: 

214 pass 

215 return NotImplemented 

216 

217 def __str__(self) -> str: 

218 return str(self.exportAsDict()) 

219 

220 def __repr__(self) -> str: 

221 return f"MetricsExample({self.exportAsDict()})" 

222 

223 def exportAsDict(self) -> dict[str, list | dict | None]: 

224 """Convert object contents to a single python dict.""" 

225 exportDict: dict[str, list | dict | None] = {"summary": self.summary, "output": self.output} 

226 if self.data is not None: 

227 exportDict["data"] = list(self.data) 

228 else: 

229 exportDict["data"] = None 

230 return exportDict 

231 

232 def _asdict(self) -> dict[str, list | dict | None]: 

233 """Convert object contents to a single Python dict. 

234 

235 This interface is used for JSON serialization. 

236 

237 Returns 

238 ------- 

239 exportDict : `dict` 

240 Object contents in the form of a dict with keys corresponding 

241 to object attributes. 

242 """ 

243 return self.exportAsDict() 

244 

245 @classmethod 

246 def makeFromDict(cls, exportDict: dict[str, list | dict | None]) -> MetricsExample: 

247 """Create a new object from a dict that is compatible with that 

248 created by `exportAsDict`. 

249 

250 Parameters 

251 ---------- 

252 exportDict : `dict` 

253 `dict` with keys "summary", "output", and (optionally) "data". 

254 

255 Returns 

256 ------- 

257 newobject : `MetricsExample` 

258 New `MetricsExample` object. 

259 """ 

260 data = exportDict["data"] if "data" in exportDict else None 

261 assert isinstance(data, list | types.NoneType) 

262 assert isinstance(exportDict["summary"], dict | types.NoneType) 

263 assert isinstance(exportDict["output"], dict | types.NoneType) 

264 return cls(exportDict["summary"], exportDict["output"], data) 

265 

266 

267class MetricsExampleModel(BaseModel): 

268 """A variant of `MetricsExample` based on model.""" 

269 

270 summary: dict[str, Any] | None = None 

271 output: dict[str, Any] | None = None 

272 data: list[Any] | None = None 

273 

274 @classmethod 

275 def from_metrics(cls, metrics: MetricsExample) -> MetricsExampleModel: 

276 """Create a model based on an example.""" 

277 d = metrics.exportAsDict() 

278 # Assume pydantic v2 but fallback to v1 

279 try: 

280 return cls.model_validate(d) 

281 except AttributeError: 

282 return cls.parse_obj(d) 

283 

284 

285@dataclasses.dataclass 

286class MetricsExampleDataclass: 

287 """A variant of `MetricsExample` based on a dataclass.""" 

288 

289 summary: dict[str, Any] | None 

290 output: dict[str, Any] | None 

291 data: list[Any] | None 

292 

293 

294class ListDelegate(StorageClassDelegate): 

295 """Parameter handler for list parameters.""" 

296 

297 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

298 """Modify the in-memory dataset using the supplied parameters, 

299 returning a possibly new object. 

300 

301 Parameters 

302 ---------- 

303 inMemoryDataset : `object` 

304 Object to modify based on the parameters. 

305 parameters : `dict` 

306 Parameters to apply. Values are specific to the parameter. 

307 Supported parameters are defined in the associated 

308 `StorageClass`. If no relevant parameters are specified the 

309 inMemoryDataset will be return unchanged. 

310 

311 Returns 

312 ------- 

313 inMemoryDataset : `object` 

314 Updated form of supplied in-memory dataset, after parameters 

315 have been used. 

316 """ 

317 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

318 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

319 if use: 

320 inMemoryDataset = inMemoryDataset[use["slice"]] 

321 return inMemoryDataset 

322 

323 

324class MetricsDelegate(StorageClassDelegate): 

325 """Parameter handler for parameters using Metrics.""" 

326 

327 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

328 """Modify the in-memory dataset using the supplied parameters, 

329 returning a possibly new object. 

330 

331 Parameters 

332 ---------- 

333 inMemoryDataset : `object` 

334 Object to modify based on the parameters. 

335 parameters : `dict` 

336 Parameters to apply. Values are specific to the parameter. 

337 Supported parameters are defined in the associated 

338 `StorageClass`. If no relevant parameters are specified the 

339 inMemoryDataset will be return unchanged. 

340 

341 Returns 

342 ------- 

343 inMemoryDataset : `object` 

344 Updated form of supplied in-memory dataset, after parameters 

345 have been used. 

346 """ 

347 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

348 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

349 if use: 

350 inMemoryDataset.data = inMemoryDataset.data[use["slice"]] 

351 return inMemoryDataset 

352 

353 def getComponent(self, composite: Any, componentName: str) -> Any: 

354 if componentName == "counter": 

355 return len(composite.data) 

356 return super().getComponent(composite, componentName) 

357 

358 @classmethod 

359 def selectResponsibleComponent(cls, readComponent: str, fromComponents: set[str | None]) -> str: 

360 forwarderMap = { 

361 "counter": "data", 

362 } 

363 forwarder = forwarderMap.get(readComponent) 

364 if forwarder is not None and forwarder in fromComponents: 

365 return forwarder 

366 raise ValueError(f"Can not calculate read component {readComponent} from {fromComponents}")