Coverage for python/lsst/daf/butler/tests/_examplePythonTypes.py: 37%

109 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28""" 

29Python classes that can be used to test datastores without requiring 

30large external dependencies on python classes such as afw or serialization 

31formats such as FITS or HDF5. 

32""" 

33 

34from __future__ import annotations 

35 

36__all__ = ( 

37 "ListDelegate", 

38 "MetricsDelegate", 

39 "MetricsExample", 

40 "registerMetricsExample", 

41 "MetricsExampleModel", 

42 "MetricsExampleDataclass", 

43) 

44 

45 

46import copy 

47import dataclasses 

48import types 

49from collections.abc import Mapping 

50from typing import TYPE_CHECKING, Any 

51 

52from lsst.daf.butler import StorageClass, StorageClassDelegate 

53from pydantic import BaseModel 

54 

55if TYPE_CHECKING: 

56 from lsst.daf.butler import Butler, Datastore, FormatterFactory 

57 

58 

59def registerMetricsExample(butler: Butler) -> None: 

60 """Modify a repository to support reading and writing 

61 `MetricsExample` objects. 

62 

63 This method allows `MetricsExample` to be used with test repositories 

64 in any package without needing to provide a custom configuration there. 

65 

66 Parameters 

67 ---------- 

68 butler : `lsst.daf.butler.Butler` 

69 The repository that needs to support `MetricsExample`. 

70 

71 Notes 

72 ----- 

73 This method enables the following storage classes: 

74 

75 ``StructuredData`` 

76 A `MetricsExample` whose ``summary``, ``output``, and ``data`` members 

77 can be retrieved as dataset components. 

78 ``StructuredDataNoComponents`` 

79 A monolithic write of a `MetricsExample`. 

80 """ 

81 yamlDict = _addFullStorageClass( 

82 butler, 

83 "StructuredDataDictYaml", 

84 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

85 pytype=dict, 

86 ) 

87 

88 yamlList = _addFullStorageClass( 

89 butler, 

90 "StructuredDataListYaml", 

91 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

92 pytype=list, 

93 parameters={"slice"}, 

94 delegate="lsst.daf.butler.tests.ListDelegate", 

95 ) 

96 

97 _addFullStorageClass( 

98 butler, 

99 "StructuredDataNoComponents", 

100 "lsst.daf.butler.formatters.pickle.PickleFormatter", 

101 pytype=MetricsExample, 

102 parameters={"slice"}, 

103 delegate="lsst.daf.butler.tests.MetricsDelegate", 

104 ) 

105 

106 _addFullStorageClass( 

107 butler, 

108 "StructuredData", 

109 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

110 pytype=MetricsExample, 

111 components={ 

112 "summary": yamlDict, 

113 "output": yamlDict, 

114 "data": yamlList, 

115 }, 

116 delegate="lsst.daf.butler.tests.MetricsDelegate", 

117 ) 

118 

119 

120def _addFullStorageClass( 

121 butler: Butler, name: str, formatter: str, *args: Any, **kwargs: Any 

122) -> StorageClass: 

123 """Create a storage class-formatter pair in a repository if it does not 

124 already exist. 

125 

126 Parameters 

127 ---------- 

128 butler : `lsst.daf.butler.Butler` 

129 The repository that needs to contain the class. 

130 name : `str` 

131 The name to use for the class. 

132 formatter : `str` 

133 The formatter to use with the storage class. Ignored if ``butler`` 

134 does not use formatters. 

135 *args 

136 **kwargs 

137 Arguments, other than ``name``, to the `~lsst.daf.butler.StorageClass` 

138 constructor. 

139 

140 Returns 

141 ------- 

142 class : `lsst.daf.butler.StorageClass` 

143 The newly created storage class, or the class of the same name 

144 previously found in the repository. 

145 """ 

146 storageRegistry = butler._datastore.storageClassFactory 

147 

148 storage = StorageClass(name, *args, **kwargs) 

149 try: 

150 storageRegistry.registerStorageClass(storage) 

151 except ValueError: 

152 storage = storageRegistry.getStorageClass(name) 

153 

154 for registry in _getAllFormatterRegistries(butler._datastore): 

155 registry.registerFormatter(storage, formatter) 

156 

157 return storage 

158 

159 

160def _getAllFormatterRegistries(datastore: Datastore) -> list[FormatterFactory]: 

161 """Return all formatter registries used by a datastore. 

162 

163 Parameters 

164 ---------- 

165 datastore : `lsst.daf.butler.Datastore` 

166 A datastore containing zero or more formatter registries. 

167 

168 Returns 

169 ------- 

170 registries : `list` [`lsst.daf.butler.FormatterFactory`] 

171 A possibly empty list of all formatter registries used 

172 by ``datastore``. 

173 """ 

174 try: 

175 datastores = datastore.datastores # type: ignore[attr-defined] 

176 except AttributeError: 

177 datastores = [datastore] 

178 

179 registries = [] 

180 for datastore in datastores: 

181 try: 

182 # Not all datastores have a formatterFactory 

183 formatterRegistry = datastore.formatterFactory # type: ignore[attr-defined] 

184 except AttributeError: 

185 pass # no formatter needed 

186 else: 

187 registries.append(formatterRegistry) 

188 return registries 

189 

190 

191class MetricsExample: 

192 """Smorgasboard of information that might be the result of some 

193 processing. 

194 

195 Parameters 

196 ---------- 

197 summary : `dict` 

198 Simple dictionary mapping key performance metrics to a scalar 

199 result. 

200 output : `dict` 

201 Structured nested data. 

202 data : `list`, optional 

203 Arbitrary array data. 

204 """ 

205 

206 def __init__( 

207 self, 

208 summary: dict[str, Any] | None = None, 

209 output: dict[str, Any] | None = None, 

210 data: list[Any] | None = None, 

211 ) -> None: 

212 self.summary = summary 

213 self.output = output 

214 self.data = data 

215 

216 def __eq__(self, other: Any) -> bool: 

217 try: 

218 return self.summary == other.summary and self.output == other.output and self.data == other.data 

219 except AttributeError: 

220 pass 

221 return NotImplemented 

222 

223 def __str__(self) -> str: 

224 return str(self.exportAsDict()) 

225 

226 def __repr__(self) -> str: 

227 return f"MetricsExample({self.exportAsDict()})" 

228 

229 def exportAsDict(self) -> dict[str, list | dict | None]: 

230 """Convert object contents to a single python dict.""" 

231 exportDict: dict[str, list | dict | None] = {"summary": self.summary, "output": self.output} 

232 if self.data is not None: 

233 exportDict["data"] = list(self.data) 

234 else: 

235 exportDict["data"] = None 

236 return exportDict 

237 

238 def _asdict(self) -> dict[str, list | dict | None]: 

239 """Convert object contents to a single Python dict. 

240 

241 This interface is used for JSON serialization. 

242 

243 Returns 

244 ------- 

245 exportDict : `dict` 

246 Object contents in the form of a dict with keys corresponding 

247 to object attributes. 

248 """ 

249 return self.exportAsDict() 

250 

251 @classmethod 

252 def makeFromDict(cls, exportDict: dict[str, list | dict | None]) -> MetricsExample: 

253 """Create a new object from a dict that is compatible with that 

254 created by `exportAsDict`. 

255 

256 Parameters 

257 ---------- 

258 exportDict : `dict` 

259 `dict` with keys "summary", "output", and (optionally) "data". 

260 

261 Returns 

262 ------- 

263 newobject : `MetricsExample` 

264 New `MetricsExample` object. 

265 """ 

266 data = exportDict["data"] if "data" in exportDict else None 

267 assert isinstance(data, list | types.NoneType) 

268 assert isinstance(exportDict["summary"], dict | types.NoneType) 

269 assert isinstance(exportDict["output"], dict | types.NoneType) 

270 return cls(exportDict["summary"], exportDict["output"], data) 

271 

272 

273class MetricsExampleModel(BaseModel): 

274 """A variant of `MetricsExample` based on model.""" 

275 

276 summary: dict[str, Any] | None = None 

277 output: dict[str, Any] | None = None 

278 data: list[Any] | None = None 

279 

280 @classmethod 

281 def from_metrics(cls, metrics: MetricsExample) -> MetricsExampleModel: 

282 """Create a model based on an example.""" 

283 d = metrics.exportAsDict() 

284 # Assume pydantic v2 but fallback to v1 

285 try: 

286 return cls.model_validate(d) # type: ignore 

287 except AttributeError: 

288 return cls.parse_obj(d) 

289 

290 

291@dataclasses.dataclass 

292class MetricsExampleDataclass: 

293 """A variant of `MetricsExample` based on a dataclass.""" 

294 

295 summary: dict[str, Any] | None 

296 output: dict[str, Any] | None 

297 data: list[Any] | None 

298 

299 

300class ListDelegate(StorageClassDelegate): 

301 """Parameter handler for list parameters.""" 

302 

303 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

304 """Modify the in-memory dataset using the supplied parameters, 

305 returning a possibly new object. 

306 

307 Parameters 

308 ---------- 

309 inMemoryDataset : `object` 

310 Object to modify based on the parameters. 

311 parameters : `dict` 

312 Parameters to apply. Values are specific to the parameter. 

313 Supported parameters are defined in the associated 

314 `StorageClass`. If no relevant parameters are specified the 

315 inMemoryDataset will be return unchanged. 

316 

317 Returns 

318 ------- 

319 inMemoryDataset : `object` 

320 Updated form of supplied in-memory dataset, after parameters 

321 have been used. 

322 """ 

323 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

324 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

325 if use: 

326 inMemoryDataset = inMemoryDataset[use["slice"]] 

327 return inMemoryDataset 

328 

329 

330class MetricsDelegate(StorageClassDelegate): 

331 """Parameter handler for parameters using Metrics.""" 

332 

333 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

334 """Modify the in-memory dataset using the supplied parameters, 

335 returning a possibly new object. 

336 

337 Parameters 

338 ---------- 

339 inMemoryDataset : `object` 

340 Object to modify based on the parameters. 

341 parameters : `dict` 

342 Parameters to apply. Values are specific to the parameter. 

343 Supported parameters are defined in the associated 

344 `StorageClass`. If no relevant parameters are specified the 

345 inMemoryDataset will be return unchanged. 

346 

347 Returns 

348 ------- 

349 inMemoryDataset : `object` 

350 Updated form of supplied in-memory dataset, after parameters 

351 have been used. 

352 """ 

353 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

354 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

355 if use: 

356 inMemoryDataset.data = inMemoryDataset.data[use["slice"]] 

357 return inMemoryDataset 

358 

359 def getComponent(self, composite: Any, componentName: str) -> Any: 

360 if componentName == "counter": 

361 return len(composite.data) 

362 return super().getComponent(composite, componentName) 

363 

364 @classmethod 

365 def selectResponsibleComponent(cls, readComponent: str, fromComponents: set[str | None]) -> str: 

366 forwarderMap = { 

367 "counter": "data", 

368 } 

369 forwarder = forwarderMap.get(readComponent) 

370 if forwarder is not None and forwarder in fromComponents: 

371 return forwarder 

372 raise ValueError(f"Can not calculate read component {readComponent} from {fromComponents}")