Coverage for python / lsst / daf / butler / tests / _examplePythonTypes.py: 34%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28""" 

29Python classes that can be used to test datastores without requiring 

30large external dependencies on python classes such as afw or serialization 

31formats such as FITS or HDF5. 

32""" 

33 

34from __future__ import annotations 

35 

36__all__ = ( 

37 "ListDelegate", 

38 "MetricsDelegate", 

39 "MetricsExample", 

40 "MetricsExampleDataclass", 

41 "MetricsExampleModel", 

42 "registerMetricsExample", 

43) 

44 

45 

46import copy 

47import dataclasses 

48import types 

49import uuid 

50from collections.abc import Mapping 

51from typing import TYPE_CHECKING, Any 

52 

53from pydantic import BaseModel 

54 

55from lsst.daf.butler import DatasetProvenance, StorageClass, StorageClassDelegate 

56 

57if TYPE_CHECKING: 

58 from lsst.daf.butler import Butler, Datastore, FormatterFactory 

59 

60 

61def registerMetricsExample(butler: Butler) -> None: 

62 """Modify a repository to support reading and writing 

63 `MetricsExample` objects. 

64 

65 This method allows `MetricsExample` to be used with test repositories 

66 in any package without needing to provide a custom configuration there. 

67 

68 Parameters 

69 ---------- 

70 butler : `lsst.daf.butler.Butler` 

71 The repository that needs to support `MetricsExample`. 

72 

73 Notes 

74 ----- 

75 This method enables the following storage classes: 

76 

77 ``StructuredData`` 

78 A `MetricsExample` whose ``summary``, ``output``, and ``data`` members 

79 can be retrieved as dataset components. 

80 ``StructuredDataNoComponents`` 

81 A monolithic write of a `MetricsExample`. 

82 

83 These definitions must match the equivalent definitions in the test YAML 

84 files. 

85 """ 

86 yamlDict = _addFullStorageClass( 

87 butler, 

88 "StructuredDataDictYaml", 

89 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

90 pytype=dict, 

91 ) 

92 

93 yamlList = _addFullStorageClass( 

94 butler, 

95 "StructuredDataListYaml", 

96 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

97 pytype=list, 

98 parameters={"slice"}, 

99 delegate="lsst.daf.butler.tests.ListDelegate", 

100 ) 

101 

102 _addFullStorageClass( 

103 butler, 

104 "StructuredDataNoComponents", 

105 "lsst.daf.butler.formatters.pickle.PickleFormatter", 

106 pytype=MetricsExample, 

107 parameters={"slice"}, 

108 delegate="lsst.daf.butler.tests.MetricsDelegate", 

109 converters={"dict": "lsst.daf.butler.tests.MetricsExample.makeFromDict"}, 

110 ) 

111 

112 _addFullStorageClass( 

113 butler, 

114 "StructuredData", 

115 "lsst.daf.butler.formatters.yaml.YamlFormatter", 

116 pytype=MetricsExample, 

117 components={ 

118 "summary": yamlDict, 

119 "output": yamlDict, 

120 "data": yamlList, 

121 }, 

122 delegate="lsst.daf.butler.tests.MetricsDelegate", 

123 ) 

124 

125 

126def _addFullStorageClass(butler: Butler, name: str, formatter: str, **kwargs: Any) -> StorageClass: 

127 """Create a storage class-formatter pair in a repository if it does not 

128 already exist. 

129 

130 Parameters 

131 ---------- 

132 butler : `lsst.daf.butler.Butler` 

133 The repository that needs to contain the class. 

134 name : `str` 

135 The name to use for the class. 

136 formatter : `str` 

137 The formatter to use with the storage class. Ignored if ``butler`` 

138 does not use formatters. 

139 **kwargs 

140 Arguments, other than ``name``, to the `~lsst.daf.butler.StorageClass` 

141 constructor. 

142 

143 Returns 

144 ------- 

145 class : `lsst.daf.butler.StorageClass` 

146 The newly created storage class, or the class of the same name 

147 previously found in the repository. 

148 """ 

149 storageRegistry = butler._datastore.storageClassFactory 

150 storage = StorageClass(name, **kwargs) 

151 try: 

152 storageRegistry.registerStorageClass(storage) 

153 except ValueError: 

154 storage = storageRegistry.getStorageClass(name) 

155 

156 for registry in _getAllFormatterRegistries(butler._datastore): 

157 registry.registerFormatter(storage, formatter) 

158 

159 return storage 

160 

161 

162def _getAllFormatterRegistries(datastore: Datastore) -> list[FormatterFactory]: 

163 """Return all formatter registries used by a datastore. 

164 

165 Parameters 

166 ---------- 

167 datastore : `lsst.daf.butler.Datastore` 

168 A datastore containing zero or more formatter registries. 

169 

170 Returns 

171 ------- 

172 registries : `list` [`lsst.daf.butler.FormatterFactory`] 

173 A possibly empty list of all formatter registries used 

174 by ``datastore``. 

175 """ 

176 try: 

177 datastores = datastore.datastores # type: ignore[attr-defined] 

178 except AttributeError: 

179 datastores = [datastore] 

180 

181 registries = [] 

182 for datastore in datastores: 

183 try: 

184 # Not all datastores have a formatterFactory 

185 formatterRegistry = datastore.formatterFactory # type: ignore[attr-defined] 

186 except AttributeError: 

187 pass # no formatter needed 

188 else: 

189 registries.append(formatterRegistry) 

190 return registries 

191 

192 

193class MetricsExample: 

194 """Smorgasboard of information that might be the result of some 

195 processing. 

196 

197 Parameters 

198 ---------- 

199 summary : `dict` 

200 Simple dictionary mapping key performance metrics to a scalar 

201 result. 

202 output : `dict` 

203 Structured nested data. 

204 data : `list`, optional 

205 Arbitrary array data. 

206 """ 

207 

208 def __init__( 

209 self, 

210 summary: dict[str, Any] | None = None, 

211 output: dict[str, Any] | None = None, 

212 data: list[Any] | None = None, 

213 ) -> None: 

214 self.summary = summary 

215 self.output = output 

216 self.data = data 

217 

218 def __eq__(self, other: Any) -> bool: 

219 try: 

220 return self.summary == other.summary and self.output == other.output and self.data == other.data 

221 except AttributeError: 

222 pass 

223 return NotImplemented 

224 

225 def __str__(self) -> str: 

226 return str(self.exportAsDict()) 

227 

228 def __repr__(self) -> str: 

229 return f"MetricsExample({self.exportAsDict()})" 

230 

231 def exportAsDict(self) -> dict[str, list | dict | None]: 

232 """Convert object contents to a single python dict.""" 

233 exportDict: dict[str, list | dict | None] = {"summary": self.summary, "output": self.output} 

234 if self.data is not None: 

235 exportDict["data"] = list(self.data) 

236 else: 

237 exportDict["data"] = None 

238 return exportDict 

239 

240 def _asdict(self) -> dict[str, list | dict | None]: 

241 """Convert object contents to a single Python dict. 

242 

243 This interface is used for JSON serialization. 

244 

245 Returns 

246 ------- 

247 exportDict : `dict` 

248 Object contents in the form of a dict with keys corresponding 

249 to object attributes. 

250 """ 

251 return self.exportAsDict() 

252 

253 @classmethod 

254 def makeFromDict(cls, exportDict: dict[str, list | dict | None]) -> MetricsExample: 

255 """Create a new object from a dict that is compatible with that 

256 created by `exportAsDict`. 

257 

258 Parameters 

259 ---------- 

260 exportDict : `dict` 

261 `dict` with keys "summary", "output", and (optionally) "data". 

262 

263 Returns 

264 ------- 

265 newobject : `MetricsExample` 

266 New `MetricsExample` object. 

267 """ 

268 data = exportDict["data"] if "data" in exportDict else None 

269 assert isinstance(data, list | types.NoneType) 

270 assert isinstance(exportDict["summary"], dict | types.NoneType) 

271 assert isinstance(exportDict["output"], dict | types.NoneType) 

272 return cls(exportDict["summary"], exportDict["output"], data) 

273 

274 @classmethod 

275 def from_model(cls, model: MetricsExampleModel) -> MetricsExample: 

276 """Create metrics from Pydantic model. 

277 

278 Parameters 

279 ---------- 

280 model : `MetricsExampleModel` 

281 Source model. 

282 

283 Returns 

284 ------- 

285 newobject : `MetricsExample` 

286 New `MetricsExample` object. 

287 """ 

288 return cls(model.summary, model.output, model.data) 

289 

290 

291class MetricsExampleModel(BaseModel): 

292 """A variant of `MetricsExample` based on model.""" 

293 

294 summary: dict[str, Any] | None = None 

295 output: dict[str, Any] | None = None 

296 data: list[Any] | None = None 

297 provenance: DatasetProvenance | None = None 

298 dataset_id: uuid.UUID | None = None 

299 

300 @classmethod 

301 def from_metrics(cls, metrics: MetricsExample) -> MetricsExampleModel: 

302 """Create a model based on an example. 

303 

304 Parameters 

305 ---------- 

306 metrics : `MetricsExample` 

307 Metrics from which to construct the model. 

308 

309 Returns 

310 ------- 

311 model : `MetricsExampleModel` 

312 New model. 

313 """ 

314 d = metrics.exportAsDict() 

315 # Assume pydantic v2 but fallback to v1 

316 try: 

317 return cls.model_validate(d) # type: ignore 

318 except AttributeError: 

319 return cls.parse_obj(d) 

320 

321 

322@dataclasses.dataclass 

323class MetricsExampleDataclass: 

324 """A variant of `MetricsExample` based on a dataclass.""" 

325 

326 summary: dict[str, Any] | None 

327 output: dict[str, Any] | None 

328 data: list[Any] | None 

329 

330 

331class ListDelegate(StorageClassDelegate): 

332 """Parameter handler for list parameters.""" 

333 

334 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

335 """Modify the in-memory dataset using the supplied parameters, 

336 returning a possibly new object. 

337 

338 Parameters 

339 ---------- 

340 inMemoryDataset : `object` 

341 Object to modify based on the parameters. 

342 parameters : `dict` 

343 Parameters to apply. Values are specific to the parameter. 

344 Supported parameters are defined in the associated 

345 `StorageClass`. If no relevant parameters are specified the 

346 inMemoryDataset will be return unchanged. 

347 

348 Returns 

349 ------- 

350 inMemoryDataset : `object` 

351 Updated form of supplied in-memory dataset, after parameters 

352 have been used. 

353 """ 

354 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

355 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

356 if use: 

357 inMemoryDataset = inMemoryDataset[use["slice"]] 

358 return inMemoryDataset 

359 

360 

361class MetricsDelegate(StorageClassDelegate): 

362 """Parameter handler for parameters using Metrics.""" 

363 

364 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

365 """Modify the in-memory dataset using the supplied parameters, 

366 returning a possibly new object. 

367 

368 Parameters 

369 ---------- 

370 inMemoryDataset : `object` 

371 Object to modify based on the parameters. 

372 parameters : `dict` 

373 Parameters to apply. Values are specific to the parameter. 

374 Supported parameters are defined in the associated 

375 `StorageClass`. If no relevant parameters are specified the 

376 inMemoryDataset will be return unchanged. 

377 

378 Returns 

379 ------- 

380 inMemoryDataset : `object` 

381 Updated form of supplied in-memory dataset, after parameters 

382 have been used. 

383 """ 

384 inMemoryDataset = copy.deepcopy(inMemoryDataset) 

385 use = self.storageClass.filterParameters(parameters, subset={"slice"}) 

386 if use: 

387 inMemoryDataset.data = inMemoryDataset.data[use["slice"]] 

388 return inMemoryDataset 

389 

390 def getComponent(self, composite: Any, componentName: str) -> Any: 

391 if componentName == "counter": 

392 return len(composite.data) 

393 return super().getComponent(composite, componentName) 

394 

395 @classmethod 

396 def selectResponsibleComponent(cls, readComponent: str, fromComponents: set[str | None]) -> str: 

397 forwarderMap = { 

398 "counter": "data", 

399 } 

400 forwarder = forwarderMap.get(readComponent) 

401 if forwarder is not None and forwarder in fromComponents: 

402 return forwarder 

403 raise ValueError(f"Can not calculate read component {readComponent} from {fromComponents}")