Coverage for python/lsst/daf/butler/_storage_class_delegate.py: 22%

110 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for reading and writing composite objects.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("DatasetComponent", "StorageClassDelegate") 

33 

34import copy 

35import logging 

36from collections.abc import Iterable, Mapping 

37from dataclasses import dataclass 

38from typing import TYPE_CHECKING, Any 

39 

40from lsst.utils.introspection import get_full_type_name 

41 

42if TYPE_CHECKING: 

43 from ._storage_class import StorageClass 

44 

45log = logging.getLogger(__name__) 

46 

47 

48@dataclass 

49class DatasetComponent: 

50 """Component of a dataset and associated information.""" 

51 

52 name: str 

53 """Name of the component. 

54 """ 

55 

56 storageClass: StorageClass 

57 """StorageClass to be used when reading or writing this component. 

58 """ 

59 

60 component: Any 

61 """Component extracted from the composite object. 

62 """ 

63 

64 

65class StorageClassDelegate: 

66 """Delegate class for StorageClass components and parameters. 

67 

68 This class delegates the handling of components and parameters for the 

69 python type associated with a particular `StorageClass`. 

70 

71 A delegate is required for any storage class that defines components 

72 (derived or otherwise) or support read parameters. It is used for 

73 composite disassembly and assembly. 

74 

75 Attributes 

76 ---------- 

77 storageClass : `StorageClass` 

78 

79 Parameters 

80 ---------- 

81 storageClass : `StorageClass` 

82 `StorageClass` to be used with this delegate. 

83 """ 

84 

85 def __init__(self, storageClass: StorageClass): 

86 assert storageClass is not None 

87 self.storageClass = storageClass 

88 

89 @staticmethod 

90 def _attrNames(componentName: str, getter: bool = True) -> tuple[str, ...]: 

91 """Return list of suitable attribute names to attempt to use. 

92 

93 Parameters 

94 ---------- 

95 componentName : `str` 

96 Name of component/attribute to look for. 

97 getter : `bool` 

98 If true, return getters, else return setters. 

99 

100 Returns 

101 ------- 

102 attrs : `tuple(str)` 

103 Tuple of strings to attempt. 

104 """ 

105 root = "get" if getter else "set" 

106 

107 # Capitalized name for getXxx must only capitalize first letter and not 

108 # downcase the rest. getVisitInfo and not getVisitinfo 

109 first = componentName[0].upper() 

110 if len(componentName) > 1: 

111 tail = componentName[1:] 

112 else: 

113 tail = "" 

114 capitalized = f"{root}{first}{tail}" 

115 return (componentName, f"{root}_{componentName}", capitalized) 

116 

117 def assemble(self, components: dict[str, Any], pytype: type | None = None) -> Any: 

118 """Construct an object from components based on storageClass. 

119 

120 This generic implementation assumes that instances of objects 

121 can be created either by passing all the components to a constructor 

122 or by calling setter methods with the name. 

123 

124 Parameters 

125 ---------- 

126 components : `dict` 

127 Collection of components from which to assemble a new composite 

128 object. Keys correspond to composite names in the `StorageClass`. 

129 pytype : `type`, optional 

130 Override the type from the 

131 :attr:`StorageClassDelegate.storageClass` 

132 to use when assembling the final object. 

133 

134 Returns 

135 ------- 

136 composite : `object` 

137 New composite object assembled from components. 

138 

139 Raises 

140 ------ 

141 ValueError 

142 Some components could not be used to create the object or, 

143 alternatively, some components were not defined in the associated 

144 StorageClass. 

145 """ 

146 if pytype is not None: 

147 cls = pytype 

148 else: 

149 cls = self.storageClass.pytype 

150 

151 # Check that the storage class components are consistent 

152 understood = set(self.storageClass.components) 

153 requested = set(components.keys()) 

154 unknown = requested - understood 

155 if unknown: 

156 raise ValueError(f"Requested component(s) not known to StorageClass: {unknown}") 

157 

158 # First try to create an instance directly using keyword args 

159 try: 

160 obj = cls(**components) 

161 except TypeError: 

162 obj = None 

163 

164 # Now try to use setters if direct instantiation didn't work 

165 if not obj: 

166 obj = cls() 

167 

168 failed = [] 

169 for name, component in components.items(): 

170 if component is None: 

171 continue 

172 for attr in self._attrNames(name, getter=False): 

173 if hasattr(obj, attr): 

174 if attr == name: # Real attribute 

175 setattr(obj, attr, component) 

176 else: 

177 setter = getattr(obj, attr) 

178 setter(component) 

179 break 

180 else: 

181 failed.append(name) 

182 

183 if failed: 

184 raise ValueError(f"Unhandled components during assembly ({failed})") 

185 

186 return obj 

187 

188 def getComponent(self, composite: Any, componentName: str) -> Any: 

189 """Attempt to retrieve component from composite object by heuristic. 

190 

191 Will attempt a direct attribute retrieval, or else getter methods of 

192 the form "get_componentName" and "getComponentName". 

193 

194 Parameters 

195 ---------- 

196 composite : `object` 

197 Item to query for the component. 

198 componentName : `str` 

199 Name of component to retrieve. 

200 

201 Returns 

202 ------- 

203 component : `object` 

204 Component extracted from composite. 

205 

206 Raises 

207 ------ 

208 AttributeError 

209 The attribute could not be read from the composite. 

210 """ 

211 component = None 

212 

213 if hasattr(composite, "__contains__") and componentName in composite: 

214 component = composite[componentName] 

215 return component 

216 

217 for attr in self._attrNames(componentName, getter=True): 

218 if hasattr(composite, attr): 

219 component = getattr(composite, attr) 

220 if attr != componentName: # We have a method 

221 component = component() 

222 break 

223 else: 

224 raise AttributeError(f"Unable to get component {componentName}") 

225 return component 

226 

227 def disassemble( 

228 self, composite: Any, subset: Iterable | None = None, override: Any | None = None 

229 ) -> dict[str, DatasetComponent]: 

230 """Disassembler a composite. 

231 

232 This is a generic implementation of a disassembler. 

233 This implementation attempts to extract components from the parent 

234 by looking for attributes of the same name or getter methods derived 

235 from the component name. 

236 

237 Parameters 

238 ---------- 

239 composite : `object` 

240 Parent composite object consisting of components to be extracted. 

241 subset : iterable, optional 

242 Iterable containing subset of components to extract from composite. 

243 Must be a subset of those defined in 

244 `StorageClassDelegate.storageClass`. 

245 override : `object`, optional 

246 Object to use for disassembly instead of parent. This can be useful 

247 when called from subclasses that have composites in a hierarchy. 

248 

249 Returns 

250 ------- 

251 components : `dict` 

252 `dict` with keys matching the components defined in 

253 `StorageClassDelegate.storageClass` 

254 and values being `DatasetComponent` instances describing the 

255 component. 

256 

257 Raises 

258 ------ 

259 ValueError 

260 A requested component can not be found in the parent using generic 

261 lookups. 

262 TypeError 

263 The parent object does not match the supplied 

264 `StorageClassDelegate.storageClass`. 

265 """ 

266 if not self.storageClass.isComposite(): 

267 raise TypeError( 

268 f"Can not disassemble something that is not a composite (storage class={self.storageClass})" 

269 ) 

270 

271 if not self.storageClass.validateInstance(composite): 

272 raise TypeError( 

273 "Unexpected type mismatch between parent and StorageClass " 

274 f"({type(composite)} != {self.storageClass.pytype})" 

275 ) 

276 

277 requested = set(self.storageClass.components) 

278 

279 if subset is not None: 

280 subset = set(subset) 

281 diff = subset - requested 

282 if diff: 

283 raise ValueError(f"Requested subset is not a subset of supported components: {diff}") 

284 requested = subset 

285 

286 if override is not None: 

287 composite = override 

288 

289 components = {} 

290 for c in list(requested): 

291 # Try three different ways to get a value associated with the 

292 # component name. 

293 try: 

294 component = self.getComponent(composite, c) 

295 except AttributeError: 

296 # Defer complaining so we get an idea of how many problems 

297 # we have 

298 pass 

299 else: 

300 # If we found a match store it in the results dict and remove 

301 # it from the list of components we are still looking for. 

302 if component is not None: 

303 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

304 requested.remove(c) 

305 

306 if requested: 

307 raise ValueError(f"Unhandled components during disassembly ({requested})") 

308 

309 return components 

310 

311 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any: 

312 """Modify the in-memory dataset using the supplied parameters. 

313 

314 Can return a possibly new object. 

315 

316 For safety, if any parameters are given to this method an 

317 exception will be raised. This is to protect the user from 

318 thinking that parameters have been applied when they have not been 

319 applied. 

320 

321 Parameters 

322 ---------- 

323 inMemoryDataset : `object` 

324 Object to modify based on the parameters. 

325 parameters : `dict` 

326 Parameters to apply. Values are specific to the parameter. 

327 Supported parameters are defined in the associated 

328 `StorageClass`. If no relevant parameters are specified the 

329 inMemoryDataset will be return unchanged. 

330 

331 Returns 

332 ------- 

333 inMemoryDataset : `object` 

334 Updated form of supplied in-memory dataset, after parameters 

335 have been used. 

336 

337 Raises 

338 ------ 

339 ValueError 

340 Parameters have been provided to this default implementation. 

341 """ 

342 if parameters: 

343 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

344 

345 return inMemoryDataset 

346 

347 @classmethod 

348 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: set[str | None]) -> str: 

349 """Select the best component for calculating a derived component. 

350 

351 Given a possible set of components to choose from, return the 

352 component that should be used to calculate the requested derived 

353 component. 

354 

355 Parameters 

356 ---------- 

357 derivedComponent : `str` 

358 The derived component that is being requested. 

359 fromComponents : `set` of `str` 

360 The available set of component options from which that derived 

361 component can be computed. `None` can be included but should 

362 be ignored. 

363 

364 Returns 

365 ------- 

366 required : `str` 

367 The component that should be used. 

368 

369 Raises 

370 ------ 

371 NotImplementedError 

372 Raised if this delegate refuses to answer the question. 

373 ValueError 

374 Raised if this delegate can not determine a relevant component 

375 from the supplied options. 

376 """ 

377 raise NotImplementedError("This delegate does not support derived components") 

378 

379 def copy(self, inMemoryDataset: Any) -> Any: 

380 """Copy the supplied python type and return the copy. 

381 

382 Parameters 

383 ---------- 

384 inMemoryDataset : `object` 

385 Object to copy. 

386 

387 Returns 

388 ------- 

389 copied : `object` 

390 A copy of the supplied object. Can be the same object if the 

391 object is known to be read-only. 

392 

393 Raises 

394 ------ 

395 NotImplementedError 

396 Raised if none of the default methods for copying work. 

397 

398 Notes 

399 ----- 

400 The default implementation uses `copy.deepcopy()`. 

401 It is generally expected that this method is the equivalent of a deep 

402 copy. Subclasses can override this method if they already know the 

403 optimal approach for deep copying. 

404 """ 

405 try: 

406 return copy.deepcopy(inMemoryDataset) 

407 except Exception as e: 

408 raise NotImplementedError( 

409 f"Unable to deep copy the supplied python type ({get_full_type_name(inMemoryDataset)}) " 

410 f"using default methods ({e})" 

411 ) from e