Coverage for python/lsst/daf/butler/core/storageClassDelegate.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "StorageClassDelegate") 

27 

28import collections.abc 

29from dataclasses import dataclass 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Mapping, 

36 Optional, 

37 Set, 

38 Tuple, 

39 Type, 

40 TYPE_CHECKING, 

41) 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .storageClass import StorageClass 

45 

46log = logging.getLogger(__name__) 

47 

48 

49@dataclass 

50class DatasetComponent: 

51 """Component of a dataset and associated information.""" 

52 

53 name: str 

54 """Name of the component. 

55 """ 

56 

57 storageClass: StorageClass 

58 """StorageClass to be used when reading or writing this component. 

59 """ 

60 

61 component: Any 

62 """Component extracted from the composite object. 

63 """ 

64 

65 

66class StorageClassDelegate: 

67 """Delegate class for StorageClass components and parameters. 

68 

69 This class delegates the handling of components and parameters for the 

70 python type associated with a particular `StorageClass`. 

71 

72 A delegate is required for any storage class that defines components 

73 (derived or otherwise) or support read parameters. It is used for 

74 composite disassembly and assembly. 

75 

76 Attributes 

77 ---------- 

78 storageClass : `StorageClass` 

79 

80 Parameters 

81 ---------- 

82 storageClass : `StorageClass` 

83 `StorageClass` to be used with this delegate. 

84 """ 

85 

86 def __init__(self, storageClass: StorageClass): 

87 assert storageClass is not None 

88 self.storageClass = storageClass 

89 

90 @staticmethod 

91 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

92 """Return list of suitable attribute names to attempt to use. 

93 

94 Parameters 

95 ---------- 

96 componentName : `str` 

97 Name of component/attribute to look for. 

98 getter : `bool` 

99 If true, return getters, else return setters. 

100 

101 Returns 

102 ------- 

103 attrs : `tuple(str)` 

104 Tuple of strings to attempt. 

105 """ 

106 root = "get" if getter else "set" 

107 

108 # Capitalized name for getXxx must only capitalize first letter and not 

109 # downcase the rest. getVisitInfo and not getVisitinfo 

110 first = componentName[0].upper() 

111 if len(componentName) > 1: 

112 tail = componentName[1:] 

113 else: 

114 tail = "" 

115 capitalized = "{}{}{}".format(root, first, tail) 

116 return (componentName, "{}_{}".format(root, componentName), capitalized) 

117 

118 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

119 """Construct an object from components based on storageClass. 

120 

121 This generic implementation assumes that instances of objects 

122 can be created either by passing all the components to a constructor 

123 or by calling setter methods with the name. 

124 

125 Parameters 

126 ---------- 

127 components : `dict` 

128 Collection of components from which to assemble a new composite 

129 object. Keys correspond to composite names in the `StorageClass`. 

130 pytype : `type`, optional 

131 Override the type from the 

132 :attr:`StorageClassDelegate.storageClass` 

133 to use when assembling the final object. 

134 

135 Returns 

136 ------- 

137 composite : `object` 

138 New composite object assembled from components. 

139 

140 Raises 

141 ------ 

142 ValueError 

143 Some components could not be used to create the object or, 

144 alternatively, some components were not defined in the associated 

145 StorageClass. 

146 """ 

147 if pytype is not None: 

148 cls = pytype 

149 else: 

150 cls = self.storageClass.pytype 

151 

152 # Check that the storage class components are consistent 

153 understood = set(self.storageClass.components) 

154 requested = set(components.keys()) 

155 unknown = requested - understood 

156 if unknown: 

157 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown)) 

158 

159 # First try to create an instance directly using keyword args 

160 try: 

161 obj = cls(**components) 

162 except TypeError: 

163 obj = None 

164 

165 # Now try to use setters if direct instantiation didn't work 

166 if not obj: 

167 obj = cls() 

168 

169 failed = [] 

170 for name, component in components.items(): 

171 if component is None: 

172 continue 

173 for attr in self._attrNames(name, getter=False): 

174 if hasattr(obj, attr): 

175 if attr == name: # Real attribute 

176 setattr(obj, attr, component) 

177 else: 

178 setter = getattr(obj, attr) 

179 setter(component) 

180 break 

181 else: 

182 failed.append(name) 

183 

184 if failed: 

185 raise ValueError("Unhandled components during assembly ({})".format(failed)) 

186 

187 return obj 

188 

189 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

190 """Extract all non-None components from a composite. 

191 

192 Parameters 

193 ---------- 

194 composite : `object` 

195 Composite from which to extract components. 

196 

197 Returns 

198 ------- 

199 comps : `dict` 

200 Non-None components extracted from the composite, indexed by the 

201 component name as derived from the 

202 `StorageClassDelegate.storageClass`. 

203 """ 

204 components = {} 

205 if self.storageClass.isComposite(): 

206 for c in self.storageClass.components: 

207 if isinstance(composite, collections.abc.Mapping): 

208 comp = composite[c] 

209 else: 

210 try: 

211 comp = self.getComponent(composite, c) 

212 except AttributeError: 

213 pass 

214 else: 

215 if comp is not None: 

216 components[c] = comp 

217 return components 

218 

219 def getComponent(self, composite: Any, componentName: str) -> Any: 

220 """Attempt to retrieve component from composite object by heuristic. 

221 

222 Will attempt a direct attribute retrieval, or else getter methods of 

223 the form "get_componentName" and "getComponentName". 

224 

225 Parameters 

226 ---------- 

227 composite : `object` 

228 Item to query for the component. 

229 componentName : `str` 

230 Name of component to retrieve. 

231 

232 Returns 

233 ------- 

234 component : `object` 

235 Component extracted from composite. 

236 

237 Raises 

238 ------ 

239 AttributeError 

240 The attribute could not be read from the composite. 

241 """ 

242 component = None 

243 

244 if hasattr(composite, "__contains__") and componentName in composite: 

245 component = composite[componentName] 

246 return component 

247 

248 for attr in self._attrNames(componentName, getter=True): 

249 if hasattr(composite, attr): 

250 component = getattr(composite, attr) 

251 if attr != componentName: # We have a method 

252 component = component() 

253 break 

254 else: 

255 raise AttributeError("Unable to get component {}".format(componentName)) 

256 return component 

257 

258 def disassemble(self, composite: Any, subset: Optional[Iterable] = None, 

259 override: bool = None) -> Dict[str, Any]: 

260 """Disassembler a composite. 

261 

262 This is a generic implementation of a disassembler. 

263 This implementation attempts to extract components from the parent 

264 by looking for attributes of the same name or getter methods derived 

265 from the component name. 

266 

267 Parameters 

268 ---------- 

269 composite : `object` 

270 Parent composite object consisting of components to be extracted. 

271 subset : iterable, optional 

272 Iterable containing subset of components to extract from composite. 

273 Must be a subset of those defined in 

274 `StorageClassDelegate.storageClass`. 

275 override : `object`, optional 

276 Object to use for disassembly instead of parent. This can be useful 

277 when called from subclasses that have composites in a hierarchy. 

278 

279 Returns 

280 ------- 

281 components : `dict` 

282 `dict` with keys matching the components defined in 

283 `StorageClassDelegate.storageClass` 

284 and values being `DatasetComponent` instances describing the 

285 component. Returns None if this is not a composite 

286 `StorageClassDelegate.storageClass`. 

287 

288 Raises 

289 ------ 

290 ValueError 

291 A requested component can not be found in the parent using generic 

292 lookups. 

293 TypeError 

294 The parent object does not match the supplied 

295 `StorageClassDelegate.storageClass`. 

296 """ 

297 if not self.storageClass.isComposite(): 

298 raise TypeError("Can not disassemble something that is not a composite" 

299 f" (storage class={self.storageClass})") 

300 

301 if not self.storageClass.validateInstance(composite): 

302 raise TypeError("Unexpected type mismatch between parent and StorageClass" 

303 " ({} != {})".format(type(composite), self.storageClass.pytype)) 

304 

305 requested = set(self.storageClass.components) 

306 

307 if subset is not None: 

308 subset = set(subset) 

309 diff = subset - requested 

310 if diff: 

311 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff)) 

312 requested = subset 

313 

314 if override is not None: 

315 composite = override 

316 

317 components = {} 

318 for c in list(requested): 

319 # Try three different ways to get a value associated with the 

320 # component name. 

321 try: 

322 component = self.getComponent(composite, c) 

323 except AttributeError: 

324 # Defer complaining so we get an idea of how many problems 

325 # we have 

326 pass 

327 else: 

328 # If we found a match store it in the results dict and remove 

329 # it from the list of components we are still looking for. 

330 if component is not None: 

331 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

332 requested.remove(c) 

333 

334 if requested: 

335 raise ValueError("Unhandled components during disassembly ({})".format(requested)) 

336 

337 return components 

338 

339 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

340 """Modify the in-memory dataset using the supplied parameters. 

341 

342 Can return a possibly new object. 

343 

344 For safety, if any parameters are given to this method an 

345 exception will be raised. This is to protect the user from 

346 thinking that parameters have been applied when they have not been 

347 applied. 

348 

349 Parameters 

350 ---------- 

351 inMemoryDataset : `object` 

352 Object to modify based on the parameters. 

353 parameters : `dict` 

354 Parameters to apply. Values are specific to the parameter. 

355 Supported parameters are defined in the associated 

356 `StorageClass`. If no relevant parameters are specified the 

357 inMemoryDataset will be return unchanged. 

358 

359 Returns 

360 ------- 

361 inMemoryDataset : `object` 

362 Updated form of supplied in-memory dataset, after parameters 

363 have been used. 

364 

365 Raises 

366 ------ 

367 ValueError 

368 Parameters have been provided to this default implementation. 

369 """ 

370 if parameters: 

371 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

372 

373 return inMemoryDataset 

374 

375 @classmethod 

376 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: Set[Optional[str]]) -> str: 

377 """Select the best component for calculating a derived component. 

378 

379 Given a possible set of components to choose from, return the 

380 component that should be used to calculate the requested derived 

381 component. 

382 

383 Parameters 

384 ---------- 

385 derivedComponent : `str` 

386 The derived component that is being requested. 

387 fromComponents : `set` of `str` 

388 The available set of component options from which that derived 

389 component can be computed. `None` can be included but should 

390 be ignored. 

391 

392 Returns 

393 ------- 

394 required : `str` 

395 The component that should be used. 

396 

397 Raises 

398 ------ 

399 NotImplementedError 

400 Raised if this delegate refuses to answer the question. 

401 ValueError 

402 Raised if this delegate can not determine a relevant component 

403 from the supplied options. 

404 """ 

405 raise NotImplementedError("This delegate does not support derived components")