Coverage for python/lsst/daf/butler/core/storageClassDelegate.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

118 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "StorageClassDelegate") 

27 

28import collections.abc 

29import logging 

30from dataclasses import dataclass 

31from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Optional, Set, Tuple, Type 

32 

33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 from .storageClass import StorageClass 

35 

36log = logging.getLogger(__name__) 

37 

38 

39@dataclass 

40class DatasetComponent: 

41 """Component of a dataset and associated information.""" 

42 

43 name: str 

44 """Name of the component. 

45 """ 

46 

47 storageClass: StorageClass 

48 """StorageClass to be used when reading or writing this component. 

49 """ 

50 

51 component: Any 

52 """Component extracted from the composite object. 

53 """ 

54 

55 

56class StorageClassDelegate: 

57 """Delegate class for StorageClass components and parameters. 

58 

59 This class delegates the handling of components and parameters for the 

60 python type associated with a particular `StorageClass`. 

61 

62 A delegate is required for any storage class that defines components 

63 (derived or otherwise) or support read parameters. It is used for 

64 composite disassembly and assembly. 

65 

66 Attributes 

67 ---------- 

68 storageClass : `StorageClass` 

69 

70 Parameters 

71 ---------- 

72 storageClass : `StorageClass` 

73 `StorageClass` to be used with this delegate. 

74 """ 

75 

76 def __init__(self, storageClass: StorageClass): 

77 assert storageClass is not None 

78 self.storageClass = storageClass 

79 

80 @staticmethod 

81 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

82 """Return list of suitable attribute names to attempt to use. 

83 

84 Parameters 

85 ---------- 

86 componentName : `str` 

87 Name of component/attribute to look for. 

88 getter : `bool` 

89 If true, return getters, else return setters. 

90 

91 Returns 

92 ------- 

93 attrs : `tuple(str)` 

94 Tuple of strings to attempt. 

95 """ 

96 root = "get" if getter else "set" 

97 

98 # Capitalized name for getXxx must only capitalize first letter and not 

99 # downcase the rest. getVisitInfo and not getVisitinfo 

100 first = componentName[0].upper() 

101 if len(componentName) > 1: 

102 tail = componentName[1:] 

103 else: 

104 tail = "" 

105 capitalized = "{}{}{}".format(root, first, tail) 

106 return (componentName, "{}_{}".format(root, componentName), capitalized) 

107 

108 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

109 """Construct an object from components based on storageClass. 

110 

111 This generic implementation assumes that instances of objects 

112 can be created either by passing all the components to a constructor 

113 or by calling setter methods with the name. 

114 

115 Parameters 

116 ---------- 

117 components : `dict` 

118 Collection of components from which to assemble a new composite 

119 object. Keys correspond to composite names in the `StorageClass`. 

120 pytype : `type`, optional 

121 Override the type from the 

122 :attr:`StorageClassDelegate.storageClass` 

123 to use when assembling the final object. 

124 

125 Returns 

126 ------- 

127 composite : `object` 

128 New composite object assembled from components. 

129 

130 Raises 

131 ------ 

132 ValueError 

133 Some components could not be used to create the object or, 

134 alternatively, some components were not defined in the associated 

135 StorageClass. 

136 """ 

137 if pytype is not None: 

138 cls = pytype 

139 else: 

140 cls = self.storageClass.pytype 

141 

142 # Check that the storage class components are consistent 

143 understood = set(self.storageClass.components) 

144 requested = set(components.keys()) 

145 unknown = requested - understood 

146 if unknown: 

147 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown)) 

148 

149 # First try to create an instance directly using keyword args 

150 try: 

151 obj = cls(**components) 

152 except TypeError: 

153 obj = None 

154 

155 # Now try to use setters if direct instantiation didn't work 

156 if not obj: 

157 obj = cls() 

158 

159 failed = [] 

160 for name, component in components.items(): 

161 if component is None: 

162 continue 

163 for attr in self._attrNames(name, getter=False): 

164 if hasattr(obj, attr): 

165 if attr == name: # Real attribute 

166 setattr(obj, attr, component) 

167 else: 

168 setter = getattr(obj, attr) 

169 setter(component) 

170 break 

171 else: 

172 failed.append(name) 

173 

174 if failed: 

175 raise ValueError("Unhandled components during assembly ({})".format(failed)) 

176 

177 return obj 

178 

179 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

180 """Extract all non-None components from a composite. 

181 

182 Parameters 

183 ---------- 

184 composite : `object` 

185 Composite from which to extract components. 

186 

187 Returns 

188 ------- 

189 comps : `dict` 

190 Non-None components extracted from the composite, indexed by the 

191 component name as derived from the 

192 `StorageClassDelegate.storageClass`. 

193 """ 

194 components = {} 

195 if self.storageClass.isComposite(): 

196 for c in self.storageClass.components: 

197 if isinstance(composite, collections.abc.Mapping): 

198 comp = composite[c] 

199 else: 

200 try: 

201 comp = self.getComponent(composite, c) 

202 except AttributeError: 

203 pass 

204 else: 

205 if comp is not None: 

206 components[c] = comp 

207 return components 

208 

209 def getComponent(self, composite: Any, componentName: str) -> Any: 

210 """Attempt to retrieve component from composite object by heuristic. 

211 

212 Will attempt a direct attribute retrieval, or else getter methods of 

213 the form "get_componentName" and "getComponentName". 

214 

215 Parameters 

216 ---------- 

217 composite : `object` 

218 Item to query for the component. 

219 componentName : `str` 

220 Name of component to retrieve. 

221 

222 Returns 

223 ------- 

224 component : `object` 

225 Component extracted from composite. 

226 

227 Raises 

228 ------ 

229 AttributeError 

230 The attribute could not be read from the composite. 

231 """ 

232 component = None 

233 

234 if hasattr(composite, "__contains__") and componentName in composite: 

235 component = composite[componentName] 

236 return component 

237 

238 for attr in self._attrNames(componentName, getter=True): 

239 if hasattr(composite, attr): 

240 component = getattr(composite, attr) 

241 if attr != componentName: # We have a method 

242 component = component() 

243 break 

244 else: 

245 raise AttributeError("Unable to get component {}".format(componentName)) 

246 return component 

247 

248 def disassemble( 

249 self, composite: Any, subset: Optional[Iterable] = None, override: bool = None 

250 ) -> Dict[str, Any]: 

251 """Disassembler a composite. 

252 

253 This is a generic implementation of a disassembler. 

254 This implementation attempts to extract components from the parent 

255 by looking for attributes of the same name or getter methods derived 

256 from the component name. 

257 

258 Parameters 

259 ---------- 

260 composite : `object` 

261 Parent composite object consisting of components to be extracted. 

262 subset : iterable, optional 

263 Iterable containing subset of components to extract from composite. 

264 Must be a subset of those defined in 

265 `StorageClassDelegate.storageClass`. 

266 override : `object`, optional 

267 Object to use for disassembly instead of parent. This can be useful 

268 when called from subclasses that have composites in a hierarchy. 

269 

270 Returns 

271 ------- 

272 components : `dict` 

273 `dict` with keys matching the components defined in 

274 `StorageClassDelegate.storageClass` 

275 and values being `DatasetComponent` instances describing the 

276 component. Returns None if this is not a composite 

277 `StorageClassDelegate.storageClass`. 

278 

279 Raises 

280 ------ 

281 ValueError 

282 A requested component can not be found in the parent using generic 

283 lookups. 

284 TypeError 

285 The parent object does not match the supplied 

286 `StorageClassDelegate.storageClass`. 

287 """ 

288 if not self.storageClass.isComposite(): 

289 raise TypeError( 

290 "Can not disassemble something that is not a composite" 

291 f" (storage class={self.storageClass})" 

292 ) 

293 

294 if not self.storageClass.validateInstance(composite): 

295 raise TypeError( 

296 "Unexpected type mismatch between parent and StorageClass" 

297 " ({} != {})".format(type(composite), self.storageClass.pytype) 

298 ) 

299 

300 requested = set(self.storageClass.components) 

301 

302 if subset is not None: 

303 subset = set(subset) 

304 diff = subset - requested 

305 if diff: 

306 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff)) 

307 requested = subset 

308 

309 if override is not None: 

310 composite = override 

311 

312 components = {} 

313 for c in list(requested): 

314 # Try three different ways to get a value associated with the 

315 # component name. 

316 try: 

317 component = self.getComponent(composite, c) 

318 except AttributeError: 

319 # Defer complaining so we get an idea of how many problems 

320 # we have 

321 pass 

322 else: 

323 # If we found a match store it in the results dict and remove 

324 # it from the list of components we are still looking for. 

325 if component is not None: 

326 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

327 requested.remove(c) 

328 

329 if requested: 

330 raise ValueError("Unhandled components during disassembly ({})".format(requested)) 

331 

332 return components 

333 

334 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

335 """Modify the in-memory dataset using the supplied parameters. 

336 

337 Can return a possibly new object. 

338 

339 For safety, if any parameters are given to this method an 

340 exception will be raised. This is to protect the user from 

341 thinking that parameters have been applied when they have not been 

342 applied. 

343 

344 Parameters 

345 ---------- 

346 inMemoryDataset : `object` 

347 Object to modify based on the parameters. 

348 parameters : `dict` 

349 Parameters to apply. Values are specific to the parameter. 

350 Supported parameters are defined in the associated 

351 `StorageClass`. If no relevant parameters are specified the 

352 inMemoryDataset will be return unchanged. 

353 

354 Returns 

355 ------- 

356 inMemoryDataset : `object` 

357 Updated form of supplied in-memory dataset, after parameters 

358 have been used. 

359 

360 Raises 

361 ------ 

362 ValueError 

363 Parameters have been provided to this default implementation. 

364 """ 

365 if parameters: 

366 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

367 

368 return inMemoryDataset 

369 

370 @classmethod 

371 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: Set[Optional[str]]) -> str: 

372 """Select the best component for calculating a derived component. 

373 

374 Given a possible set of components to choose from, return the 

375 component that should be used to calculate the requested derived 

376 component. 

377 

378 Parameters 

379 ---------- 

380 derivedComponent : `str` 

381 The derived component that is being requested. 

382 fromComponents : `set` of `str` 

383 The available set of component options from which that derived 

384 component can be computed. `None` can be included but should 

385 be ignored. 

386 

387 Returns 

388 ------- 

389 required : `str` 

390 The component that should be used. 

391 

392 Raises 

393 ------ 

394 NotImplementedError 

395 Raised if this delegate refuses to answer the question. 

396 ValueError 

397 Raised if this delegate can not determine a relevant component 

398 from the supplied options. 

399 """ 

400 raise NotImplementedError("This delegate does not support derived components")