Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "StorageClassDelegate") 

27 

28import collections 

29from dataclasses import dataclass 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Mapping, 

36 Optional, 

37 Set, 

38 Tuple, 

39 Type, 

40 TYPE_CHECKING, 

41) 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .storageClass import StorageClass 

45 

46log = logging.getLogger(__name__) 

47 

48 

49@dataclass 

50class DatasetComponent: 

51 """Component of a dataset and associated information. 

52 """ 

53 

54 name: str 

55 """Name of the component. 

56 """ 

57 

58 storageClass: StorageClass 

59 """StorageClass to be used when reading or writing this component. 

60 """ 

61 

62 component: Any 

63 """Component extracted from the composite object. 

64 """ 

65 

66 

67class StorageClassDelegate: 

68 """Class to delegate the handling of components and parameters for the 

69 python type associated with a particular `StorageClass`. 

70 

71 A delegate is required for any storage class that defines components 

72 (derived or otherwise) or support read parameters. It is used for 

73 composite disassembly and assembly. 

74 

75 Attributes 

76 ---------- 

77 storageClass : `StorageClass` 

78 

79 Parameters 

80 ---------- 

81 storageClass : `StorageClass` 

82 `StorageClass` to be used with this delegate. 

83 """ 

84 

85 def __init__(self, storageClass: StorageClass): 

86 assert storageClass is not None 

87 self.storageClass = storageClass 

88 

89 @staticmethod 

90 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

91 """Return list of suitable attribute names to attempt to use. 

92 

93 Parameters 

94 ---------- 

95 componentName : `str` 

96 Name of component/attribute to look for. 

97 getter : `bool` 

98 If true, return getters, else return setters. 

99 

100 Returns 

101 ------- 

102 attrs : `tuple(str)` 

103 Tuple of strings to attempt. 

104 """ 

105 root = "get" if getter else "set" 

106 

107 # Capitalized name for getXxx must only capitalize first letter and not 

108 # downcase the rest. getVisitInfo and not getVisitinfo 

109 first = componentName[0].upper() 

110 if len(componentName) > 1: 

111 tail = componentName[1:] 

112 else: 

113 tail = "" 

114 capitalized = "{}{}{}".format(root, first, tail) 

115 return (componentName, "{}_{}".format(root, componentName), capitalized) 

116 

117 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

118 """Construct an object from components based on storageClass. 

119 

120 This generic implementation assumes that instances of objects 

121 can be created either by passing all the components to a constructor 

122 or by calling setter methods with the name. 

123 

124 Parameters 

125 ---------- 

126 components : `dict` 

127 Collection of components from which to assemble a new composite 

128 object. Keys correspond to composite names in the `StorageClass`. 

129 pytype : `type`, optional 

130 Override the type from the 

131 :attr:`StorageClassDelegate.storageClass` 

132 to use when assembling the final object. 

133 

134 Returns 

135 ------- 

136 composite : `object` 

137 New composite object assembled from components. 

138 

139 Raises 

140 ------ 

141 ValueError 

142 Some components could not be used to create the object or, 

143 alternatively, some components were not defined in the associated 

144 StorageClass. 

145 """ 

146 if pytype is not None: 

147 cls = pytype 

148 else: 

149 cls = self.storageClass.pytype 

150 

151 # Check that the storage class components are consistent 

152 understood = set(self.storageClass.components) 

153 requested = set(components.keys()) 

154 unknown = requested - understood 

155 if unknown: 

156 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown)) 

157 

158 # First try to create an instance directly using keyword args 

159 try: 

160 obj = cls(**components) 

161 except TypeError: 

162 obj = None 

163 

164 # Now try to use setters if direct instantiation didn't work 

165 if not obj: 

166 obj = cls() 

167 

168 failed = [] 

169 for name, component in components.items(): 

170 if component is None: 

171 continue 

172 for attr in self._attrNames(name, getter=False): 

173 if hasattr(obj, attr): 

174 if attr == name: # Real attribute 

175 setattr(obj, attr, component) 

176 else: 

177 setter = getattr(obj, attr) 

178 setter(component) 

179 break 

180 else: 

181 failed.append(name) 

182 

183 if failed: 

184 raise ValueError("Unhandled components during assembly ({})".format(failed)) 

185 

186 return obj 

187 

188 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

189 """Extract all non-None components from a composite. 

190 

191 Parameters 

192 ---------- 

193 composite : `object` 

194 Composite from which to extract components. 

195 

196 Returns 

197 ------- 

198 comps : `dict` 

199 Non-None components extracted from the composite, indexed by the 

200 component name as derived from the 

201 `StorageClassDelegate.storageClass`. 

202 """ 

203 components = {} 

204 if self.storageClass.isComposite(): 

205 for c in self.storageClass.components: 

206 if isinstance(composite, collections.abc.Mapping): 

207 comp = composite[c] 

208 else: 

209 try: 

210 comp = self.getComponent(composite, c) 

211 except AttributeError: 

212 pass 

213 else: 

214 if comp is not None: 

215 components[c] = comp 

216 return components 

217 

218 def getComponent(self, composite: Any, componentName: str) -> Any: 

219 """Attempt to retrieve component from composite object by heuristic. 

220 

221 Will attempt a direct attribute retrieval, or else getter methods of 

222 the form "get_componentName" and "getComponentName". 

223 

224 Parameters 

225 ---------- 

226 composite : `object` 

227 Item to query for the component. 

228 componentName : `str` 

229 Name of component to retrieve. 

230 

231 Returns 

232 ------- 

233 component : `object` 

234 Component extracted from composite. 

235 

236 Raises 

237 ------ 

238 AttributeError 

239 The attribute could not be read from the composite. 

240 """ 

241 component = None 

242 

243 if hasattr(composite, "__contains__") and componentName in composite: 

244 component = composite[componentName] 

245 return component 

246 

247 for attr in self._attrNames(componentName, getter=True): 

248 if hasattr(composite, attr): 

249 component = getattr(composite, attr) 

250 if attr != componentName: # We have a method 

251 component = component() 

252 break 

253 else: 

254 raise AttributeError("Unable to get component {}".format(componentName)) 

255 return component 

256 

257 def disassemble(self, composite: Any, subset: Optional[Iterable] = None, 

258 override: bool = None) -> Dict[str, Any]: 

259 """Generic implementation of a disassembler. 

260 

261 This implementation attempts to extract components from the parent 

262 by looking for attributes of the same name or getter methods derived 

263 from the component name. 

264 

265 Parameters 

266 ---------- 

267 composite : `object` 

268 Parent composite object consisting of components to be extracted. 

269 subset : iterable, optional 

270 Iterable containing subset of components to extract from composite. 

271 Must be a subset of those defined in 

272 `StorageClassDelegate.storageClass`. 

273 override : `object`, optional 

274 Object to use for disassembly instead of parent. This can be useful 

275 when called from subclasses that have composites in a hierarchy. 

276 

277 Returns 

278 ------- 

279 components : `dict` 

280 `dict` with keys matching the components defined in 

281 `StorageClassDelegate.storageClass` 

282 and values being `DatasetComponent` instances describing the 

283 component. Returns None if this is not a composite 

284 `StorageClassDelegate.storageClass`. 

285 

286 Raises 

287 ------ 

288 ValueError 

289 A requested component can not be found in the parent using generic 

290 lookups. 

291 TypeError 

292 The parent object does not match the supplied 

293 `StorageClassDelegate.storageClass`. 

294 """ 

295 if not self.storageClass.isComposite(): 

296 raise TypeError("Can not disassemble something that is not a composite" 

297 f" (storage class={self.storageClass})") 

298 

299 if not self.storageClass.validateInstance(composite): 

300 raise TypeError("Unexpected type mismatch between parent and StorageClass" 

301 " ({} != {})".format(type(composite), self.storageClass.pytype)) 

302 

303 requested = set(self.storageClass.components) 

304 

305 if subset is not None: 

306 subset = set(subset) 

307 diff = subset - requested 

308 if diff: 

309 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff)) 

310 requested = subset 

311 

312 if override is not None: 

313 composite = override 

314 

315 components = {} 

316 for c in list(requested): 

317 # Try three different ways to get a value associated with the 

318 # component name. 

319 try: 

320 component = self.getComponent(composite, c) 

321 except AttributeError: 

322 # Defer complaining so we get an idea of how many problems 

323 # we have 

324 pass 

325 else: 

326 # If we found a match store it in the results dict and remove 

327 # it from the list of components we are still looking for. 

328 if component is not None: 

329 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

330 requested.remove(c) 

331 

332 if requested: 

333 raise ValueError("Unhandled components during disassembly ({})".format(requested)) 

334 

335 return components 

336 

337 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

338 """Modify the in-memory dataset using the supplied parameters, 

339 returning a possibly new object. 

340 

341 For safety, if any parameters are given to this method an 

342 exception will be raised. This is to protect the user from 

343 thinking that parameters have been applied when they have not been 

344 applied. 

345 

346 Parameters 

347 ---------- 

348 inMemoryDataset : `object` 

349 Object to modify based on the parameters. 

350 parameters : `dict` 

351 Parameters to apply. Values are specific to the parameter. 

352 Supported parameters are defined in the associated 

353 `StorageClass`. If no relevant parameters are specified the 

354 inMemoryDataset will be return unchanged. 

355 

356 Returns 

357 ------- 

358 inMemoryDataset : `object` 

359 Updated form of supplied in-memory dataset, after parameters 

360 have been used. 

361 

362 Raises 

363 ------ 

364 ValueError 

365 Parameters have been provided to this default implementation. 

366 """ 

367 if parameters: 

368 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

369 

370 return inMemoryDataset 

371 

372 @classmethod 

373 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: Set[Optional[str]]) -> str: 

374 """Given a possible set of components to choose from, return the 

375 component that should be used to calculate the requested derived 

376 component. 

377 

378 Parameters 

379 ---------- 

380 derivedComponent : `str` 

381 The derived component that is being requested. 

382 fromComponents : `set` of `str` 

383 The available set of component options from which that derived 

384 component can be computed. `None` can be included but should 

385 be ignored. 

386 

387 Returns 

388 ------- 

389 required : `str` 

390 The component that should be used. 

391 

392 Raises 

393 ------ 

394 NotImplementedError 

395 Raised if this delegate refuses to answer the question. 

396 ValueError 

397 Raised if this delegate can not determine a relevant component 

398 from the supplied options. 

399 """ 

400 raise NotImplementedError("This delegate does not support derived components")