Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "CompositeAssembler") 

27 

28import collections 

29from dataclasses import dataclass 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Mapping, 

36 Optional, 

37 Set, 

38 Tuple, 

39 Type, 

40 TYPE_CHECKING, 

41) 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .storageClass import StorageClass 

45 

46log = logging.getLogger(__name__) 

47 

48 

49@dataclass 

50class DatasetComponent: 

51 """Component of a dataset and associated information. 

52 """ 

53 

54 name: str 

55 """Name of the component. 

56 """ 

57 

58 storageClass: StorageClass 

59 """StorageClass to be used when reading or writing this component. 

60 """ 

61 

62 component: Any 

63 """Component extracted from the composite object. 

64 """ 

65 

66 

67class CompositeAssembler: 

68 """Class for providing assembler and disassembler support for composites. 

69 

70 Attributes 

71 ---------- 

72 storageClass : `StorageClass` 

73 

74 Parameters 

75 ---------- 

76 storageClass : `StorageClass` 

77 `StorageClass` to be used with this assembler. 

78 """ 

79 

80 def __init__(self, storageClass: StorageClass): 

81 assert storageClass is not None 

82 self.storageClass = storageClass 

83 

84 @staticmethod 

85 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

86 """Return list of suitable attribute names to attempt to use. 

87 

88 Parameters 

89 ---------- 

90 componentName : `str` 

91 Name of component/attribute to look for. 

92 getter : `bool` 

93 If true, return getters, else return setters. 

94 

95 Returns 

96 ------- 

97 attrs : `tuple(str)` 

98 Tuple of strings to attempt. 

99 """ 

100 root = "get" if getter else "set" 

101 

102 # Capitalized name for getXxx must only capitalize first letter and not 

103 # downcase the rest. getVisitInfo and not getVisitinfo 

104 first = componentName[0].upper() 

105 if len(componentName) > 1: 

106 tail = componentName[1:] 

107 else: 

108 tail = "" 

109 capitalized = "{}{}{}".format(root, first, tail) 

110 return (componentName, "{}_{}".format(root, componentName), capitalized) 

111 

112 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

113 """Construct an object from components based on storageClass. 

114 

115 This generic implementation assumes that instances of objects 

116 can be created either by passing all the components to a constructor 

117 or by calling setter methods with the name. 

118 

119 Parameters 

120 ---------- 

121 components : `dict` 

122 Collection of components from which to assemble a new composite 

123 object. Keys correspond to composite names in the `StorageClass`. 

124 pytype : `type`, optional 

125 Override the type from the :attr:`CompositeAssembler.storageClass` 

126 to use when assembling the final object. 

127 

128 Returns 

129 ------- 

130 composite : `object` 

131 New composite object assembled from components. 

132 

133 Raises 

134 ------ 

135 ValueError 

136 Some components could not be used to create the object or, 

137 alternatively, some components were not defined in the associated 

138 StorageClass. 

139 """ 

140 if pytype is not None: 

141 cls = pytype 

142 else: 

143 cls = self.storageClass.pytype 

144 

145 # Check that the storage class components are consistent 

146 understood = set(self.storageClass.components) 

147 requested = set(components.keys()) 

148 unknown = requested - understood 

149 if unknown: 

150 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown)) 

151 

152 # First try to create an instance directly using keyword args 

153 try: 

154 obj = cls(**components) 

155 except TypeError: 

156 obj = None 

157 

158 # Now try to use setters if direct instantiation didn't work 

159 if not obj: 

160 obj = cls() 

161 

162 failed = [] 

163 for name, component in components.items(): 

164 if component is None: 

165 continue 

166 for attr in self._attrNames(name, getter=False): 

167 if hasattr(obj, attr): 

168 if attr == name: # Real attribute 

169 setattr(obj, attr, component) 

170 else: 

171 setter = getattr(obj, attr) 

172 setter(component) 

173 break 

174 else: 

175 failed.append(name) 

176 

177 if failed: 

178 raise ValueError("Unhandled components during assembly ({})".format(failed)) 

179 

180 return obj 

181 

182 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

183 """Extract all non-None components from a composite. 

184 

185 Parameters 

186 ---------- 

187 composite : `object` 

188 Composite from which to extract components. 

189 

190 Returns 

191 ------- 

192 comps : `dict` 

193 Non-None components extracted from the composite, indexed by the 

194 component name as derived from the 

195 `CompositeAssembler.storageClass`. 

196 """ 

197 components = {} 

198 if self.storageClass.isComposite(): 

199 for c in self.storageClass.components: 

200 if isinstance(composite, collections.abc.Mapping): 

201 comp = composite[c] 

202 else: 

203 try: 

204 comp = self.getComponent(composite, c) 

205 except AttributeError: 

206 pass 

207 else: 

208 if comp is not None: 

209 components[c] = comp 

210 return components 

211 

212 def getComponent(self, composite: Any, componentName: str) -> Any: 

213 """Attempt to retrieve component from composite object by heuristic. 

214 

215 Will attempt a direct attribute retrieval, or else getter methods of 

216 the form "get_componentName" and "getComponentName". 

217 

218 Parameters 

219 ---------- 

220 composite : `object` 

221 Item to query for the component. 

222 componentName : `str` 

223 Name of component to retrieve. 

224 

225 Returns 

226 ------- 

227 component : `object` 

228 Component extracted from composite. 

229 

230 Raises 

231 ------ 

232 AttributeError 

233 The attribute could not be read from the composite. 

234 """ 

235 component = None 

236 

237 if hasattr(composite, "__contains__") and componentName in composite: 

238 component = composite[componentName] 

239 return component 

240 

241 for attr in self._attrNames(componentName, getter=True): 

242 if hasattr(composite, attr): 

243 component = getattr(composite, attr) 

244 if attr != componentName: # We have a method 

245 component = component() 

246 break 

247 else: 

248 raise AttributeError("Unable to get component {}".format(componentName)) 

249 return component 

250 

251 def disassemble(self, composite: Any, subset: Optional[Iterable] = None, 

252 override: bool = None) -> Dict[str, Any]: 

253 """Generic implementation of a disassembler. 

254 

255 This implementation attempts to extract components from the parent 

256 by looking for attributes of the same name or getter methods derived 

257 from the component name. 

258 

259 Parameters 

260 ---------- 

261 composite : `object` 

262 Parent composite object consisting of components to be extracted. 

263 subset : iterable, optional 

264 Iterable containing subset of components to extract from composite. 

265 Must be a subset of those defined in 

266 `CompositeAssembler.storageClass`. 

267 override : `object`, optional 

268 Object to use for disassembly instead of parent. This can be useful 

269 when called from subclasses that have composites in a hierarchy. 

270 

271 Returns 

272 ------- 

273 components : `dict` 

274 `dict` with keys matching the components defined in 

275 `CompositeAssembler.storageClass` 

276 and values being `DatasetComponent` instances describing the 

277 component. Returns None if this is not a composite 

278 `CompositeAssembler.storageClass`. 

279 

280 Raises 

281 ------ 

282 ValueError 

283 A requested component can not be found in the parent using generic 

284 lookups. 

285 TypeError 

286 The parent object does not match the supplied 

287 `CompositeAssembler.storageClass`. 

288 """ 

289 if not self.storageClass.isComposite(): 

290 raise TypeError("Can not disassemble something that is not a composite" 

291 f" (storage class={self.storageClass})") 

292 

293 if not self.storageClass.validateInstance(composite): 

294 raise TypeError("Unexpected type mismatch between parent and StorageClass" 

295 " ({} != {})".format(type(composite), self.storageClass.pytype)) 

296 

297 requested = set(self.storageClass.components) 

298 

299 if subset is not None: 

300 subset = set(subset) 

301 diff = subset - requested 

302 if diff: 

303 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff)) 

304 requested = subset 

305 

306 if override is not None: 

307 composite = override 

308 

309 components = {} 

310 for c in list(requested): 

311 # Try three different ways to get a value associated with the 

312 # component name. 

313 try: 

314 component = self.getComponent(composite, c) 

315 except AttributeError: 

316 # Defer complaining so we get an idea of how many problems 

317 # we have 

318 pass 

319 else: 

320 # If we found a match store it in the results dict and remove 

321 # it from the list of components we are still looking for. 

322 if component is not None: 

323 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

324 requested.remove(c) 

325 

326 if requested: 

327 raise ValueError("Unhandled components during disassembly ({})".format(requested)) 

328 

329 return components 

330 

331 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

332 """Modify the in-memory dataset using the supplied parameters, 

333 returning a possibly new object. 

334 

335 For safety, if any parameters are given to this method an 

336 exception will be raised. This is to protect the user from 

337 thinking that parameters have been applied when they have not been 

338 applied. 

339 

340 Parameters 

341 ---------- 

342 inMemoryDataset : `object` 

343 Object to modify based on the parameters. 

344 parameters : `dict` 

345 Parameters to apply. Values are specific to the parameter. 

346 Supported parameters are defined in the associated 

347 `StorageClass`. If no relevant parameters are specified the 

348 inMemoryDataset will be return unchanged. 

349 

350 Returns 

351 ------- 

352 inMemoryDataset : `object` 

353 Updated form of supplied in-memory dataset, after parameters 

354 have been used. 

355 

356 Raises 

357 ------ 

358 ValueError 

359 Parameters have been provided to this default implementation. 

360 """ 

361 if parameters: 

362 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

363 

364 return inMemoryDataset 

365 

366 @classmethod 

367 def selectResponsibleComponent(cls, readComponent: str, fromComponents: Set[Optional[str]]) -> str: 

368 """Given a possible set of components to choose from, return the 

369 component that should be used to calculate the requested read 

370 component. 

371 

372 Parameters 

373 ---------- 

374 readComponent : `str` 

375 The component that is being requested. 

376 fromComponents : `set` of `str` 

377 The available set of component options from which that read 

378 component can be derived. `None` can be included but should 

379 be ignored. 

380 

381 Returns 

382 ------- 

383 required : `str` 

384 The component that should be used. 

385 

386 Raises 

387 ------ 

388 NotImplementedError 

389 Raised if this assembler refuses to answer the question. 

390 ValueError 

391 Raised if this assembler can not determine a relevant component 

392 from the supplied options. 

393 """ 

394 raise NotImplementedError("This assembler does not support read-only components")