Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "CompositeAssembler") 

27 

28import collections 

29from dataclasses import dataclass 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Mapping, 

36 Optional, 

37 Tuple, 

38 Type, 

39 TYPE_CHECKING, 

40) 

41 

42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43, because the condition on line 42 was never true

43 from .storageClass import StorageClass 

44 

45log = logging.getLogger(__name__) 

46 

47 

48@dataclass 

49class DatasetComponent: 

50 """Component of a dataset and associated information. 

51 """ 

52 

53 name: str 

54 """Name of the component. 

55 """ 

56 

57 storageClass: StorageClass 

58 """StorageClass to be used when reading or writing this component. 

59 """ 

60 

61 component: Any 

62 """Component extracted from the composite object. 

63 """ 

64 

65 

66class CompositeAssembler: 

67 """Class for providing assembler and disassembler support for composites. 

68 

69 Attributes 

70 ---------- 

71 storageClass : `StorageClass` 

72 

73 Parameters 

74 ---------- 

75 storageClass : `StorageClass` 

76 `StorageClass` to be used with this assembler. 

77 """ 

78 

79 def __init__(self, storageClass: StorageClass): 

80 assert storageClass is not None 

81 self.storageClass = storageClass 

82 

83 @staticmethod 

84 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

85 """Return list of suitable attribute names to attempt to use. 

86 

87 Parameters 

88 ---------- 

89 componentName : `str` 

90 Name of component/attribute to look for. 

91 getter : `bool` 

92 If true, return getters, else return setters. 

93 

94 Returns 

95 ------- 

96 attrs : `tuple(str)` 

97 Tuple of strings to attempt. 

98 """ 

99 root = "get" if getter else "set" 

100 

101 # Capitalized name for getXxx must only capitalize first letter and not 

102 # downcase the rest. getVisitInfo and not getVisitinfo 

103 first = componentName[0].upper() 

104 if len(componentName) > 1: 

105 tail = componentName[1:] 

106 else: 

107 tail = "" 

108 capitalized = "{}{}{}".format(root, first, tail) 

109 return (componentName, "{}_{}".format(root, componentName), capitalized) 

110 

111 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

112 """Construct an object from components based on storageClass. 

113 

114 This generic implementation assumes that instances of objects 

115 can be created either by passing all the components to a constructor 

116 or by calling setter methods with the name. 

117 

118 Parameters 

119 ---------- 

120 components : `dict` 

121 Collection of components from which to assemble a new composite 

122 object. Keys correspond to composite names in the `StorageClass`. 

123 pytype : `type`, optional 

124 Override the type from the :attr:`CompositeAssembler.storageClass` 

125 to use when assembling the final object. 

126 

127 Returns 

128 ------- 

129 composite : `object` 

130 New composite object assembled from components. 

131 

132 Raises 

133 ------ 

134 ValueError 

135 Some components could not be used to create the object or, 

136 alternatively, some components were not defined in the associated 

137 StorageClass. 

138 """ 

139 if pytype is not None: 

140 cls = pytype 

141 else: 

142 cls = self.storageClass.pytype 

143 

144 # Check that the storage class components are consistent 

145 understood = set(self.storageClass.components) 

146 requested = set(components.keys()) 

147 unknown = requested - understood 

148 if unknown: 

149 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown)) 

150 

151 # First try to create an instance directly using keyword args 

152 try: 

153 obj = cls(**components) 

154 except TypeError: 

155 obj = None 

156 

157 # Now try to use setters if direct instantiation didn't work 

158 if not obj: 

159 obj = cls() 

160 

161 failed = [] 

162 for name, component in components.items(): 

163 if component is None: 

164 continue 

165 for attr in self._attrNames(name, getter=False): 

166 if hasattr(obj, attr): 

167 if attr == name: # Real attribute 

168 setattr(obj, attr, component) 

169 else: 

170 setter = getattr(obj, attr) 

171 setter(component) 

172 break 

173 else: 

174 failed.append(name) 

175 

176 if failed: 

177 raise ValueError("Unhandled components during assembly ({})".format(failed)) 

178 

179 return obj 

180 

181 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

182 """Extract all non-None components from a composite. 

183 

184 Parameters 

185 ---------- 

186 composite : `object` 

187 Composite from which to extract components. 

188 

189 Returns 

190 ------- 

191 comps : `dict` 

192 Non-None components extracted from the composite, indexed by the 

193 component name as derived from the 

194 `CompositeAssembler.storageClass`. 

195 """ 

196 components = {} 

197 if self.storageClass.isComposite(): 

198 for c in self.storageClass.components: 

199 if isinstance(composite, collections.abc.Mapping): 

200 comp = composite[c] 

201 else: 

202 try: 

203 comp = self.getComponent(composite, c) 

204 except AttributeError: 

205 pass 

206 else: 

207 if comp is not None: 

208 components[c] = comp 

209 return components 

210 

211 def getComponent(self, composite: Any, componentName: str) -> Any: 

212 """Attempt to retrieve component from composite object by heuristic. 

213 

214 Will attempt a direct attribute retrieval, or else getter methods of 

215 the form "get_componentName" and "getComponentName". 

216 

217 Parameters 

218 ---------- 

219 composite : `object` 

220 Item to query for the component. 

221 componentName : `str` 

222 Name of component to retrieve. 

223 

224 Returns 

225 ------- 

226 component : `object` 

227 Component extracted from composite. 

228 

229 Raises 

230 ------ 

231 AttributeError 

232 The attribute could not be read from the composite. 

233 """ 

234 component = None 

235 

236 if hasattr(composite, "__contains__") and componentName in composite: 

237 component = composite[componentName] 

238 return component 

239 

240 for attr in self._attrNames(componentName, getter=True): 

241 if hasattr(composite, attr): 

242 component = getattr(composite, attr) 

243 if attr != componentName: # We have a method 

244 component = component() 

245 break 

246 else: 

247 raise AttributeError("Unable to get component {}".format(componentName)) 

248 return component 

249 

250 def disassemble(self, composite: Any, subset: Optional[Iterable] = None, 

251 override: bool = None) -> Dict[str, Any]: 

252 """Generic implementation of a disassembler. 

253 

254 This implementation attempts to extract components from the parent 

255 by looking for attributes of the same name or getter methods derived 

256 from the component name. 

257 

258 Parameters 

259 ---------- 

260 composite : `object` 

261 Parent composite object consisting of components to be extracted. 

262 subset : iterable, optional 

263 Iterable containing subset of components to extract from composite. 

264 Must be a subset of those defined in 

265 `CompositeAssembler.storageClass`. 

266 override : `object`, optional 

267 Object to use for disassembly instead of parent. This can be useful 

268 when called from subclasses that have composites in a hierarchy. 

269 

270 Returns 

271 ------- 

272 components : `dict` 

273 `dict` with keys matching the components defined in 

274 `CompositeAssembler.storageClass` 

275 and values being `DatasetComponent` instances describing the 

276 component. Returns None if this is not a composite 

277 `CompositeAssembler.storageClass`. 

278 

279 Raises 

280 ------ 

281 ValueError 

282 A requested component can not be found in the parent using generic 

283 lookups. 

284 TypeError 

285 The parent object does not match the supplied 

286 `CompositeAssembler.storageClass`. 

287 """ 

288 if not self.storageClass.isComposite(): 

289 raise TypeError("Can not disassemble something that is not a composite" 

290 f" (storage class={self.storageClass})") 

291 

292 if not self.storageClass.validateInstance(composite): 

293 raise TypeError("Unexpected type mismatch between parent and StorageClass" 

294 " ({} != {})".format(type(composite), self.storageClass.pytype)) 

295 

296 requested = set(self.storageClass.components) 

297 

298 if subset is not None: 

299 subset = set(subset) 

300 diff = subset - requested 

301 if diff: 

302 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff)) 

303 requested = subset 

304 

305 if override is not None: 

306 composite = override 

307 

308 components = {} 

309 for c in list(requested): 

310 # Try three different ways to get a value associated with the 

311 # component name. 

312 try: 

313 component = self.getComponent(composite, c) 

314 except AttributeError: 

315 # Defer complaining so we get an idea of how many problems 

316 # we have 

317 pass 

318 else: 

319 # If we found a match store it in the results dict and remove 

320 # it from the list of components we are still looking for. 

321 if component is not None: 

322 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

323 requested.remove(c) 

324 

325 if requested: 

326 raise ValueError("Unhandled components during disassembly ({})".format(requested)) 

327 

328 return components 

329 

330 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

331 """Modify the in-memory dataset using the supplied parameters, 

332 returning a possibly new object. 

333 

334 For safety, if any parameters are given to this method an 

335 exception will be raised. This is to protect the user from 

336 thinking that parameters have been applied when they have not been 

337 applied. 

338 

339 Parameters 

340 ---------- 

341 inMemoryDataset : `object` 

342 Object to modify based on the parameters. 

343 parameters : `dict` 

344 Parameters to apply. Values are specific to the parameter. 

345 Supported parameters are defined in the associated 

346 `StorageClass`. If no relevant parameters are specified the 

347 inMemoryDataset will be return unchanged. 

348 

349 Returns 

350 ------- 

351 inMemoryDataset : `object` 

352 Updated form of supplied in-memory dataset, after parameters 

353 have been used. 

354 

355 Raises 

356 ------ 

357 ValueError 

358 Parameters have been provided to this default implementation. 

359 """ 

360 if parameters: 

361 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

362 

363 return inMemoryDataset