Coverage for python/lsst/daf/butler/core/storageClassDelegate.py: 18%

123 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 02:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("DatasetComponent", "StorageClassDelegate") 

27 

28import collections.abc 

29import copy 

30import logging 

31from dataclasses import dataclass 

32from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Optional, Set, Tuple, Type 

33 

34from lsst.utils.introspection import get_full_type_name 

35 

36if TYPE_CHECKING: 

37 from .storageClass import StorageClass 

38 

39log = logging.getLogger(__name__) 

40 

41 

42@dataclass 

43class DatasetComponent: 

44 """Component of a dataset and associated information.""" 

45 

46 name: str 

47 """Name of the component. 

48 """ 

49 

50 storageClass: StorageClass 

51 """StorageClass to be used when reading or writing this component. 

52 """ 

53 

54 component: Any 

55 """Component extracted from the composite object. 

56 """ 

57 

58 

59class StorageClassDelegate: 

60 """Delegate class for StorageClass components and parameters. 

61 

62 This class delegates the handling of components and parameters for the 

63 python type associated with a particular `StorageClass`. 

64 

65 A delegate is required for any storage class that defines components 

66 (derived or otherwise) or support read parameters. It is used for 

67 composite disassembly and assembly. 

68 

69 Attributes 

70 ---------- 

71 storageClass : `StorageClass` 

72 

73 Parameters 

74 ---------- 

75 storageClass : `StorageClass` 

76 `StorageClass` to be used with this delegate. 

77 """ 

78 

79 def __init__(self, storageClass: StorageClass): 

80 assert storageClass is not None 

81 self.storageClass = storageClass 

82 

83 @staticmethod 

84 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]: 

85 """Return list of suitable attribute names to attempt to use. 

86 

87 Parameters 

88 ---------- 

89 componentName : `str` 

90 Name of component/attribute to look for. 

91 getter : `bool` 

92 If true, return getters, else return setters. 

93 

94 Returns 

95 ------- 

96 attrs : `tuple(str)` 

97 Tuple of strings to attempt. 

98 """ 

99 root = "get" if getter else "set" 

100 

101 # Capitalized name for getXxx must only capitalize first letter and not 

102 # downcase the rest. getVisitInfo and not getVisitinfo 

103 first = componentName[0].upper() 

104 if len(componentName) > 1: 

105 tail = componentName[1:] 

106 else: 

107 tail = "" 

108 capitalized = f"{root}{first}{tail}" 

109 return (componentName, f"{root}_{componentName}", capitalized) 

110 

111 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any: 

112 """Construct an object from components based on storageClass. 

113 

114 This generic implementation assumes that instances of objects 

115 can be created either by passing all the components to a constructor 

116 or by calling setter methods with the name. 

117 

118 Parameters 

119 ---------- 

120 components : `dict` 

121 Collection of components from which to assemble a new composite 

122 object. Keys correspond to composite names in the `StorageClass`. 

123 pytype : `type`, optional 

124 Override the type from the 

125 :attr:`StorageClassDelegate.storageClass` 

126 to use when assembling the final object. 

127 

128 Returns 

129 ------- 

130 composite : `object` 

131 New composite object assembled from components. 

132 

133 Raises 

134 ------ 

135 ValueError 

136 Some components could not be used to create the object or, 

137 alternatively, some components were not defined in the associated 

138 StorageClass. 

139 """ 

140 if pytype is not None: 

141 cls = pytype 

142 else: 

143 cls = self.storageClass.pytype 

144 

145 # Check that the storage class components are consistent 

146 understood = set(self.storageClass.components) 

147 requested = set(components.keys()) 

148 unknown = requested - understood 

149 if unknown: 

150 raise ValueError(f"Requested component(s) not known to StorageClass: {unknown}") 

151 

152 # First try to create an instance directly using keyword args 

153 try: 

154 obj = cls(**components) 

155 except TypeError: 

156 obj = None 

157 

158 # Now try to use setters if direct instantiation didn't work 

159 if not obj: 

160 obj = cls() 

161 

162 failed = [] 

163 for name, component in components.items(): 

164 if component is None: 

165 continue 

166 for attr in self._attrNames(name, getter=False): 

167 if hasattr(obj, attr): 

168 if attr == name: # Real attribute 

169 setattr(obj, attr, component) 

170 else: 

171 setter = getattr(obj, attr) 

172 setter(component) 

173 break 

174 else: 

175 failed.append(name) 

176 

177 if failed: 

178 raise ValueError(f"Unhandled components during assembly ({failed})") 

179 

180 return obj 

181 

182 def getValidComponents(self, composite: Any) -> Dict[str, Any]: 

183 """Extract all non-None components from a composite. 

184 

185 Parameters 

186 ---------- 

187 composite : `object` 

188 Composite from which to extract components. 

189 

190 Returns 

191 ------- 

192 comps : `dict` 

193 Non-None components extracted from the composite, indexed by the 

194 component name as derived from the 

195 `StorageClassDelegate.storageClass`. 

196 """ 

197 components = {} 

198 if self.storageClass.isComposite(): 

199 for c in self.storageClass.components: 

200 if isinstance(composite, collections.abc.Mapping): 

201 comp = composite[c] 

202 else: 

203 try: 

204 comp = self.getComponent(composite, c) 

205 except AttributeError: 

206 pass 

207 else: 

208 if comp is not None: 

209 components[c] = comp 

210 return components 

211 

212 def getComponent(self, composite: Any, componentName: str) -> Any: 

213 """Attempt to retrieve component from composite object by heuristic. 

214 

215 Will attempt a direct attribute retrieval, or else getter methods of 

216 the form "get_componentName" and "getComponentName". 

217 

218 Parameters 

219 ---------- 

220 composite : `object` 

221 Item to query for the component. 

222 componentName : `str` 

223 Name of component to retrieve. 

224 

225 Returns 

226 ------- 

227 component : `object` 

228 Component extracted from composite. 

229 

230 Raises 

231 ------ 

232 AttributeError 

233 The attribute could not be read from the composite. 

234 """ 

235 component = None 

236 

237 if hasattr(composite, "__contains__") and componentName in composite: 

238 component = composite[componentName] 

239 return component 

240 

241 for attr in self._attrNames(componentName, getter=True): 

242 if hasattr(composite, attr): 

243 component = getattr(composite, attr) 

244 if attr != componentName: # We have a method 

245 component = component() 

246 break 

247 else: 

248 raise AttributeError(f"Unable to get component {componentName}") 

249 return component 

250 

251 def disassemble( 

252 self, composite: Any, subset: Optional[Iterable] = None, override: Optional[Any] = None 

253 ) -> Dict[str, DatasetComponent]: 

254 """Disassembler a composite. 

255 

256 This is a generic implementation of a disassembler. 

257 This implementation attempts to extract components from the parent 

258 by looking for attributes of the same name or getter methods derived 

259 from the component name. 

260 

261 Parameters 

262 ---------- 

263 composite : `object` 

264 Parent composite object consisting of components to be extracted. 

265 subset : iterable, optional 

266 Iterable containing subset of components to extract from composite. 

267 Must be a subset of those defined in 

268 `StorageClassDelegate.storageClass`. 

269 override : `object`, optional 

270 Object to use for disassembly instead of parent. This can be useful 

271 when called from subclasses that have composites in a hierarchy. 

272 

273 Returns 

274 ------- 

275 components : `dict` 

276 `dict` with keys matching the components defined in 

277 `StorageClassDelegate.storageClass` 

278 and values being `DatasetComponent` instances describing the 

279 component. 

280 

281 Raises 

282 ------ 

283 ValueError 

284 A requested component can not be found in the parent using generic 

285 lookups. 

286 TypeError 

287 The parent object does not match the supplied 

288 `StorageClassDelegate.storageClass`. 

289 """ 

290 if not self.storageClass.isComposite(): 

291 raise TypeError( 

292 f"Can not disassemble something that is not a composite (storage class={self.storageClass})" 

293 ) 

294 

295 if not self.storageClass.validateInstance(composite): 

296 raise TypeError( 

297 "Unexpected type mismatch between parent and StorageClass " 

298 f"({type(composite)} != {self.storageClass.pytype})" 

299 ) 

300 

301 requested = set(self.storageClass.components) 

302 

303 if subset is not None: 

304 subset = set(subset) 

305 diff = subset - requested 

306 if diff: 

307 raise ValueError(f"Requested subset is not a subset of supported components: {diff}") 

308 requested = subset 

309 

310 if override is not None: 

311 composite = override 

312 

313 components = {} 

314 for c in list(requested): 

315 # Try three different ways to get a value associated with the 

316 # component name. 

317 try: 

318 component = self.getComponent(composite, c) 

319 except AttributeError: 

320 # Defer complaining so we get an idea of how many problems 

321 # we have 

322 pass 

323 else: 

324 # If we found a match store it in the results dict and remove 

325 # it from the list of components we are still looking for. 

326 if component is not None: 

327 components[c] = DatasetComponent(c, self.storageClass.components[c], component) 

328 requested.remove(c) 

329 

330 if requested: 

331 raise ValueError(f"Unhandled components during disassembly ({requested})") 

332 

333 return components 

334 

335 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any: 

336 """Modify the in-memory dataset using the supplied parameters. 

337 

338 Can return a possibly new object. 

339 

340 For safety, if any parameters are given to this method an 

341 exception will be raised. This is to protect the user from 

342 thinking that parameters have been applied when they have not been 

343 applied. 

344 

345 Parameters 

346 ---------- 

347 inMemoryDataset : `object` 

348 Object to modify based on the parameters. 

349 parameters : `dict` 

350 Parameters to apply. Values are specific to the parameter. 

351 Supported parameters are defined in the associated 

352 `StorageClass`. If no relevant parameters are specified the 

353 inMemoryDataset will be return unchanged. 

354 

355 Returns 

356 ------- 

357 inMemoryDataset : `object` 

358 Updated form of supplied in-memory dataset, after parameters 

359 have been used. 

360 

361 Raises 

362 ------ 

363 ValueError 

364 Parameters have been provided to this default implementation. 

365 """ 

366 if parameters: 

367 raise ValueError(f"Parameters ({parameters}) provided to default implementation.") 

368 

369 return inMemoryDataset 

370 

371 @classmethod 

372 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: Set[Optional[str]]) -> str: 

373 """Select the best component for calculating a derived component. 

374 

375 Given a possible set of components to choose from, return the 

376 component that should be used to calculate the requested derived 

377 component. 

378 

379 Parameters 

380 ---------- 

381 derivedComponent : `str` 

382 The derived component that is being requested. 

383 fromComponents : `set` of `str` 

384 The available set of component options from which that derived 

385 component can be computed. `None` can be included but should 

386 be ignored. 

387 

388 Returns 

389 ------- 

390 required : `str` 

391 The component that should be used. 

392 

393 Raises 

394 ------ 

395 NotImplementedError 

396 Raised if this delegate refuses to answer the question. 

397 ValueError 

398 Raised if this delegate can not determine a relevant component 

399 from the supplied options. 

400 """ 

401 raise NotImplementedError("This delegate does not support derived components") 

402 

403 def copy(self, inMemoryDataset: Any) -> Any: 

404 """Copy the supplied python type and return the copy. 

405 

406 Parameters 

407 ---------- 

408 inMemoryDataset : `object` 

409 Object to copy. 

410 

411 Returns 

412 ------- 

413 copied : `object` 

414 A copy of the supplied object. Can be the same object if the 

415 object is known to be read-only. 

416 

417 Raises 

418 ------ 

419 NotImplementedError 

420 Raised if none of the default methods for copying work. 

421 

422 Notes 

423 ----- 

424 The default implementation uses `copy.deepcopy()`. 

425 It is generally expected that this method is the equivalent of a deep 

426 copy. Subclasses can override this method if they already know the 

427 optimal approach for deep copying. 

428 """ 

429 

430 try: 

431 return copy.deepcopy(inMemoryDataset) 

432 except Exception as e: 

433 raise NotImplementedError( 

434 f"Unable to deep copy the supplied python type ({get_full_type_name(inMemoryDataset)}) " 

435 f"using default methods ({e})" 

436 )