Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Support for Storage Classes.""" 

25 

26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

27 

28import builtins 

29import logging 

30 

31from typing import ( 

32 Any, 

33 Collection, 

34 Dict, 

35 List, 

36 Optional, 

37 Set, 

38 Sequence, 

39 Tuple, 

40 Type, 

41 Union, 

42) 

43 

44from lsst.utils import doImport 

45from .utils import Singleton, getFullTypeName 

46from .assembler import CompositeAssembler 

47from .config import ConfigSubset, Config 

48from .configSupport import LookupKey 

49 

50log = logging.getLogger(__name__) 

51 

52 

53class StorageClassConfig(ConfigSubset): 

54 component = "storageClasses" 

55 defaultConfigFile = "storageClasses.yaml" 

56 

57 

58class StorageClass: 

59 """Class describing how a label maps to a particular Python type. 

60 

61 Parameters 

62 ---------- 

63 name : `str` 

64 Name to use for this class. 

65 pytype : `type` or `str` 

66 Python type (or name of type) to associate with the `StorageClass` 

67 components : `dict`, optional 

68 `dict` mapping name of a component to another `StorageClass`. 

69 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

70 Parameters understood by this `StorageClass`. 

71 assembler : `str`, optional 

72 Fully qualified name of class supporting assembly and disassembly 

73 of a `pytype` instance. 

74 """ 

75 _cls_name: str = "BaseStorageClass" 

76 _cls_components: Optional[Dict[str, StorageClass]] = None 

77 _cls_parameters: Optional[Union[Set[str], Sequence[str]]] = None 

78 _cls_assembler: Optional[str] = None 

79 _cls_pytype: Optional[Union[Type, str]] = None 

80 defaultAssembler: Type = CompositeAssembler 

81 defaultAssemblerName: str = getFullTypeName(defaultAssembler) 

82 

83 def __init__(self, name: Optional[str] = None, 

84 pytype: Optional[Union[Type, str]] = None, 

85 components: Optional[Dict[str, StorageClass]] = None, 

86 parameters: Optional[Union[Sequence, Set]] = None, 

87 assembler: Optional[str] = None): 

88 if name is None: 

89 name = self._cls_name 

90 if pytype is None: 

91 pytype = self._cls_pytype 

92 if components is None: 

93 components = self._cls_components 

94 if parameters is None: 

95 parameters = self._cls_parameters 

96 if assembler is None: 

97 assembler = self._cls_assembler 

98 self.name = name 

99 

100 if pytype is None: 

101 pytype = object 

102 

103 self._pytype: Optional[Type] 

104 if not isinstance(pytype, str): 

105 # Already have a type so store it and get the name 

106 self._pytypeName = getFullTypeName(pytype) 

107 self._pytype = pytype 

108 else: 

109 # Store the type name and defer loading of type 

110 self._pytypeName = pytype 

111 self._pytype = None 

112 

113 self._components = components if components is not None else {} 

114 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

115 # if the assembler is not None also set it and clear the default 

116 # assembler 

117 self._assembler: Optional[Type] 

118 self._assemblerClassName: Optional[str] 

119 if assembler is not None: 

120 self._assemblerClassName = assembler 

121 self._assembler = None 

122 elif components is not None: 

123 # We set a default assembler for composites so that a class is 

124 # guaranteed to support something if it is a composite. 

125 log.debug("Setting default assembler for %s", self.name) 

126 self._assembler = self.defaultAssembler 

127 self._assemblerClassName = self.defaultAssemblerName 

128 else: 

129 self._assembler = None 

130 self._assemblerClassName = None 

131 

132 @property 

133 def components(self) -> Dict[str, StorageClass]: 

134 """Component names mapped to associated `StorageClass` 

135 """ 

136 return self._components 

137 

138 @property 

139 def parameters(self) -> Set[str]: 

140 """`set` of names of parameters supported by this `StorageClass` 

141 """ 

142 return set(self._parameters) 

143 

144 @property 

145 def pytype(self) -> Type: 

146 """Python type associated with this `StorageClass`.""" 

147 if self._pytype is not None: 

148 return self._pytype 

149 

150 if hasattr(builtins, self._pytypeName): 

151 pytype = getattr(builtins, self._pytypeName) 

152 else: 

153 pytype = doImport(self._pytypeName) 

154 self._pytype = pytype 

155 return self._pytype 

156 

157 @property 

158 def assemblerClass(self) -> Optional[Type]: 

159 """Class to use to (dis)assemble an object from components.""" 

160 if self._assembler is not None: 

161 return self._assembler 

162 if self._assemblerClassName is None: 

163 return None 

164 self._assembler = doImport(self._assemblerClassName) 

165 return self._assembler 

166 

167 def assembler(self) -> CompositeAssembler: 

168 """Return an instance of an assembler. 

169 

170 Returns 

171 ------- 

172 assembler : `CompositeAssembler` 

173 Instance of the assembler associated with this `StorageClass`. 

174 Assembler is constructed with this `StorageClass`. 

175 

176 Raises 

177 ------ 

178 TypeError 

179 This StorageClass has no associated assembler. 

180 """ 

181 cls = self.assemblerClass 

182 if cls is None: 

183 raise TypeError(f"No assembler class is associated with StorageClass {self.name}") 

184 return cls(storageClass=self) 

185 

186 def isComposite(self) -> bool: 

187 """Boolean indicating whether this `StorageClass` is a composite 

188 or not. 

189 

190 Returns 

191 ------- 

192 isComposite : `bool` 

193 `True` if this `StorageClass` is a composite, `False` 

194 otherwise. 

195 """ 

196 if self.components: 

197 return True 

198 return False 

199 

200 def _lookupNames(self) -> Tuple[LookupKey, ...]: 

201 """Keys to use when looking up this DatasetRef in a configuration. 

202 

203 The names are returned in order of priority. 

204 

205 Returns 

206 ------- 

207 names : `tuple` of `LookupKey` 

208 Tuple of a `LookupKey` using the `StorageClass` name. 

209 """ 

210 return (LookupKey(name=self.name), ) 

211 

212 def knownParameters(self) -> Set[str]: 

213 """Return set of all parameters known to this `StorageClass` 

214 

215 The set includes parameters understood by components of a composite. 

216 

217 Returns 

218 ------- 

219 known : `set` 

220 All parameter keys of this `StorageClass` and the component 

221 storage classes. 

222 """ 

223 known = set(self._parameters) 

224 for sc in self.components.values(): 

225 known.update(sc.knownParameters()) 

226 return known 

227 

228 def validateParameters(self, parameters: Collection = None) -> None: 

229 """Check that the parameters are known to this `StorageClass` 

230 

231 Does not check the values. 

232 

233 Parameters 

234 ---------- 

235 parameters : `~collections.abc.Collection`, optional 

236 Collection containing the parameters. Can be `dict`-like or 

237 `set`-like. The parameter values are not checked. 

238 If no parameters are supplied, always returns without error. 

239 

240 Raises 

241 ------ 

242 KeyError 

243 Some parameters are not understood by this `StorageClass`. 

244 """ 

245 # No parameters is always okay 

246 if not parameters: 

247 return 

248 

249 # Extract the important information into a set. Works for dict and 

250 # list. 

251 external = set(parameters) 

252 

253 diff = external - self.knownParameters() 

254 if diff: 

255 s = "s" if len(diff) > 1 else "" 

256 unknown = '\', \''.join(diff) 

257 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

258 

259 def filterParameters(self, parameters: Dict[str, Any], 

260 subset: Collection = None) -> Dict[str, Any]: 

261 """Filter out parameters that are not known to this StorageClass 

262 

263 Parameters 

264 ---------- 

265 parameters : `dict`, optional 

266 Candidate parameters. Can be `None` if no parameters have 

267 been provided. 

268 subset : `~collections.abc.Collection`, optional 

269 Subset of supported parameters that the caller is interested 

270 in using. The subset must be known to the `StorageClass` 

271 if specified. 

272 

273 Returns 

274 ------- 

275 filtered : `dict` 

276 Valid parameters. Empty `dict` if none are suitable. 

277 """ 

278 if not parameters: 

279 return {} 

280 subset = set(subset) if subset is not None else set() 

281 known = self.knownParameters() 

282 if not subset.issubset(known): 

283 raise ValueError(f"Requested subset ({subset}) is not a subset of" 

284 f" known parameters ({known})") 

285 return {k: parameters[k] for k in known if k in parameters} 

286 

287 def validateInstance(self, instance: Any) -> bool: 

288 """Check that the supplied Python object has the expected Python type 

289 

290 Parameters 

291 ---------- 

292 instance : `object` 

293 Object to check. 

294 

295 Returns 

296 ------- 

297 isOk : `bool` 

298 True if the supplied instance object can be handled by this 

299 `StorageClass`, False otherwise. 

300 """ 

301 return isinstance(instance, self.pytype) 

302 

303 def __eq__(self, other: Any) -> bool: 

304 """Equality checks name, pytype name, assembler name, and components""" 

305 

306 if not isinstance(other, StorageClass): 

307 return False 

308 

309 if self.name != other.name: 

310 return False 

311 

312 # We must compare pytype and assembler by name since we do not want 

313 # to trigger an import of external module code here 

314 if self._assemblerClassName != other._assemblerClassName: 

315 return False 

316 if self._pytypeName != other._pytypeName: 

317 return False 

318 

319 # Ensure we have the same component keys in each 

320 if set(self.components.keys()) != set(other.components.keys()): 

321 return False 

322 

323 # Same parameters 

324 if self.parameters != other.parameters: 

325 return False 

326 

327 # Ensure that all the components have the same type 

328 for k in self.components: 

329 if self.components[k] != other.components[k]: 

330 return False 

331 

332 # If we got to this point everything checks out 

333 return True 

334 

335 def __hash__(self) -> int: 

336 return hash(self.name) 

337 

338 def __repr__(self) -> str: 

339 optionals: Dict[str, Any] = {} 

340 if self._pytypeName != "object": 

341 optionals["pytype"] = self._pytypeName 

342 if self._assemblerClassName is not None: 

343 optionals["assembler"] = self._assemblerClassName 

344 if self._parameters: 

345 optionals["parameters"] = self._parameters 

346 if self.components: 

347 optionals["components"] = self.components 

348 

349 # order is preserved in the dict 

350 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

351 

352 # Start with mandatory fields 

353 r = f"{self.__class__.__name__}({self.name!r}" 

354 if options: 

355 r = r + ", " + options 

356 r = r + ")" 

357 return r 

358 

359 def __str__(self) -> str: 

360 return self.name 

361 

362 

363class StorageClassFactory(metaclass=Singleton): 

364 """Factory for `StorageClass` instances. 

365 

366 This class is a singleton, with each instance sharing the pool of 

367 StorageClasses. Since code can not know whether it is the first 

368 time the instance has been created, the constructor takes no arguments. 

369 To populate the factory with storage classes, a call to 

370 `~StorageClassFactory.addFromConfig()` should be made. 

371 

372 Parameters 

373 ---------- 

374 config : `StorageClassConfig` or `str`, optional 

375 Load configuration. In a ButlerConfig` the relevant configuration 

376 is located in the ``storageClasses`` section. 

377 """ 

378 

379 def __init__(self, config: Optional[Union[StorageClassConfig, str]] = None): 

380 self._storageClasses: Dict[str, StorageClass] = {} 

381 self._configs: List[StorageClassConfig] = [] 

382 

383 # Always seed with the default config 

384 self.addFromConfig(StorageClassConfig()) 

385 

386 if config is not None: 

387 self.addFromConfig(config) 

388 

389 def __str__(self) -> str: 

390 """Return summary of factory. 

391 

392 Returns 

393 ------- 

394 summary : `str` 

395 Summary of the factory status. 

396 """ 

397 sep = "\n" 

398 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

399 

400StorageClasses 

401-------------- 

402{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

403""" 

404 

405 def __contains__(self, storageClassOrName: Union[StorageClass, str]) -> bool: 

406 """Indicates whether the storage class exists in the factory. 

407 

408 Parameters 

409 ---------- 

410 storageClassOrName : `str` or `StorageClass` 

411 If `str` is given existence of the named StorageClass 

412 in the factory is checked. If `StorageClass` is given 

413 existence and equality are checked. 

414 

415 Returns 

416 ------- 

417 in : `bool` 

418 True if the supplied string is present, or if the supplied 

419 `StorageClass` is present and identical. 

420 

421 Notes 

422 ----- 

423 The two different checks (one for "key" and one for "value") based on 

424 the type of the given argument mean that it is possible for 

425 StorageClass.name to be in the factory but StorageClass to not be 

426 in the factory. 

427 """ 

428 if isinstance(storageClassOrName, str): 

429 return storageClassOrName in self._storageClasses 

430 elif isinstance(storageClassOrName, StorageClass): 

431 if storageClassOrName.name in self._storageClasses: 

432 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

433 return False 

434 

435 def addFromConfig(self, config: Union[StorageClassConfig, Config, str]) -> None: 

436 """Add more `StorageClass` definitions from a config file. 

437 

438 Parameters 

439 ---------- 

440 config : `StorageClassConfig`, `Config` or `str` 

441 Storage class configuration. Can contain a ``storageClasses`` 

442 key if part of a global configuration. 

443 """ 

444 sconfig = StorageClassConfig(config) 

445 self._configs.append(sconfig) 

446 

447 # Since we can not assume that we will get definitions of 

448 # components or parents before their classes are defined 

449 # we have a helper function that we can call recursively 

450 # to extract definitions from the configuration. 

451 def processStorageClass(name: str, sconfig: StorageClassConfig) -> None: 

452 # Maybe we've already processed this through recursion 

453 if name not in sconfig: 

454 return 

455 info = sconfig.pop(name) 

456 

457 # Always create the storage class so we can ensure that 

458 # we are not trying to overwrite with a different definition 

459 components = None 

460 if "components" in info: 

461 components = {} 

462 for cname, ctype in info["components"].items(): 

463 if ctype not in self: 

464 processStorageClass(ctype, sconfig) 

465 components[cname] = self.getStorageClass(ctype) 

466 

467 # Extract scalar items from dict that are needed for 

468 # StorageClass Constructor 

469 storageClassKwargs = {k: info[k] for k in ("pytype", "assembler", "parameters") if k in info} 

470 

471 # Fill in other items 

472 storageClassKwargs["components"] = components 

473 

474 # Create the new storage class and register it 

475 baseClass = None 

476 if "inheritsFrom" in info: 

477 baseName = info["inheritsFrom"] 

478 if baseName not in self: 

479 processStorageClass(baseName, sconfig) 

480 baseClass = type(self.getStorageClass(baseName)) 

481 

482 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

483 newStorageClass = newStorageClassType() 

484 self.registerStorageClass(newStorageClass) 

485 

486 for name in list(sconfig.keys()): 

487 processStorageClass(name, sconfig) 

488 

489 @staticmethod 

490 def makeNewStorageClass(name: str, 

491 baseClass: Optional[Type[StorageClass]] = StorageClass, 

492 **kwargs: Any) -> Type[StorageClass]: 

493 """Create a new Python class as a subclass of `StorageClass`. 

494 

495 Parameters 

496 ---------- 

497 name : `str` 

498 Name to use for this class. 

499 baseClass : `type`, optional 

500 Base class for this `StorageClass`. Must be either `StorageClass` 

501 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

502 be used. 

503 

504 Returns 

505 ------- 

506 newtype : `type` subclass of `StorageClass` 

507 Newly created Python type. 

508 """ 

509 

510 if baseClass is None: 

511 baseClass = StorageClass 

512 if not issubclass(baseClass, StorageClass): 

513 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

514 

515 # convert the arguments to use different internal names 

516 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

517 clsargs["_cls_name"] = name 

518 

519 # Some container items need to merge with the base class values 

520 # so that a child can inherit but override one bit. 

521 # lists (which you get from configs) are treated as sets for this to 

522 # work consistently. 

523 for k in ("components", "parameters"): 

524 classKey = f"_cls_{k}" 

525 if classKey in clsargs: 

526 baseValue = getattr(baseClass, classKey, None) 

527 if baseValue is not None: 

528 currentValue = clsargs[classKey] 

529 if isinstance(currentValue, dict): 

530 newValue = baseValue.copy() 

531 else: 

532 newValue = set(baseValue) 

533 newValue.update(currentValue) 

534 clsargs[classKey] = newValue 

535 

536 # If we have parameters they should be a frozen set so that the 

537 # parameters in the class can not be modified. 

538 pk = "_cls_parameters" 

539 if pk in clsargs: 

540 clsargs[pk] = frozenset(clsargs[pk]) 

541 

542 return type(f"StorageClass{name}", (baseClass,), clsargs) 

543 

544 def getStorageClass(self, storageClassName: str) -> StorageClass: 

545 """Get a StorageClass instance associated with the supplied name. 

546 

547 Parameters 

548 ---------- 

549 storageClassName : `str` 

550 Name of the storage class to retrieve. 

551 

552 Returns 

553 ------- 

554 instance : `StorageClass` 

555 Instance of the correct `StorageClass`. 

556 

557 Raises 

558 ------ 

559 KeyError 

560 The requested storage class name is not registered. 

561 """ 

562 return self._storageClasses[storageClassName] 

563 

564 def registerStorageClass(self, storageClass: StorageClass) -> None: 

565 """Store the `StorageClass` in the factory. 

566 

567 Will be indexed by `StorageClass.name` and will return instances 

568 of the supplied `StorageClass`. 

569 

570 Parameters 

571 ---------- 

572 storageClass : `StorageClass` 

573 Type of the Python `StorageClass` to register. 

574 

575 Raises 

576 ------ 

577 ValueError 

578 If a storage class has already been registered with 

579 storageClassName and the previous definition differs. 

580 """ 

581 if storageClass.name in self._storageClasses: 

582 existing = self.getStorageClass(storageClass.name) 

583 if existing != storageClass: 

584 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) " 

585 f"differs from current definition ({existing})") 

586 else: 

587 self._storageClasses[storageClass.name] = storageClass 

588 

589 def _unregisterStorageClass(self, storageClassName: str) -> None: 

590 """Remove the named StorageClass from the factory. 

591 

592 Parameters 

593 ---------- 

594 storageClassName : `str` 

595 Name of storage class to remove. 

596 

597 Raises 

598 ------ 

599 KeyError 

600 The named storage class is not registered. 

601 

602 Notes 

603 ----- 

604 This method is intended to simplify testing of StorageClassFactory 

605 functionality and it is not expected to be required for normal usage. 

606 """ 

607 del self._storageClasses[storageClassName]