Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Support for Storage Classes.""" 

25 

26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

27 

28import builtins 

29import copy 

30import logging 

31 

32from typing import ( 

33 Any, 

34 Collection, 

35 Dict, 

36 List, 

37 Mapping, 

38 Optional, 

39 Set, 

40 Sequence, 

41 Tuple, 

42 Type, 

43 Union, 

44) 

45 

46from lsst.utils import doImport 

47from .utils import Singleton, getFullTypeName 

48from .storageClassDelegate import StorageClassDelegate 

49from .config import ConfigSubset, Config 

50from .configSupport import LookupKey 

51 

52log = logging.getLogger(__name__) 

53 

54 

55class StorageClassConfig(ConfigSubset): 

56 component = "storageClasses" 

57 defaultConfigFile = "storageClasses.yaml" 

58 

59 

60class StorageClass: 

61 """Class describing how a label maps to a particular Python type. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name to use for this class. 

67 pytype : `type` or `str` 

68 Python type (or name of type) to associate with the `StorageClass` 

69 components : `dict`, optional 

70 `dict` mapping name of a component to another `StorageClass`. 

71 derivedComponents : `dict`, optional 

72 `dict` mapping name of a derived component to another `StorageClass`. 

73 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

74 Parameters understood by this `StorageClass` that can control 

75 reading of data from datastores. 

76 delegate : `str`, optional 

77 Fully qualified name of class supporting assembly and disassembly 

78 of a `pytype` instance. 

79 """ 

80 _cls_name: str = "BaseStorageClass" 

81 _cls_components: Optional[Dict[str, StorageClass]] = None 

82 _cls_derivedComponents: Optional[Dict[str, StorageClass]] = None 

83 _cls_parameters: Optional[Union[Set[str], Sequence[str]]] = None 

84 _cls_delegate: Optional[str] = None 

85 _cls_pytype: Optional[Union[Type, str]] = None 

86 defaultDelegate: Type = StorageClassDelegate 

87 defaultDelegateName: str = getFullTypeName(defaultDelegate) 

88 

89 def __init__(self, name: Optional[str] = None, 

90 pytype: Optional[Union[Type, str]] = None, 

91 components: Optional[Dict[str, StorageClass]] = None, 

92 derivedComponents: Optional[Dict[str, StorageClass]] = None, 

93 parameters: Optional[Union[Sequence, Set]] = None, 

94 delegate: Optional[str] = None): 

95 if name is None: 

96 name = self._cls_name 

97 if pytype is None: 97 ↛ 99line 97 didn't jump to line 99, because the condition on line 97 was never false

98 pytype = self._cls_pytype 

99 if components is None: 99 ↛ 101line 99 didn't jump to line 101, because the condition on line 99 was never false

100 components = self._cls_components 

101 if derivedComponents is None: 101 ↛ 103line 101 didn't jump to line 103, because the condition on line 101 was never false

102 derivedComponents = self._cls_derivedComponents 

103 if parameters is None: 103 ↛ 105line 103 didn't jump to line 105, because the condition on line 103 was never false

104 parameters = self._cls_parameters 

105 if delegate is None: 105 ↛ 107line 105 didn't jump to line 107, because the condition on line 105 was never false

106 delegate = self._cls_delegate 

107 self.name = name 

108 

109 if pytype is None: 

110 pytype = object 

111 

112 self._pytype: Optional[Type] 

113 if not isinstance(pytype, str): 

114 # Already have a type so store it and get the name 

115 self._pytypeName = getFullTypeName(pytype) 

116 self._pytype = pytype 

117 else: 

118 # Store the type name and defer loading of type 

119 self._pytypeName = pytype 

120 self._pytype = None 

121 

122 self._components = components if components is not None else {} 

123 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

124 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

125 # if the delegate is not None also set it and clear the default 

126 # delegate 

127 self._delegate: Optional[Type] 

128 self._delegateClassName: Optional[str] 

129 if delegate is not None: 

130 self._delegateClassName = delegate 

131 self._delegate = None 

132 elif components is not None: 

133 # We set a default delegate for composites so that a class is 

134 # guaranteed to support something if it is a composite. 

135 log.debug("Setting default delegate for %s", self.name) 

136 self._delegate = self.defaultDelegate 

137 self._delegateClassName = self.defaultDelegateName 

138 else: 

139 self._delegate = None 

140 self._delegateClassName = None 

141 

142 @property 

143 def components(self) -> Dict[str, StorageClass]: 

144 """Component names mapped to associated `StorageClass` 

145 """ 

146 return self._components 

147 

148 @property 

149 def derivedComponents(self) -> Dict[str, StorageClass]: 

150 """Derived component names mapped to associated `StorageClass` 

151 """ 

152 return self._derivedComponents 

153 

154 @property 

155 def parameters(self) -> Set[str]: 

156 """`set` of names of parameters supported by this `StorageClass` 

157 """ 

158 return set(self._parameters) 

159 

160 @property 

161 def pytype(self) -> Type: 

162 """Python type associated with this `StorageClass`.""" 

163 if self._pytype is not None: 

164 return self._pytype 

165 

166 if hasattr(builtins, self._pytypeName): 

167 pytype = getattr(builtins, self._pytypeName) 

168 else: 

169 pytype = doImport(self._pytypeName) 

170 self._pytype = pytype 

171 return self._pytype 

172 

173 @property 

174 def delegateClass(self) -> Optional[Type]: 

175 """Class to use to delegate type-specific actions.""" 

176 if self._delegate is not None: 

177 return self._delegate 

178 if self._delegateClassName is None: 

179 return None 

180 self._delegate = doImport(self._delegateClassName) 

181 return self._delegate 

182 

183 def allComponents(self) -> Mapping[str, StorageClass]: 

184 """Return a mapping of all the derived and read/write components 

185 to the corresponding storage class. 

186 

187 Returns 

188 ------- 

189 comp : `dict` of [`str`, `StorageClass`] 

190 The component name to storage class mapping. 

191 """ 

192 components = copy.copy(self.components) 

193 components.update(self.derivedComponents) 

194 return components 

195 

196 def delegate(self) -> StorageClassDelegate: 

197 """Return an instance of a storage class delegate. 

198 

199 Returns 

200 ------- 

201 delegate : `StorageClassDelegate` 

202 Instance of the delegate associated with this `StorageClass`. 

203 The delegate is constructed with this `StorageClass`. 

204 

205 Raises 

206 ------ 

207 TypeError 

208 This StorageClass has no associated delegate. 

209 """ 

210 cls = self.delegateClass 

211 if cls is None: 

212 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

213 return cls(storageClass=self) 

214 

215 def isComposite(self) -> bool: 

216 """Boolean indicating whether this `StorageClass` is a composite 

217 or not. 

218 

219 Returns 

220 ------- 

221 isComposite : `bool` 

222 `True` if this `StorageClass` is a composite, `False` 

223 otherwise. 

224 """ 

225 if self.components: 

226 return True 

227 return False 

228 

229 def _lookupNames(self) -> Tuple[LookupKey, ...]: 

230 """Keys to use when looking up this DatasetRef in a configuration. 

231 

232 The names are returned in order of priority. 

233 

234 Returns 

235 ------- 

236 names : `tuple` of `LookupKey` 

237 Tuple of a `LookupKey` using the `StorageClass` name. 

238 """ 

239 return (LookupKey(name=self.name), ) 

240 

241 def knownParameters(self) -> Set[str]: 

242 """Return set of all parameters known to this `StorageClass` 

243 

244 The set includes parameters understood by components of a composite. 

245 

246 Returns 

247 ------- 

248 known : `set` 

249 All parameter keys of this `StorageClass` and the component 

250 storage classes. 

251 """ 

252 known = set(self._parameters) 

253 for sc in self.components.values(): 

254 known.update(sc.knownParameters()) 

255 return known 

256 

257 def validateParameters(self, parameters: Collection = None) -> None: 

258 """Check that the parameters are known to this `StorageClass` 

259 

260 Does not check the values. 

261 

262 Parameters 

263 ---------- 

264 parameters : `~collections.abc.Collection`, optional 

265 Collection containing the parameters. Can be `dict`-like or 

266 `set`-like. The parameter values are not checked. 

267 If no parameters are supplied, always returns without error. 

268 

269 Raises 

270 ------ 

271 KeyError 

272 Some parameters are not understood by this `StorageClass`. 

273 """ 

274 # No parameters is always okay 

275 if not parameters: 

276 return 

277 

278 # Extract the important information into a set. Works for dict and 

279 # list. 

280 external = set(parameters) 

281 

282 diff = external - self.knownParameters() 

283 if diff: 

284 s = "s" if len(diff) > 1 else "" 

285 unknown = '\', \''.join(diff) 

286 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

287 

288 def filterParameters(self, parameters: Dict[str, Any], 

289 subset: Collection = None) -> Dict[str, Any]: 

290 """Filter out parameters that are not known to this StorageClass 

291 

292 Parameters 

293 ---------- 

294 parameters : `dict`, optional 

295 Candidate parameters. Can be `None` if no parameters have 

296 been provided. 

297 subset : `~collections.abc.Collection`, optional 

298 Subset of supported parameters that the caller is interested 

299 in using. The subset must be known to the `StorageClass` 

300 if specified. If `None` the supplied parameters will all 

301 be checked, else only the keys in this set will be checked. 

302 

303 Returns 

304 ------- 

305 filtered : `dict` 

306 Valid parameters. Empty `dict` if none are suitable. 

307 

308 Raises 

309 ------ 

310 ValueError 

311 Raised if the provided subset is not a subset of the supported 

312 parameters or if it is an empty set. 

313 """ 

314 if not parameters: 

315 return {} 

316 

317 known = self.knownParameters() 

318 

319 if subset is not None: 

320 if not subset: 

321 raise ValueError("Specified a parameter subset but it was empty") 

322 subset = set(subset) 

323 if not subset.issubset(known): 

324 raise ValueError(f"Requested subset ({subset}) is not a subset of" 

325 f" known parameters ({known})") 

326 wanted = subset 

327 else: 

328 wanted = known 

329 

330 return {k: parameters[k] for k in wanted if k in parameters} 

331 

332 def validateInstance(self, instance: Any) -> bool: 

333 """Check that the supplied Python object has the expected Python type 

334 

335 Parameters 

336 ---------- 

337 instance : `object` 

338 Object to check. 

339 

340 Returns 

341 ------- 

342 isOk : `bool` 

343 True if the supplied instance object can be handled by this 

344 `StorageClass`, False otherwise. 

345 """ 

346 return isinstance(instance, self.pytype) 

347 

348 def __eq__(self, other: Any) -> bool: 

349 """Equality checks name, pytype name, delegate name, and components""" 

350 

351 if not isinstance(other, StorageClass): 

352 return False 

353 

354 if self.name != other.name: 

355 return False 

356 

357 # We must compare pytype and delegate by name since we do not want 

358 # to trigger an import of external module code here 

359 if self._delegateClassName != other._delegateClassName: 

360 return False 

361 if self._pytypeName != other._pytypeName: 

362 return False 

363 

364 # Ensure we have the same component keys in each 

365 if set(self.components.keys()) != set(other.components.keys()): 

366 return False 

367 

368 # Same parameters 

369 if self.parameters != other.parameters: 

370 return False 

371 

372 # Ensure that all the components have the same type 

373 for k in self.components: 

374 if self.components[k] != other.components[k]: 

375 return False 

376 

377 # If we got to this point everything checks out 

378 return True 

379 

380 def __hash__(self) -> int: 

381 return hash(self.name) 

382 

383 def __repr__(self) -> str: 

384 optionals: Dict[str, Any] = {} 

385 if self._pytypeName != "object": 

386 optionals["pytype"] = self._pytypeName 

387 if self._delegateClassName is not None: 

388 optionals["delegate"] = self._delegateClassName 

389 if self._parameters: 

390 optionals["parameters"] = self._parameters 

391 if self.components: 

392 optionals["components"] = self.components 

393 

394 # order is preserved in the dict 

395 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

396 

397 # Start with mandatory fields 

398 r = f"{self.__class__.__name__}({self.name!r}" 

399 if options: 

400 r = r + ", " + options 

401 r = r + ")" 

402 return r 

403 

404 def __str__(self) -> str: 

405 return self.name 

406 

407 

408class StorageClassFactory(metaclass=Singleton): 

409 """Factory for `StorageClass` instances. 

410 

411 This class is a singleton, with each instance sharing the pool of 

412 StorageClasses. Since code can not know whether it is the first 

413 time the instance has been created, the constructor takes no arguments. 

414 To populate the factory with storage classes, a call to 

415 `~StorageClassFactory.addFromConfig()` should be made. 

416 

417 Parameters 

418 ---------- 

419 config : `StorageClassConfig` or `str`, optional 

420 Load configuration. In a ButlerConfig` the relevant configuration 

421 is located in the ``storageClasses`` section. 

422 """ 

423 

424 def __init__(self, config: Optional[Union[StorageClassConfig, str]] = None): 

425 self._storageClasses: Dict[str, StorageClass] = {} 

426 self._configs: List[StorageClassConfig] = [] 

427 

428 # Always seed with the default config 

429 self.addFromConfig(StorageClassConfig()) 

430 

431 if config is not None: 431 ↛ 432line 431 didn't jump to line 432, because the condition on line 431 was never true

432 self.addFromConfig(config) 

433 

434 def __str__(self) -> str: 

435 """Return summary of factory. 

436 

437 Returns 

438 ------- 

439 summary : `str` 

440 Summary of the factory status. 

441 """ 

442 sep = "\n" 

443 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

444 

445StorageClasses 

446-------------- 

447{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

448""" 

449 

450 def __contains__(self, storageClassOrName: Union[StorageClass, str]) -> bool: 

451 """Indicates whether the storage class exists in the factory. 

452 

453 Parameters 

454 ---------- 

455 storageClassOrName : `str` or `StorageClass` 

456 If `str` is given existence of the named StorageClass 

457 in the factory is checked. If `StorageClass` is given 

458 existence and equality are checked. 

459 

460 Returns 

461 ------- 

462 in : `bool` 

463 True if the supplied string is present, or if the supplied 

464 `StorageClass` is present and identical. 

465 

466 Notes 

467 ----- 

468 The two different checks (one for "key" and one for "value") based on 

469 the type of the given argument mean that it is possible for 

470 StorageClass.name to be in the factory but StorageClass to not be 

471 in the factory. 

472 """ 

473 if isinstance(storageClassOrName, str): 473 ↛ 475line 473 didn't jump to line 475, because the condition on line 473 was never false

474 return storageClassOrName in self._storageClasses 

475 elif isinstance(storageClassOrName, StorageClass): 

476 if storageClassOrName.name in self._storageClasses: 

477 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

478 return False 

479 

480 def addFromConfig(self, config: Union[StorageClassConfig, Config, str]) -> None: 

481 """Add more `StorageClass` definitions from a config file. 

482 

483 Parameters 

484 ---------- 

485 config : `StorageClassConfig`, `Config` or `str` 

486 Storage class configuration. Can contain a ``storageClasses`` 

487 key if part of a global configuration. 

488 """ 

489 sconfig = StorageClassConfig(config) 

490 self._configs.append(sconfig) 

491 

492 # Since we can not assume that we will get definitions of 

493 # components or parents before their classes are defined 

494 # we have a helper function that we can call recursively 

495 # to extract definitions from the configuration. 

496 def processStorageClass(name: str, sconfig: StorageClassConfig) -> None: 

497 # Maybe we've already processed this through recursion 

498 if name not in sconfig: 

499 return 

500 info = sconfig.pop(name) 

501 

502 # Always create the storage class so we can ensure that 

503 # we are not trying to overwrite with a different definition 

504 components = None 

505 

506 # Extract scalar items from dict that are needed for 

507 # StorageClass Constructor 

508 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

509 

510 for compName in ("components", "derivedComponents"): 

511 if compName not in info: 

512 continue 

513 components = {} 

514 for cname, ctype in info[compName].items(): 

515 if ctype not in self: 

516 processStorageClass(ctype, sconfig) 

517 components[cname] = self.getStorageClass(ctype) 

518 

519 # Fill in other items 

520 storageClassKwargs[compName] = components 

521 

522 # Create the new storage class and register it 

523 baseClass = None 

524 if "inheritsFrom" in info: 

525 baseName = info["inheritsFrom"] 

526 if baseName not in self: 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true

527 processStorageClass(baseName, sconfig) 

528 baseClass = type(self.getStorageClass(baseName)) 

529 

530 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

531 newStorageClass = newStorageClassType() 

532 self.registerStorageClass(newStorageClass) 

533 

534 for name in list(sconfig.keys()): 

535 processStorageClass(name, sconfig) 

536 

537 @staticmethod 

538 def makeNewStorageClass(name: str, 

539 baseClass: Optional[Type[StorageClass]] = StorageClass, 

540 **kwargs: Any) -> Type[StorageClass]: 

541 """Create a new Python class as a subclass of `StorageClass`. 

542 

543 Parameters 

544 ---------- 

545 name : `str` 

546 Name to use for this class. 

547 baseClass : `type`, optional 

548 Base class for this `StorageClass`. Must be either `StorageClass` 

549 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

550 be used. 

551 

552 Returns 

553 ------- 

554 newtype : `type` subclass of `StorageClass` 

555 Newly created Python type. 

556 """ 

557 

558 if baseClass is None: 

559 baseClass = StorageClass 

560 if not issubclass(baseClass, StorageClass): 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true

561 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

562 

563 # convert the arguments to use different internal names 

564 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

565 clsargs["_cls_name"] = name 

566 

567 # Some container items need to merge with the base class values 

568 # so that a child can inherit but override one bit. 

569 # lists (which you get from configs) are treated as sets for this to 

570 # work consistently. 

571 for k in ("components", "parameters", "derivedComponents"): 

572 classKey = f"_cls_{k}" 

573 if classKey in clsargs: 

574 baseValue = getattr(baseClass, classKey, None) 

575 if baseValue is not None: 

576 currentValue = clsargs[classKey] 

577 if isinstance(currentValue, dict): 577 ↛ 580line 577 didn't jump to line 580, because the condition on line 577 was never false

578 newValue = baseValue.copy() 

579 else: 

580 newValue = set(baseValue) 

581 newValue.update(currentValue) 

582 clsargs[classKey] = newValue 

583 

584 # If we have parameters they should be a frozen set so that the 

585 # parameters in the class can not be modified. 

586 pk = "_cls_parameters" 

587 if pk in clsargs: 

588 clsargs[pk] = frozenset(clsargs[pk]) 

589 

590 return type(f"StorageClass{name}", (baseClass,), clsargs) 

591 

592 def getStorageClass(self, storageClassName: str) -> StorageClass: 

593 """Get a StorageClass instance associated with the supplied name. 

594 

595 Parameters 

596 ---------- 

597 storageClassName : `str` 

598 Name of the storage class to retrieve. 

599 

600 Returns 

601 ------- 

602 instance : `StorageClass` 

603 Instance of the correct `StorageClass`. 

604 

605 Raises 

606 ------ 

607 KeyError 

608 The requested storage class name is not registered. 

609 """ 

610 return self._storageClasses[storageClassName] 

611 

612 def registerStorageClass(self, storageClass: StorageClass) -> None: 

613 """Store the `StorageClass` in the factory. 

614 

615 Will be indexed by `StorageClass.name` and will return instances 

616 of the supplied `StorageClass`. 

617 

618 Parameters 

619 ---------- 

620 storageClass : `StorageClass` 

621 Type of the Python `StorageClass` to register. 

622 

623 Raises 

624 ------ 

625 ValueError 

626 If a storage class has already been registered with 

627 storageClassName and the previous definition differs. 

628 """ 

629 if storageClass.name in self._storageClasses: 629 ↛ 630line 629 didn't jump to line 630, because the condition on line 629 was never true

630 existing = self.getStorageClass(storageClass.name) 

631 if existing != storageClass: 

632 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) " 

633 f"differs from current definition ({existing})") 

634 else: 

635 self._storageClasses[storageClass.name] = storageClass 

636 

637 def _unregisterStorageClass(self, storageClassName: str) -> None: 

638 """Remove the named StorageClass from the factory. 

639 

640 Parameters 

641 ---------- 

642 storageClassName : `str` 

643 Name of storage class to remove. 

644 

645 Raises 

646 ------ 

647 KeyError 

648 The named storage class is not registered. 

649 

650 Notes 

651 ----- 

652 This method is intended to simplify testing of StorageClassFactory 

653 functionality and it is not expected to be required for normal usage. 

654 """ 

655 del self._storageClasses[storageClassName]