Coverage for python/lsst/daf/butler/core/storageClass.py: 43%

363 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-09 02:11 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Support for Storage Classes.""" 

25 

26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

27 

28import builtins 

29import copy 

30import itertools 

31import logging 

32from typing import ( 

33 Any, 

34 Collection, 

35 Dict, 

36 ItemsView, 

37 Iterator, 

38 KeysView, 

39 List, 

40 Mapping, 

41 Optional, 

42 Sequence, 

43 Set, 

44 Tuple, 

45 Type, 

46 Union, 

47 ValuesView, 

48) 

49 

50from lsst.utils import doImportType 

51from lsst.utils.classes import Singleton 

52from lsst.utils.introspection import get_full_type_name 

53 

54from .config import Config, ConfigSubset 

55from .configSupport import LookupKey 

56from .storageClassDelegate import StorageClassDelegate 

57 

58log = logging.getLogger(__name__) 

59 

60 

61class StorageClassConfig(ConfigSubset): 

62 """Configuration class for defining Storage Classes.""" 

63 

64 component = "storageClasses" 

65 defaultConfigFile = "storageClasses.yaml" 

66 

67 

68class StorageClass: 

69 """Class describing how a label maps to a particular Python type. 

70 

71 Parameters 

72 ---------- 

73 name : `str` 

74 Name to use for this class. 

75 pytype : `type` or `str` 

76 Python type (or name of type) to associate with the `StorageClass` 

77 components : `dict`, optional 

78 `dict` mapping name of a component to another `StorageClass`. 

79 derivedComponents : `dict`, optional 

80 `dict` mapping name of a derived component to another `StorageClass`. 

81 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

82 Parameters understood by this `StorageClass` that can control 

83 reading of data from datastores. 

84 delegate : `str`, optional 

85 Fully qualified name of class supporting assembly and disassembly 

86 of a `pytype` instance. 

87 converters : `dict` [`str`, `str`], optional 

88 Mapping of python type to function that can be called to convert 

89 that python type to the valid type of this storage class. 

90 """ 

91 

92 _cls_name: str = "BaseStorageClass" 

93 _cls_components: Optional[Dict[str, StorageClass]] = None 

94 _cls_derivedComponents: Optional[Dict[str, StorageClass]] = None 

95 _cls_parameters: Optional[Union[Set[str], Sequence[str]]] = None 

96 _cls_delegate: Optional[str] = None 

97 _cls_pytype: Optional[Union[Type, str]] = None 

98 _cls_converters: Optional[Dict[str, str]] = None 

99 defaultDelegate: Type = StorageClassDelegate 

100 defaultDelegateName: str = get_full_type_name(defaultDelegate) 

101 

102 def __init__( 

103 self, 

104 name: Optional[str] = None, 

105 pytype: Optional[Union[Type, str]] = None, 

106 components: Optional[Dict[str, StorageClass]] = None, 

107 derivedComponents: Optional[Dict[str, StorageClass]] = None, 

108 parameters: Optional[Union[Sequence, Set]] = None, 

109 delegate: Optional[str] = None, 

110 converters: Optional[Dict[str, str]] = None, 

111 ): 

112 if name is None: 

113 name = self._cls_name 

114 if pytype is None: 114 ↛ 116line 114 didn't jump to line 116, because the condition on line 114 was never false

115 pytype = self._cls_pytype 

116 if components is None: 116 ↛ 118line 116 didn't jump to line 118, because the condition on line 116 was never false

117 components = self._cls_components 

118 if derivedComponents is None: 118 ↛ 120line 118 didn't jump to line 120, because the condition on line 118 was never false

119 derivedComponents = self._cls_derivedComponents 

120 if parameters is None: 120 ↛ 122line 120 didn't jump to line 122, because the condition on line 120 was never false

121 parameters = self._cls_parameters 

122 if delegate is None: 122 ↛ 126line 122 didn't jump to line 126, because the condition on line 122 was never false

123 delegate = self._cls_delegate 

124 

125 # Merge converters with class defaults. 

126 self._converters = {} 

127 if self._cls_converters is not None: 

128 self._converters.update(self._cls_converters) 

129 if converters: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true

130 self._converters.update(converters) 

131 

132 # Version of converters where the python types have been 

133 # Do not try to import anything until needed. 

134 self._converters_by_type: Optional[Dict[Type, Type]] = None 

135 

136 self.name = name 

137 

138 if pytype is None: 

139 pytype = object 

140 

141 self._pytype: Optional[Type] 

142 if not isinstance(pytype, str): 

143 # Already have a type so store it and get the name 

144 self._pytypeName = get_full_type_name(pytype) 

145 self._pytype = pytype 

146 else: 

147 # Store the type name and defer loading of type 

148 self._pytypeName = pytype 

149 self._pytype = None 

150 

151 if components is not None: 

152 if len(components) == 1: 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true

153 raise ValueError( 

154 f"Composite storage class {name} is not allowed to have" 

155 f" only one component '{next(iter(components))}'." 

156 " Did you mean it to be a derived component?" 

157 ) 

158 self._components = components 

159 else: 

160 self._components = {} 

161 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

162 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

163 # if the delegate is not None also set it and clear the default 

164 # delegate 

165 self._delegate: Optional[Type] 

166 self._delegateClassName: Optional[str] 

167 if delegate is not None: 

168 self._delegateClassName = delegate 

169 self._delegate = None 

170 elif components is not None: 170 ↛ 173line 170 didn't jump to line 173, because the condition on line 170 was never true

171 # We set a default delegate for composites so that a class is 

172 # guaranteed to support something if it is a composite. 

173 log.debug("Setting default delegate for %s", self.name) 

174 self._delegate = self.defaultDelegate 

175 self._delegateClassName = self.defaultDelegateName 

176 else: 

177 self._delegate = None 

178 self._delegateClassName = None 

179 

180 @property 

181 def components(self) -> Dict[str, StorageClass]: 

182 """Return the components associated with this `StorageClass`.""" 

183 return self._components 

184 

185 @property 

186 def derivedComponents(self) -> Dict[str, StorageClass]: 

187 """Return derived components associated with `StorageClass`.""" 

188 return self._derivedComponents 

189 

190 @property 

191 def converters(self) -> Dict[str, str]: 

192 """Return the type converters supported by this `StorageClass`.""" 

193 return self._converters 

194 

195 @property 

196 def converters_by_type(self) -> Dict[Type, Type]: 

197 """Return the type converters as python types.""" 

198 if self._converters_by_type is None: 

199 self._converters_by_type = {} 

200 

201 # Loop over list because the dict can be edited in loop. 

202 for candidate_type_str, converter_str in list(self.converters.items()): 

203 if hasattr(builtins, candidate_type_str): 

204 candidate_type = getattr(builtins, candidate_type_str) 

205 else: 

206 try: 

207 candidate_type = doImportType(candidate_type_str) 

208 except ImportError as e: 

209 log.warning( 

210 "Unable to import type %s associated with storage class %s (%s)", 

211 candidate_type_str, 

212 self.name, 

213 e, 

214 ) 

215 del self.converters[candidate_type_str] 

216 continue 

217 

218 try: 

219 converter = doImportType(converter_str) 

220 except ImportError as e: 

221 log.warning( 

222 "Unable to import conversion function %s associated with storage class %s " 

223 "required to convert type %s (%s)", 

224 converter_str, 

225 self.name, 

226 candidate_type_str, 

227 e, 

228 ) 

229 del self.converters[candidate_type_str] 

230 continue 

231 if not callable(converter): 

232 # doImportType is annotated to return a Type but in actual 

233 # fact it can return Any except ModuleType because package 

234 # variables can be accessed. This make mypy believe it 

235 # is impossible for the return value to not be a callable 

236 # so we must ignore the warning. 

237 log.warning( # type: ignore 

238 "Conversion function %s associated with storage class " 

239 "%s to convert type %s is not a callable.", 

240 converter_str, 

241 self.name, 

242 candidate_type_str, 

243 ) 

244 del self.converters[candidate_type_str] 

245 continue 

246 self._converters_by_type[candidate_type] = converter 

247 return self._converters_by_type 

248 

249 @property 

250 def parameters(self) -> Set[str]: 

251 """Return `set` of names of supported parameters.""" 

252 return set(self._parameters) 

253 

254 @property 

255 def pytype(self) -> Type: 

256 """Return Python type associated with this `StorageClass`.""" 

257 if self._pytype is not None: 

258 return self._pytype 

259 

260 if hasattr(builtins, self._pytypeName): 

261 pytype = getattr(builtins, self._pytypeName) 

262 else: 

263 pytype = doImportType(self._pytypeName) 

264 self._pytype = pytype 

265 return self._pytype 

266 

267 @property 

268 def delegateClass(self) -> Optional[Type]: 

269 """Class to use to delegate type-specific actions.""" 

270 if self._delegate is not None: 

271 return self._delegate 

272 if self._delegateClassName is None: 

273 return None 

274 delegate_class = doImportType(self._delegateClassName) 

275 self._delegate = delegate_class 

276 return self._delegate 

277 

278 def allComponents(self) -> Mapping[str, StorageClass]: 

279 """Return all defined components. 

280 

281 This mapping includes all the derived and read/write components 

282 for the corresponding storage class. 

283 

284 Returns 

285 ------- 

286 comp : `dict` of [`str`, `StorageClass`] 

287 The component name to storage class mapping. 

288 """ 

289 components = copy.copy(self.components) 

290 components.update(self.derivedComponents) 

291 return components 

292 

293 def delegate(self) -> StorageClassDelegate: 

294 """Return an instance of a storage class delegate. 

295 

296 Returns 

297 ------- 

298 delegate : `StorageClassDelegate` 

299 Instance of the delegate associated with this `StorageClass`. 

300 The delegate is constructed with this `StorageClass`. 

301 

302 Raises 

303 ------ 

304 TypeError 

305 This StorageClass has no associated delegate. 

306 """ 

307 cls = self.delegateClass 

308 if cls is None: 

309 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

310 return cls(storageClass=self) 

311 

312 def isComposite(self) -> bool: 

313 """Return Boolean indicating whether this is a composite or not. 

314 

315 Returns 

316 ------- 

317 isComposite : `bool` 

318 `True` if this `StorageClass` is a composite, `False` 

319 otherwise. 

320 """ 

321 if self.components: 

322 return True 

323 return False 

324 

325 def _lookupNames(self) -> Tuple[LookupKey, ...]: 

326 """Keys to use when looking up this DatasetRef in a configuration. 

327 

328 The names are returned in order of priority. 

329 

330 Returns 

331 ------- 

332 names : `tuple` of `LookupKey` 

333 Tuple of a `LookupKey` using the `StorageClass` name. 

334 """ 

335 return (LookupKey(name=self.name),) 

336 

337 def knownParameters(self) -> Set[str]: 

338 """Return set of all parameters known to this `StorageClass`. 

339 

340 The set includes parameters understood by components of a composite. 

341 

342 Returns 

343 ------- 

344 known : `set` 

345 All parameter keys of this `StorageClass` and the component 

346 storage classes. 

347 """ 

348 known = set(self._parameters) 

349 for sc in self.components.values(): 

350 known.update(sc.knownParameters()) 

351 return known 

352 

353 def validateParameters(self, parameters: Collection | None = None) -> None: 

354 """Check that the parameters are known to this `StorageClass`. 

355 

356 Does not check the values. 

357 

358 Parameters 

359 ---------- 

360 parameters : `~collections.abc.Collection`, optional 

361 Collection containing the parameters. Can be `dict`-like or 

362 `set`-like. The parameter values are not checked. 

363 If no parameters are supplied, always returns without error. 

364 

365 Raises 

366 ------ 

367 KeyError 

368 Some parameters are not understood by this `StorageClass`. 

369 """ 

370 # No parameters is always okay 

371 if not parameters: 

372 return 

373 

374 # Extract the important information into a set. Works for dict and 

375 # list. 

376 external = set(parameters) 

377 

378 diff = external - self.knownParameters() 

379 if diff: 

380 s = "s" if len(diff) > 1 else "" 

381 unknown = "', '".join(diff) 

382 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

383 

384 def filterParameters( 

385 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

386 ) -> Mapping[str, Any]: 

387 """Filter out parameters that are not known to this `StorageClass`. 

388 

389 Parameters 

390 ---------- 

391 parameters : `Mapping`, optional 

392 Candidate parameters. Can be `None` if no parameters have 

393 been provided. 

394 subset : `~collections.abc.Collection`, optional 

395 Subset of supported parameters that the caller is interested 

396 in using. The subset must be known to the `StorageClass` 

397 if specified. If `None` the supplied parameters will all 

398 be checked, else only the keys in this set will be checked. 

399 

400 Returns 

401 ------- 

402 filtered : `Mapping` 

403 Valid parameters. Empty `dict` if none are suitable. 

404 

405 Raises 

406 ------ 

407 ValueError 

408 Raised if the provided subset is not a subset of the supported 

409 parameters or if it is an empty set. 

410 """ 

411 if not parameters: 

412 return {} 

413 

414 known = self.knownParameters() 

415 

416 if subset is not None: 

417 if not subset: 

418 raise ValueError("Specified a parameter subset but it was empty") 

419 subset = set(subset) 

420 if not subset.issubset(known): 

421 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

422 wanted = subset 

423 else: 

424 wanted = known 

425 

426 return {k: parameters[k] for k in wanted if k in parameters} 

427 

428 def validateInstance(self, instance: Any) -> bool: 

429 """Check that the supplied Python object has the expected Python type. 

430 

431 Parameters 

432 ---------- 

433 instance : `object` 

434 Object to check. 

435 

436 Returns 

437 ------- 

438 isOk : `bool` 

439 True if the supplied instance object can be handled by this 

440 `StorageClass`, False otherwise. 

441 """ 

442 return isinstance(instance, self.pytype) 

443 

444 def is_type(self, other: Type, compare_types: bool = False) -> bool: 

445 """Return Boolean indicating whether the supplied type matches 

446 the type in this `StorageClass`. 

447 

448 Parameters 

449 ---------- 

450 other : `Type` 

451 The type to be checked. 

452 compare_types : `bool`, optional 

453 If `True` the python type will be used in the comparison 

454 if the type names do not match. This may trigger an import 

455 of code and so can be slower. 

456 

457 Returns 

458 ------- 

459 match : `bool` 

460 `True` if the types are equal. 

461 

462 Notes 

463 ----- 

464 If this `StorageClass` has not yet imported the Python type the 

465 check is done against the full type name, this prevents an attempt 

466 to import the type when it will likely not match. 

467 """ 

468 if self._pytype: 

469 return self._pytype is other 

470 

471 other_name = get_full_type_name(other) 

472 if self._pytypeName == other_name: 

473 return True 

474 

475 if compare_types: 

476 # Must protect against the import failing. 

477 try: 

478 return self.pytype is other 

479 except Exception: 

480 pass 

481 

482 return False 

483 

484 def can_convert(self, other: StorageClass) -> bool: 

485 """Return `True` if this storage class can convert python types 

486 in the other storage class. 

487 

488 Parameters 

489 ---------- 

490 other : `StorageClass` 

491 The storage class to check. 

492 

493 Returns 

494 ------- 

495 can : `bool` 

496 `True` if this storage class has a registered converter for 

497 the python type associated with the other storage class. That 

498 converter will convert the other python type to the one associated 

499 with this storage class. 

500 """ 

501 if other.name == self.name: 

502 # Identical storage classes are compatible. 

503 return True 

504 

505 # It may be that the storage class being compared is not 

506 # available because the python type can't be imported. In that 

507 # case conversion must be impossible. 

508 try: 

509 other_pytype = other.pytype 

510 except Exception: 

511 return False 

512 

513 # Or even this storage class itself can not have the type imported. 

514 try: 

515 self_pytype = self.pytype 

516 except Exception: 

517 return False 

518 

519 if issubclass(other_pytype, self_pytype): 

520 # Storage classes have different names but the same python type. 

521 return True 

522 

523 for candidate_type in self.converters_by_type: 

524 if issubclass(other_pytype, candidate_type): 

525 return True 

526 return False 

527 

528 def coerce_type(self, incorrect: Any) -> Any: 

529 """Coerce the supplied incorrect instance to the python type 

530 associated with this `StorageClass`. 

531 

532 Parameters 

533 ---------- 

534 incorrect : `object` 

535 An object that might be the incorrect type. 

536 

537 Returns 

538 ------- 

539 correct : `object` 

540 An object that matches the python type of this `StorageClass`. 

541 Can be the same object as given. If `None`, `None` will be 

542 returned. 

543 

544 Raises 

545 ------ 

546 TypeError 

547 Raised if no conversion can be found. 

548 """ 

549 if incorrect is None: 

550 return None 

551 

552 # Possible this is the correct type already. 

553 if self.validateInstance(incorrect): 

554 return incorrect 

555 

556 # Check each registered converter. 

557 for candidate_type, converter in self.converters_by_type.items(): 

558 if isinstance(incorrect, candidate_type): 

559 try: 

560 return converter(incorrect) 

561 except Exception: 

562 log.error( 

563 "Converter %s failed to convert type %s", 

564 get_full_type_name(converter), 

565 get_full_type_name(incorrect), 

566 ) 

567 raise 

568 raise TypeError( 

569 "Type does not match and no valid converter found to convert" 

570 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

571 ) 

572 

573 def __eq__(self, other: Any) -> bool: 

574 """Equality checks name, pytype name, delegate name, and components.""" 

575 if not isinstance(other, StorageClass): 

576 return NotImplemented 

577 

578 if self.name != other.name: 

579 return False 

580 

581 # We must compare pytype and delegate by name since we do not want 

582 # to trigger an import of external module code here 

583 if self._delegateClassName != other._delegateClassName: 

584 return False 

585 if self._pytypeName != other._pytypeName: 

586 return False 

587 

588 # Ensure we have the same component keys in each 

589 if set(self.components.keys()) != set(other.components.keys()): 

590 return False 

591 

592 # Same parameters 

593 if self.parameters != other.parameters: 

594 return False 

595 

596 # Ensure that all the components have the same type 

597 for k in self.components: 

598 if self.components[k] != other.components[k]: 

599 return False 

600 

601 # If we got to this point everything checks out 

602 return True 

603 

604 def __hash__(self) -> int: 

605 return hash(self.name) 

606 

607 def __repr__(self) -> str: 

608 optionals: Dict[str, Any] = {} 

609 if self._pytypeName != "object": 

610 optionals["pytype"] = self._pytypeName 

611 if self._delegateClassName is not None: 

612 optionals["delegate"] = self._delegateClassName 

613 if self._parameters: 

614 optionals["parameters"] = self._parameters 

615 if self.components: 

616 optionals["components"] = self.components 

617 if self.converters: 

618 optionals["converters"] = self.converters 

619 

620 # order is preserved in the dict 

621 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

622 

623 # Start with mandatory fields 

624 r = f"{self.__class__.__name__}({self.name!r}" 

625 if options: 

626 r = r + ", " + options 

627 r = r + ")" 

628 return r 

629 

630 def __str__(self) -> str: 

631 return self.name 

632 

633 

634class StorageClassFactory(metaclass=Singleton): 

635 """Factory for `StorageClass` instances. 

636 

637 This class is a singleton, with each instance sharing the pool of 

638 StorageClasses. Since code can not know whether it is the first 

639 time the instance has been created, the constructor takes no arguments. 

640 To populate the factory with storage classes, a call to 

641 `~StorageClassFactory.addFromConfig()` should be made. 

642 

643 Parameters 

644 ---------- 

645 config : `StorageClassConfig` or `str`, optional 

646 Load configuration. In a ButlerConfig` the relevant configuration 

647 is located in the ``storageClasses`` section. 

648 """ 

649 

650 def __init__(self, config: Optional[Union[StorageClassConfig, str]] = None): 

651 self._storageClasses: Dict[str, StorageClass] = {} 

652 self._configs: List[StorageClassConfig] = [] 

653 

654 # Always seed with the default config 

655 self.addFromConfig(StorageClassConfig()) 

656 

657 if config is not None: 657 ↛ 658line 657 didn't jump to line 658, because the condition on line 657 was never true

658 self.addFromConfig(config) 

659 

660 def __str__(self) -> str: 

661 """Return summary of factory. 

662 

663 Returns 

664 ------- 

665 summary : `str` 

666 Summary of the factory status. 

667 """ 

668 sep = "\n" 

669 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

670 

671StorageClasses 

672-------------- 

673{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

674""" 

675 

676 def __contains__(self, storageClassOrName: Union[StorageClass, str]) -> bool: 

677 """Indicate whether the storage class exists in the factory. 

678 

679 Parameters 

680 ---------- 

681 storageClassOrName : `str` or `StorageClass` 

682 If `str` is given existence of the named StorageClass 

683 in the factory is checked. If `StorageClass` is given 

684 existence and equality are checked. 

685 

686 Returns 

687 ------- 

688 in : `bool` 

689 True if the supplied string is present, or if the supplied 

690 `StorageClass` is present and identical. 

691 

692 Notes 

693 ----- 

694 The two different checks (one for "key" and one for "value") based on 

695 the type of the given argument mean that it is possible for 

696 StorageClass.name to be in the factory but StorageClass to not be 

697 in the factory. 

698 """ 

699 if isinstance(storageClassOrName, str): 699 ↛ 701line 699 didn't jump to line 701, because the condition on line 699 was never false

700 return storageClassOrName in self._storageClasses 

701 elif isinstance(storageClassOrName, StorageClass): 

702 if storageClassOrName.name in self._storageClasses: 

703 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

704 return False 

705 

706 def __len__(self) -> int: 

707 return len(self._storageClasses) 

708 

709 def __iter__(self) -> Iterator[str]: 

710 return iter(self._storageClasses) 

711 

712 def values(self) -> ValuesView[StorageClass]: 

713 return self._storageClasses.values() 

714 

715 def keys(self) -> KeysView[str]: 

716 return self._storageClasses.keys() 

717 

718 def items(self) -> ItemsView[str, StorageClass]: 

719 return self._storageClasses.items() 

720 

721 def addFromConfig(self, config: Union[StorageClassConfig, Config, str]) -> None: 

722 """Add more `StorageClass` definitions from a config file. 

723 

724 Parameters 

725 ---------- 

726 config : `StorageClassConfig`, `Config` or `str` 

727 Storage class configuration. Can contain a ``storageClasses`` 

728 key if part of a global configuration. 

729 """ 

730 sconfig = StorageClassConfig(config) 

731 self._configs.append(sconfig) 

732 

733 # Since we can not assume that we will get definitions of 

734 # components or parents before their classes are defined 

735 # we have a helper function that we can call recursively 

736 # to extract definitions from the configuration. 

737 def processStorageClass(name: str, sconfig: StorageClassConfig, msg: str = "") -> None: 

738 # Maybe we've already processed this through recursion 

739 if name not in sconfig: 

740 return 

741 info = sconfig.pop(name) 

742 

743 # Always create the storage class so we can ensure that 

744 # we are not trying to overwrite with a different definition 

745 components = None 

746 

747 # Extract scalar items from dict that are needed for 

748 # StorageClass Constructor 

749 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

750 

751 if "converters" in info: 

752 storageClassKwargs["converters"] = info["converters"].toDict() 

753 

754 for compName in ("components", "derivedComponents"): 

755 if compName not in info: 

756 continue 

757 components = {} 

758 for cname, ctype in info[compName].items(): 

759 if ctype not in self: 

760 processStorageClass(ctype, sconfig, msg) 

761 components[cname] = self.getStorageClass(ctype) 

762 

763 # Fill in other items 

764 storageClassKwargs[compName] = components 

765 

766 # Create the new storage class and register it 

767 baseClass = None 

768 if "inheritsFrom" in info: 

769 baseName = info["inheritsFrom"] 

770 if baseName not in self: 770 ↛ 771line 770 didn't jump to line 771, because the condition on line 770 was never true

771 processStorageClass(baseName, sconfig, msg) 

772 baseClass = type(self.getStorageClass(baseName)) 

773 

774 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

775 newStorageClass = newStorageClassType() 

776 self.registerStorageClass(newStorageClass, msg=msg) 

777 

778 # In case there is a problem, construct a context message for any 

779 # error reporting. 

780 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

781 context = f"when adding definitions from {', '.join(files)}" if files else "" 

782 log.debug("Adding definitions from config %s", ", ".join(files)) 

783 

784 for name in list(sconfig.keys()): 

785 processStorageClass(name, sconfig, context) 

786 

787 @staticmethod 

788 def makeNewStorageClass( 

789 name: str, baseClass: Optional[Type[StorageClass]] = StorageClass, **kwargs: Any 

790 ) -> Type[StorageClass]: 

791 """Create a new Python class as a subclass of `StorageClass`. 

792 

793 Parameters 

794 ---------- 

795 name : `str` 

796 Name to use for this class. 

797 baseClass : `type`, optional 

798 Base class for this `StorageClass`. Must be either `StorageClass` 

799 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

800 be used. 

801 

802 Returns 

803 ------- 

804 newtype : `type` subclass of `StorageClass` 

805 Newly created Python type. 

806 """ 

807 if baseClass is None: 

808 baseClass = StorageClass 

809 if not issubclass(baseClass, StorageClass): 809 ↛ 810line 809 didn't jump to line 810, because the condition on line 809 was never true

810 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

811 

812 # convert the arguments to use different internal names 

813 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

814 clsargs["_cls_name"] = name 

815 

816 # Some container items need to merge with the base class values 

817 # so that a child can inherit but override one bit. 

818 # lists (which you get from configs) are treated as sets for this to 

819 # work consistently. 

820 for k in ("components", "parameters", "derivedComponents", "converters"): 

821 classKey = f"_cls_{k}" 

822 if classKey in clsargs: 

823 baseValue = getattr(baseClass, classKey, None) 

824 if baseValue is not None: 

825 currentValue = clsargs[classKey] 

826 if isinstance(currentValue, dict): 826 ↛ 829line 826 didn't jump to line 829, because the condition on line 826 was never false

827 newValue = baseValue.copy() 

828 else: 

829 newValue = set(baseValue) 

830 newValue.update(currentValue) 

831 clsargs[classKey] = newValue 

832 

833 # If we have parameters they should be a frozen set so that the 

834 # parameters in the class can not be modified. 

835 pk = "_cls_parameters" 

836 if pk in clsargs: 

837 clsargs[pk] = frozenset(clsargs[pk]) 

838 

839 return type(f"StorageClass{name}", (baseClass,), clsargs) 

840 

841 def getStorageClass(self, storageClassName: str) -> StorageClass: 

842 """Get a StorageClass instance associated with the supplied name. 

843 

844 Parameters 

845 ---------- 

846 storageClassName : `str` 

847 Name of the storage class to retrieve. 

848 

849 Returns 

850 ------- 

851 instance : `StorageClass` 

852 Instance of the correct `StorageClass`. 

853 

854 Raises 

855 ------ 

856 KeyError 

857 The requested storage class name is not registered. 

858 """ 

859 return self._storageClasses[storageClassName] 

860 

861 def findStorageClass(self, pytype: Type, compare_types: bool = False) -> StorageClass: 

862 """Find the storage class associated with this python type. 

863 

864 Parameters 

865 ---------- 

866 pytype : `type` 

867 The Python type to be matched. 

868 compare_types : `bool`, optional 

869 If `False`, the type will be checked against name of the python 

870 type. This comparison is always done first. If `True` and the 

871 string comparison failed, each candidate storage class will be 

872 forced to have its type imported. This can be significantly slower. 

873 

874 Returns 

875 ------- 

876 storageClass : `StorageClass` 

877 The matching storage class. 

878 

879 Raises 

880 ------ 

881 KeyError 

882 Raised if no match could be found. 

883 

884 Notes 

885 ----- 

886 It is possible for a python type to be associated with multiple 

887 storage classes. This method will currently return the first that 

888 matches. 

889 """ 

890 result = self._find_storage_class(pytype, False) 

891 if result: 

892 return result 

893 

894 if compare_types: 

895 # The fast comparison failed and we were asked to try the 

896 # variant that might involve code imports. 

897 result = self._find_storage_class(pytype, True) 

898 if result: 

899 return result 

900 

901 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}") 

902 

903 def _find_storage_class(self, pytype: Type, compare_types: bool) -> Optional[StorageClass]: 

904 """Iterate through all storage classes to find a match. 

905 

906 Parameters 

907 ---------- 

908 pytype : `type` 

909 The Python type to be matched. 

910 compare_types : `bool`, optional 

911 Whether to use type name matching or explicit type matching. 

912 The latter can be slower. 

913 

914 Returns 

915 ------- 

916 storageClass : `StorageClass` or `None` 

917 The matching storage class, or `None` if no match was found. 

918 

919 Notes 

920 ----- 

921 Helper method for ``findStorageClass``. 

922 """ 

923 for storageClass in self.values(): 

924 if storageClass.is_type(pytype, compare_types=compare_types): 

925 return storageClass 

926 return None 

927 

928 def registerStorageClass(self, storageClass: StorageClass, msg: Optional[str] = None) -> None: 

929 """Store the `StorageClass` in the factory. 

930 

931 Will be indexed by `StorageClass.name` and will return instances 

932 of the supplied `StorageClass`. 

933 

934 Parameters 

935 ---------- 

936 storageClass : `StorageClass` 

937 Type of the Python `StorageClass` to register. 

938 msg : `str`, optional 

939 Additional message string to be included in any error message. 

940 

941 Raises 

942 ------ 

943 ValueError 

944 If a storage class has already been registered with 

945 that storage class name and the previous definition differs. 

946 """ 

947 if storageClass.name in self._storageClasses: 947 ↛ 948line 947 didn't jump to line 948, because the condition on line 947 was never true

948 existing = self.getStorageClass(storageClass.name) 

949 if existing != storageClass: 

950 errmsg = f" {msg}" if msg else "" 

951 raise ValueError( 

952 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

953 f"differs from current definition ({existing!r}){errmsg}" 

954 ) 

955 else: 

956 self._storageClasses[storageClass.name] = storageClass 

957 

958 def _unregisterStorageClass(self, storageClassName: str) -> None: 

959 """Remove the named StorageClass from the factory. 

960 

961 Parameters 

962 ---------- 

963 storageClassName : `str` 

964 Name of storage class to remove. 

965 

966 Raises 

967 ------ 

968 KeyError 

969 The named storage class is not registered. 

970 

971 Notes 

972 ----- 

973 This method is intended to simplify testing of StorageClassFactory 

974 functionality and it is not expected to be required for normal usage. 

975 """ 

976 del self._storageClasses[storageClassName]