Coverage for python/lsst/daf/butler/core/storageClass.py: 45%

357 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for Storage Classes.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

33 

34import builtins 

35import itertools 

36import logging 

37from collections import ChainMap 

38from collections.abc import ( 

39 Callable, 

40 Collection, 

41 ItemsView, 

42 Iterator, 

43 KeysView, 

44 Mapping, 

45 Sequence, 

46 Set, 

47 ValuesView, 

48) 

49from typing import Any 

50 

51from lsst.utils import doImportType 

52from lsst.utils.classes import Singleton 

53from lsst.utils.introspection import get_full_type_name 

54 

55from .config import Config, ConfigSubset 

56from .configSupport import LookupKey 

57from .storageClassDelegate import StorageClassDelegate 

58 

59log = logging.getLogger(__name__) 

60 

61 

62class StorageClassConfig(ConfigSubset): 

63 """Configuration class for defining Storage Classes.""" 

64 

65 component = "storageClasses" 

66 defaultConfigFile = "storageClasses.yaml" 

67 

68 

69class StorageClass: 

70 """Class describing how a label maps to a particular Python type. 

71 

72 Parameters 

73 ---------- 

74 name : `str` 

75 Name to use for this class. 

76 pytype : `type` or `str` 

77 Python type (or name of type) to associate with the `StorageClass` 

78 components : `dict`, optional 

79 `dict` mapping name of a component to another `StorageClass`. 

80 derivedComponents : `dict`, optional 

81 `dict` mapping name of a derived component to another `StorageClass`. 

82 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

83 Parameters understood by this `StorageClass` that can control 

84 reading of data from datastores. 

85 delegate : `str`, optional 

86 Fully qualified name of class supporting assembly and disassembly 

87 of a `pytype` instance. 

88 converters : `dict` [`str`, `str`], optional 

89 Mapping of python type to function that can be called to convert 

90 that python type to the valid type of this storage class. 

91 """ 

92 

93 _cls_name: str = "BaseStorageClass" 

94 _cls_components: dict[str, StorageClass] | None = None 

95 _cls_derivedComponents: dict[str, StorageClass] | None = None 

96 _cls_parameters: Set[str] | Sequence[str] | None = None 

97 _cls_delegate: str | None = None 

98 _cls_pytype: type | str | None = None 

99 _cls_converters: dict[str, str] | None = None 

100 defaultDelegate: type = StorageClassDelegate 

101 defaultDelegateName: str = get_full_type_name(defaultDelegate) 

102 

103 def __init__( 

104 self, 

105 name: str | None = None, 

106 pytype: type | str | None = None, 

107 components: dict[str, StorageClass] | None = None, 

108 derivedComponents: dict[str, StorageClass] | None = None, 

109 parameters: Sequence[str] | Set[str] | None = None, 

110 delegate: str | None = None, 

111 converters: dict[str, str] | None = None, 

112 ): 

113 if name is None: 

114 name = self._cls_name 

115 if pytype is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false

116 pytype = self._cls_pytype 

117 if components is None: 117 ↛ 119line 117 didn't jump to line 119, because the condition on line 117 was never false

118 components = self._cls_components 

119 if derivedComponents is None: 119 ↛ 121line 119 didn't jump to line 121, because the condition on line 119 was never false

120 derivedComponents = self._cls_derivedComponents 

121 if parameters is None: 121 ↛ 123line 121 didn't jump to line 123, because the condition on line 121 was never false

122 parameters = self._cls_parameters 

123 if delegate is None: 123 ↛ 127line 123 didn't jump to line 127, because the condition on line 123 was never false

124 delegate = self._cls_delegate 

125 

126 # Merge converters with class defaults. 

127 self._converters = {} 

128 if self._cls_converters is not None: 

129 self._converters.update(self._cls_converters) 

130 if converters: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true

131 self._converters.update(converters) 

132 

133 # Version of converters where the python types have been 

134 # Do not try to import anything until needed. 

135 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

136 

137 self.name = name 

138 

139 if pytype is None: 

140 pytype = object 

141 

142 self._pytype: type | None 

143 if not isinstance(pytype, str): 

144 # Already have a type so store it and get the name 

145 self._pytypeName = get_full_type_name(pytype) 

146 self._pytype = pytype 

147 else: 

148 # Store the type name and defer loading of type 

149 self._pytypeName = pytype 

150 self._pytype = None 

151 

152 if components is not None: 

153 if len(components) == 1: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 raise ValueError( 

155 f"Composite storage class {name} is not allowed to have" 

156 f" only one component '{next(iter(components))}'." 

157 " Did you mean it to be a derived component?" 

158 ) 

159 self._components = components 

160 else: 

161 self._components = {} 

162 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

163 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

164 # if the delegate is not None also set it and clear the default 

165 # delegate 

166 self._delegate: type | None 

167 self._delegateClassName: str | None 

168 if delegate is not None: 

169 self._delegateClassName = delegate 

170 self._delegate = None 

171 elif components is not None: 171 ↛ 174line 171 didn't jump to line 174, because the condition on line 171 was never true

172 # We set a default delegate for composites so that a class is 

173 # guaranteed to support something if it is a composite. 

174 log.debug("Setting default delegate for %s", self.name) 

175 self._delegate = self.defaultDelegate 

176 self._delegateClassName = self.defaultDelegateName 

177 else: 

178 self._delegate = None 

179 self._delegateClassName = None 

180 

181 @property 

182 def components(self) -> Mapping[str, StorageClass]: 

183 """Return the components associated with this `StorageClass`.""" 

184 return self._components 

185 

186 @property 

187 def derivedComponents(self) -> Mapping[str, StorageClass]: 

188 """Return derived components associated with `StorageClass`.""" 

189 return self._derivedComponents 

190 

191 @property 

192 def converters(self) -> Mapping[str, str]: 

193 """Return the type converters supported by this `StorageClass`.""" 

194 return self._converters 

195 

196 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

197 """Return the type converters as python types.""" 

198 if self._converters_by_type is None: 

199 self._converters_by_type = {} 

200 

201 # Loop over list because the dict can be edited in loop. 

202 for candidate_type_str, converter_str in list(self.converters.items()): 

203 if hasattr(builtins, candidate_type_str): 

204 candidate_type = getattr(builtins, candidate_type_str) 

205 else: 

206 try: 

207 candidate_type = doImportType(candidate_type_str) 

208 except ImportError as e: 

209 log.warning( 

210 "Unable to import type %s associated with storage class %s (%s)", 

211 candidate_type_str, 

212 self.name, 

213 e, 

214 ) 

215 del self._converters[candidate_type_str] 

216 continue 

217 

218 try: 

219 converter = doImportType(converter_str) 

220 except ImportError as e: 

221 log.warning( 

222 "Unable to import conversion function %s associated with storage class %s " 

223 "required to convert type %s (%s)", 

224 converter_str, 

225 self.name, 

226 candidate_type_str, 

227 e, 

228 ) 

229 del self._converters[candidate_type_str] 

230 continue 

231 if not callable(converter): 

232 # doImportType is annotated to return a Type but in actual 

233 # fact it can return Any except ModuleType because package 

234 # variables can be accessed. This make mypy believe it 

235 # is impossible for the return value to not be a callable 

236 # so we must ignore the warning. 

237 log.warning( # type: ignore 

238 "Conversion function %s associated with storage class " 

239 "%s to convert type %s is not a callable.", 

240 converter_str, 

241 self.name, 

242 candidate_type_str, 

243 ) 

244 del self._converters[candidate_type_str] 

245 continue 

246 self._converters_by_type[candidate_type] = converter 

247 return self._converters_by_type 

248 

249 @property 

250 def parameters(self) -> set[str]: 

251 """Return `set` of names of supported parameters.""" 

252 return set(self._parameters) 

253 

254 @property 

255 def pytype(self) -> type: 

256 """Return Python type associated with this `StorageClass`.""" 

257 if self._pytype is not None: 

258 return self._pytype 

259 

260 if hasattr(builtins, self._pytypeName): 

261 pytype = getattr(builtins, self._pytypeName) 

262 else: 

263 pytype = doImportType(self._pytypeName) 

264 self._pytype = pytype 

265 return self._pytype 

266 

267 @property 

268 def delegateClass(self) -> type | None: 

269 """Class to use to delegate type-specific actions.""" 

270 if self._delegate is not None: 

271 return self._delegate 

272 if self._delegateClassName is None: 

273 return None 

274 delegate_class = doImportType(self._delegateClassName) 

275 self._delegate = delegate_class 

276 return self._delegate 

277 

278 def allComponents(self) -> Mapping[str, StorageClass]: 

279 """Return all defined components. 

280 

281 This mapping includes all the derived and read/write components 

282 for the corresponding storage class. 

283 

284 Returns 

285 ------- 

286 comp : `dict` of [`str`, `StorageClass`] 

287 The component name to storage class mapping. 

288 """ 

289 return ChainMap(self._components, self._derivedComponents) 

290 

291 def delegate(self) -> StorageClassDelegate: 

292 """Return an instance of a storage class delegate. 

293 

294 Returns 

295 ------- 

296 delegate : `StorageClassDelegate` 

297 Instance of the delegate associated with this `StorageClass`. 

298 The delegate is constructed with this `StorageClass`. 

299 

300 Raises 

301 ------ 

302 TypeError 

303 This StorageClass has no associated delegate. 

304 """ 

305 cls = self.delegateClass 

306 if cls is None: 

307 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

308 return cls(storageClass=self) 

309 

310 def isComposite(self) -> bool: 

311 """Return Boolean indicating whether this is a composite or not. 

312 

313 Returns 

314 ------- 

315 isComposite : `bool` 

316 `True` if this `StorageClass` is a composite, `False` 

317 otherwise. 

318 """ 

319 if self.components: 

320 return True 

321 return False 

322 

323 def _lookupNames(self) -> tuple[LookupKey, ...]: 

324 """Keys to use when looking up this DatasetRef in a configuration. 

325 

326 The names are returned in order of priority. 

327 

328 Returns 

329 ------- 

330 names : `tuple` of `LookupKey` 

331 Tuple of a `LookupKey` using the `StorageClass` name. 

332 """ 

333 return (LookupKey(name=self.name),) 

334 

335 def knownParameters(self) -> set[str]: 

336 """Return set of all parameters known to this `StorageClass`. 

337 

338 The set includes parameters understood by components of a composite. 

339 

340 Returns 

341 ------- 

342 known : `set` 

343 All parameter keys of this `StorageClass` and the component 

344 storage classes. 

345 """ 

346 known = set(self._parameters) 

347 for sc in self.components.values(): 

348 known.update(sc.knownParameters()) 

349 return known 

350 

351 def validateParameters(self, parameters: Collection | None = None) -> None: 

352 """Check that the parameters are known to this `StorageClass`. 

353 

354 Does not check the values. 

355 

356 Parameters 

357 ---------- 

358 parameters : `~collections.abc.Collection`, optional 

359 Collection containing the parameters. Can be `dict`-like or 

360 `set`-like. The parameter values are not checked. 

361 If no parameters are supplied, always returns without error. 

362 

363 Raises 

364 ------ 

365 KeyError 

366 Some parameters are not understood by this `StorageClass`. 

367 """ 

368 # No parameters is always okay 

369 if not parameters: 

370 return 

371 

372 # Extract the important information into a set. Works for dict and 

373 # list. 

374 external = set(parameters) 

375 

376 diff = external - self.knownParameters() 

377 if diff: 

378 s = "s" if len(diff) > 1 else "" 

379 unknown = "', '".join(diff) 

380 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

381 

382 def filterParameters( 

383 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

384 ) -> Mapping[str, Any]: 

385 """Filter out parameters that are not known to this `StorageClass`. 

386 

387 Parameters 

388 ---------- 

389 parameters : `~collections.abc.Mapping`, optional 

390 Candidate parameters. Can be `None` if no parameters have 

391 been provided. 

392 subset : `~collections.abc.Collection`, optional 

393 Subset of supported parameters that the caller is interested 

394 in using. The subset must be known to the `StorageClass` 

395 if specified. If `None` the supplied parameters will all 

396 be checked, else only the keys in this set will be checked. 

397 

398 Returns 

399 ------- 

400 filtered : `~collections.abc.Mapping` 

401 Valid parameters. Empty `dict` if none are suitable. 

402 

403 Raises 

404 ------ 

405 ValueError 

406 Raised if the provided subset is not a subset of the supported 

407 parameters or if it is an empty set. 

408 """ 

409 if not parameters: 

410 return {} 

411 

412 known = self.knownParameters() 

413 

414 if subset is not None: 

415 if not subset: 

416 raise ValueError("Specified a parameter subset but it was empty") 

417 subset = set(subset) 

418 if not subset.issubset(known): 

419 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

420 wanted = subset 

421 else: 

422 wanted = known 

423 

424 return {k: parameters[k] for k in wanted if k in parameters} 

425 

426 def validateInstance(self, instance: Any) -> bool: 

427 """Check that the supplied Python object has the expected Python type. 

428 

429 Parameters 

430 ---------- 

431 instance : `object` 

432 Object to check. 

433 

434 Returns 

435 ------- 

436 isOk : `bool` 

437 True if the supplied instance object can be handled by this 

438 `StorageClass`, False otherwise. 

439 """ 

440 return isinstance(instance, self.pytype) 

441 

442 def is_type(self, other: type, compare_types: bool = False) -> bool: 

443 """Return Boolean indicating whether the supplied type matches 

444 the type in this `StorageClass`. 

445 

446 Parameters 

447 ---------- 

448 other : `type` 

449 The type to be checked. 

450 compare_types : `bool`, optional 

451 If `True` the python type will be used in the comparison 

452 if the type names do not match. This may trigger an import 

453 of code and so can be slower. 

454 

455 Returns 

456 ------- 

457 match : `bool` 

458 `True` if the types are equal. 

459 

460 Notes 

461 ----- 

462 If this `StorageClass` has not yet imported the Python type the 

463 check is done against the full type name, this prevents an attempt 

464 to import the type when it will likely not match. 

465 """ 

466 if self._pytype: 

467 return self._pytype is other 

468 

469 other_name = get_full_type_name(other) 

470 if self._pytypeName == other_name: 

471 return True 

472 

473 if compare_types: 

474 # Must protect against the import failing. 

475 try: 

476 return self.pytype is other 

477 except Exception: 

478 pass 

479 

480 return False 

481 

482 def can_convert(self, other: StorageClass) -> bool: 

483 """Return `True` if this storage class can convert python types 

484 in the other storage class. 

485 

486 Parameters 

487 ---------- 

488 other : `StorageClass` 

489 The storage class to check. 

490 

491 Returns 

492 ------- 

493 can : `bool` 

494 `True` if this storage class has a registered converter for 

495 the python type associated with the other storage class. That 

496 converter will convert the other python type to the one associated 

497 with this storage class. 

498 """ 

499 if other.name == self.name: 

500 # Identical storage classes are compatible. 

501 return True 

502 

503 # It may be that the storage class being compared is not 

504 # available because the python type can't be imported. In that 

505 # case conversion must be impossible. 

506 try: 

507 other_pytype = other.pytype 

508 except Exception: 

509 return False 

510 

511 # Or even this storage class itself can not have the type imported. 

512 try: 

513 self_pytype = self.pytype 

514 except Exception: 

515 return False 

516 

517 if issubclass(other_pytype, self_pytype): 

518 # Storage classes have different names but the same python type. 

519 return True 

520 

521 for candidate_type in self._get_converters_by_type(): 

522 if issubclass(other_pytype, candidate_type): 

523 return True 

524 return False 

525 

526 def coerce_type(self, incorrect: Any) -> Any: 

527 """Coerce the supplied incorrect instance to the python type 

528 associated with this `StorageClass`. 

529 

530 Parameters 

531 ---------- 

532 incorrect : `object` 

533 An object that might be the incorrect type. 

534 

535 Returns 

536 ------- 

537 correct : `object` 

538 An object that matches the python type of this `StorageClass`. 

539 Can be the same object as given. If `None`, `None` will be 

540 returned. 

541 

542 Raises 

543 ------ 

544 TypeError 

545 Raised if no conversion can be found. 

546 """ 

547 if incorrect is None: 

548 return None 

549 

550 # Possible this is the correct type already. 

551 if self.validateInstance(incorrect): 

552 return incorrect 

553 

554 # Check each registered converter. 

555 for candidate_type, converter in self._get_converters_by_type().items(): 

556 if isinstance(incorrect, candidate_type): 

557 try: 

558 return converter(incorrect) 

559 except Exception: 

560 log.error( 

561 "Converter %s failed to convert type %s", 

562 get_full_type_name(converter), 

563 get_full_type_name(incorrect), 

564 ) 

565 raise 

566 raise TypeError( 

567 "Type does not match and no valid converter found to convert" 

568 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

569 ) 

570 

571 def __eq__(self, other: Any) -> bool: 

572 """Equality checks name, pytype name, delegate name, and components.""" 

573 if not isinstance(other, StorageClass): 

574 return NotImplemented 

575 

576 if self.name != other.name: 

577 return False 

578 

579 # We must compare pytype and delegate by name since we do not want 

580 # to trigger an import of external module code here 

581 if self._delegateClassName != other._delegateClassName: 

582 return False 

583 if self._pytypeName != other._pytypeName: 

584 return False 

585 

586 # Ensure we have the same component keys in each 

587 if set(self.components.keys()) != set(other.components.keys()): 

588 return False 

589 

590 # Same parameters 

591 if self.parameters != other.parameters: 

592 return False 

593 

594 # Ensure that all the components have the same type 

595 return all(self.components[k] == other.components[k] for k in self.components) 

596 

597 def __hash__(self) -> int: 

598 return hash(self.name) 

599 

600 def __repr__(self) -> str: 

601 optionals: dict[str, Any] = {} 

602 if self._pytypeName != "object": 

603 optionals["pytype"] = self._pytypeName 

604 if self._delegateClassName is not None: 

605 optionals["delegate"] = self._delegateClassName 

606 if self._parameters: 

607 optionals["parameters"] = self._parameters 

608 if self.components: 

609 optionals["components"] = self.components 

610 if self.converters: 

611 optionals["converters"] = self.converters 

612 

613 # order is preserved in the dict 

614 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

615 

616 # Start with mandatory fields 

617 r = f"{self.__class__.__name__}({self.name!r}" 

618 if options: 

619 r = r + ", " + options 

620 r = r + ")" 

621 return r 

622 

623 def __str__(self) -> str: 

624 return self.name 

625 

626 

627class StorageClassFactory(metaclass=Singleton): 

628 """Factory for `StorageClass` instances. 

629 

630 This class is a singleton, with each instance sharing the pool of 

631 StorageClasses. Since code can not know whether it is the first 

632 time the instance has been created, the constructor takes no arguments. 

633 To populate the factory with storage classes, a call to 

634 `~StorageClassFactory.addFromConfig()` should be made. 

635 

636 Parameters 

637 ---------- 

638 config : `StorageClassConfig` or `str`, optional 

639 Load configuration. In a ButlerConfig` the relevant configuration 

640 is located in the ``storageClasses`` section. 

641 """ 

642 

643 def __init__(self, config: StorageClassConfig | str | None = None): 

644 self._storageClasses: dict[str, StorageClass] = {} 

645 self._configs: list[StorageClassConfig] = [] 

646 

647 # Always seed with the default config 

648 self.addFromConfig(StorageClassConfig()) 

649 

650 if config is not None: 650 ↛ 651line 650 didn't jump to line 651, because the condition on line 650 was never true

651 self.addFromConfig(config) 

652 

653 def __str__(self) -> str: 

654 """Return summary of factory. 

655 

656 Returns 

657 ------- 

658 summary : `str` 

659 Summary of the factory status. 

660 """ 

661 sep = "\n" 

662 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

663 

664StorageClasses 

665-------------- 

666{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

667""" 

668 

669 def __contains__(self, storageClassOrName: StorageClass | str) -> bool: 

670 """Indicate whether the storage class exists in the factory. 

671 

672 Parameters 

673 ---------- 

674 storageClassOrName : `str` or `StorageClass` 

675 If `str` is given existence of the named StorageClass 

676 in the factory is checked. If `StorageClass` is given 

677 existence and equality are checked. 

678 

679 Returns 

680 ------- 

681 in : `bool` 

682 True if the supplied string is present, or if the supplied 

683 `StorageClass` is present and identical. 

684 

685 Notes 

686 ----- 

687 The two different checks (one for "key" and one for "value") based on 

688 the type of the given argument mean that it is possible for 

689 StorageClass.name to be in the factory but StorageClass to not be 

690 in the factory. 

691 """ 

692 if isinstance(storageClassOrName, str): 692 ↛ 694line 692 didn't jump to line 694, because the condition on line 692 was never false

693 return storageClassOrName in self._storageClasses 

694 elif isinstance(storageClassOrName, StorageClass) and storageClassOrName.name in self._storageClasses: 

695 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

696 return False 

697 

698 def __len__(self) -> int: 

699 return len(self._storageClasses) 

700 

701 def __iter__(self) -> Iterator[str]: 

702 return iter(self._storageClasses) 

703 

704 def values(self) -> ValuesView[StorageClass]: 

705 return self._storageClasses.values() 

706 

707 def keys(self) -> KeysView[str]: 

708 return self._storageClasses.keys() 

709 

710 def items(self) -> ItemsView[str, StorageClass]: 

711 return self._storageClasses.items() 

712 

713 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

714 """Add more `StorageClass` definitions from a config file. 

715 

716 Parameters 

717 ---------- 

718 config : `StorageClassConfig`, `Config` or `str` 

719 Storage class configuration. Can contain a ``storageClasses`` 

720 key if part of a global configuration. 

721 """ 

722 sconfig = StorageClassConfig(config) 

723 self._configs.append(sconfig) 

724 

725 # Since we can not assume that we will get definitions of 

726 # components or parents before their classes are defined 

727 # we have a helper function that we can call recursively 

728 # to extract definitions from the configuration. 

729 def processStorageClass(name: str, sconfig: StorageClassConfig, msg: str = "") -> None: 

730 # Maybe we've already processed this through recursion 

731 if name not in sconfig: 

732 return 

733 info = sconfig.pop(name) 

734 

735 # Always create the storage class so we can ensure that 

736 # we are not trying to overwrite with a different definition 

737 components = None 

738 

739 # Extract scalar items from dict that are needed for 

740 # StorageClass Constructor 

741 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

742 

743 if "converters" in info: 

744 storageClassKwargs["converters"] = info["converters"].toDict() 

745 

746 for compName in ("components", "derivedComponents"): 

747 if compName not in info: 

748 continue 

749 components = {} 

750 for cname, ctype in info[compName].items(): 

751 if ctype not in self: 

752 processStorageClass(ctype, sconfig, msg) 

753 components[cname] = self.getStorageClass(ctype) 

754 

755 # Fill in other items 

756 storageClassKwargs[compName] = components 

757 

758 # Create the new storage class and register it 

759 baseClass = None 

760 if "inheritsFrom" in info: 

761 baseName = info["inheritsFrom"] 

762 if baseName not in self: 762 ↛ 763line 762 didn't jump to line 763, because the condition on line 762 was never true

763 processStorageClass(baseName, sconfig, msg) 

764 baseClass = type(self.getStorageClass(baseName)) 

765 

766 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

767 newStorageClass = newStorageClassType() 

768 self.registerStorageClass(newStorageClass, msg=msg) 

769 

770 # In case there is a problem, construct a context message for any 

771 # error reporting. 

772 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

773 context = f"when adding definitions from {', '.join(files)}" if files else "" 

774 log.debug("Adding definitions from config %s", ", ".join(files)) 

775 

776 for name in list(sconfig.keys()): 

777 processStorageClass(name, sconfig, context) 

778 

779 @staticmethod 

780 def makeNewStorageClass( 

781 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any 

782 ) -> type[StorageClass]: 

783 """Create a new Python class as a subclass of `StorageClass`. 

784 

785 Parameters 

786 ---------- 

787 name : `str` 

788 Name to use for this class. 

789 baseClass : `type`, optional 

790 Base class for this `StorageClass`. Must be either `StorageClass` 

791 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

792 be used. 

793 

794 Returns 

795 ------- 

796 newtype : `type` subclass of `StorageClass` 

797 Newly created Python type. 

798 """ 

799 if baseClass is None: 

800 baseClass = StorageClass 

801 if not issubclass(baseClass, StorageClass): 801 ↛ 802line 801 didn't jump to line 802, because the condition on line 801 was never true

802 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

803 

804 # convert the arguments to use different internal names 

805 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

806 clsargs["_cls_name"] = name 

807 

808 # Some container items need to merge with the base class values 

809 # so that a child can inherit but override one bit. 

810 # lists (which you get from configs) are treated as sets for this to 

811 # work consistently. 

812 for k in ("components", "parameters", "derivedComponents", "converters"): 

813 classKey = f"_cls_{k}" 

814 if classKey in clsargs: 

815 baseValue = getattr(baseClass, classKey, None) 

816 if baseValue is not None: 

817 currentValue = clsargs[classKey] 

818 if isinstance(currentValue, dict): 818 ↛ 821line 818 didn't jump to line 821, because the condition on line 818 was never false

819 newValue = baseValue.copy() 

820 else: 

821 newValue = set(baseValue) 

822 newValue.update(currentValue) 

823 clsargs[classKey] = newValue 

824 

825 # If we have parameters they should be a frozen set so that the 

826 # parameters in the class can not be modified. 

827 pk = "_cls_parameters" 

828 if pk in clsargs: 

829 clsargs[pk] = frozenset(clsargs[pk]) 

830 

831 return type(f"StorageClass{name}", (baseClass,), clsargs) 

832 

833 def getStorageClass(self, storageClassName: str) -> StorageClass: 

834 """Get a StorageClass instance associated with the supplied name. 

835 

836 Parameters 

837 ---------- 

838 storageClassName : `str` 

839 Name of the storage class to retrieve. 

840 

841 Returns 

842 ------- 

843 instance : `StorageClass` 

844 Instance of the correct `StorageClass`. 

845 

846 Raises 

847 ------ 

848 KeyError 

849 The requested storage class name is not registered. 

850 """ 

851 return self._storageClasses[storageClassName] 

852 

853 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

854 """Find the storage class associated with this python type. 

855 

856 Parameters 

857 ---------- 

858 pytype : `type` 

859 The Python type to be matched. 

860 compare_types : `bool`, optional 

861 If `False`, the type will be checked against name of the python 

862 type. This comparison is always done first. If `True` and the 

863 string comparison failed, each candidate storage class will be 

864 forced to have its type imported. This can be significantly slower. 

865 

866 Returns 

867 ------- 

868 storageClass : `StorageClass` 

869 The matching storage class. 

870 

871 Raises 

872 ------ 

873 KeyError 

874 Raised if no match could be found. 

875 

876 Notes 

877 ----- 

878 It is possible for a python type to be associated with multiple 

879 storage classes. This method will currently return the first that 

880 matches. 

881 """ 

882 result = self._find_storage_class(pytype, False) 

883 if result: 

884 return result 

885 

886 if compare_types: 

887 # The fast comparison failed and we were asked to try the 

888 # variant that might involve code imports. 

889 result = self._find_storage_class(pytype, True) 

890 if result: 

891 return result 

892 

893 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}") 

894 

895 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

896 """Iterate through all storage classes to find a match. 

897 

898 Parameters 

899 ---------- 

900 pytype : `type` 

901 The Python type to be matched. 

902 compare_types : `bool`, optional 

903 Whether to use type name matching or explicit type matching. 

904 The latter can be slower. 

905 

906 Returns 

907 ------- 

908 storageClass : `StorageClass` or `None` 

909 The matching storage class, or `None` if no match was found. 

910 

911 Notes 

912 ----- 

913 Helper method for ``findStorageClass``. 

914 """ 

915 for storageClass in self.values(): 

916 if storageClass.is_type(pytype, compare_types=compare_types): 

917 return storageClass 

918 return None 

919 

920 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

921 """Store the `StorageClass` in the factory. 

922 

923 Will be indexed by `StorageClass.name` and will return instances 

924 of the supplied `StorageClass`. 

925 

926 Parameters 

927 ---------- 

928 storageClass : `StorageClass` 

929 Type of the Python `StorageClass` to register. 

930 msg : `str`, optional 

931 Additional message string to be included in any error message. 

932 

933 Raises 

934 ------ 

935 ValueError 

936 If a storage class has already been registered with 

937 that storage class name and the previous definition differs. 

938 """ 

939 if storageClass.name in self._storageClasses: 939 ↛ 940line 939 didn't jump to line 940, because the condition on line 939 was never true

940 existing = self.getStorageClass(storageClass.name) 

941 if existing != storageClass: 

942 errmsg = f" {msg}" if msg else "" 

943 raise ValueError( 

944 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

945 f"differs from current definition ({existing!r}){errmsg}" 

946 ) 

947 else: 

948 self._storageClasses[storageClass.name] = storageClass 

949 

950 def _unregisterStorageClass(self, storageClassName: str) -> None: 

951 """Remove the named StorageClass from the factory. 

952 

953 Parameters 

954 ---------- 

955 storageClassName : `str` 

956 Name of storage class to remove. 

957 

958 Raises 

959 ------ 

960 KeyError 

961 The named storage class is not registered. 

962 

963 Notes 

964 ----- 

965 This method is intended to simplify testing of StorageClassFactory 

966 functionality and it is not expected to be required for normal usage. 

967 """ 

968 del self._storageClasses[storageClassName]