Coverage for python/lsst/daf/butler/core/storageClass.py: 43%

361 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 09:13 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Support for Storage Classes.""" 

25 

26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

27 

28import builtins 

29import itertools 

30import logging 

31from collections import ChainMap 

32from collections.abc import ( 

33 Callable, 

34 Collection, 

35 ItemsView, 

36 Iterator, 

37 KeysView, 

38 Mapping, 

39 Sequence, 

40 Set, 

41 ValuesView, 

42) 

43from typing import Any 

44 

45from lsst.utils import doImportType 

46from lsst.utils.classes import Singleton 

47from lsst.utils.introspection import get_full_type_name 

48 

49from .config import Config, ConfigSubset 

50from .configSupport import LookupKey 

51from .storageClassDelegate import StorageClassDelegate 

52 

53log = logging.getLogger(__name__) 

54 

55 

56class StorageClassConfig(ConfigSubset): 

57 """Configuration class for defining Storage Classes.""" 

58 

59 component = "storageClasses" 

60 defaultConfigFile = "storageClasses.yaml" 

61 

62 

63class StorageClass: 

64 """Class describing how a label maps to a particular Python type. 

65 

66 Parameters 

67 ---------- 

68 name : `str` 

69 Name to use for this class. 

70 pytype : `type` or `str` 

71 Python type (or name of type) to associate with the `StorageClass` 

72 components : `dict`, optional 

73 `dict` mapping name of a component to another `StorageClass`. 

74 derivedComponents : `dict`, optional 

75 `dict` mapping name of a derived component to another `StorageClass`. 

76 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

77 Parameters understood by this `StorageClass` that can control 

78 reading of data from datastores. 

79 delegate : `str`, optional 

80 Fully qualified name of class supporting assembly and disassembly 

81 of a `pytype` instance. 

82 converters : `dict` [`str`, `str`], optional 

83 Mapping of python type to function that can be called to convert 

84 that python type to the valid type of this storage class. 

85 """ 

86 

87 _cls_name: str = "BaseStorageClass" 

88 _cls_components: dict[str, StorageClass] | None = None 

89 _cls_derivedComponents: dict[str, StorageClass] | None = None 

90 _cls_parameters: Set[str] | Sequence[str] | None = None 

91 _cls_delegate: str | None = None 

92 _cls_pytype: type | str | None = None 

93 _cls_converters: dict[str, str] | None = None 

94 defaultDelegate: type = StorageClassDelegate 

95 defaultDelegateName: str = get_full_type_name(defaultDelegate) 

96 

97 def __init__( 

98 self, 

99 name: str | None = None, 

100 pytype: type | str | None = None, 

101 components: dict[str, StorageClass] | None = None, 

102 derivedComponents: dict[str, StorageClass] | None = None, 

103 parameters: Sequence[str] | Set[str] | None = None, 

104 delegate: str | None = None, 

105 converters: dict[str, str] | None = None, 

106 ): 

107 if name is None: 

108 name = self._cls_name 

109 if pytype is None: 109 ↛ 111line 109 didn't jump to line 111, because the condition on line 109 was never false

110 pytype = self._cls_pytype 

111 if components is None: 111 ↛ 113line 111 didn't jump to line 113, because the condition on line 111 was never false

112 components = self._cls_components 

113 if derivedComponents is None: 113 ↛ 115line 113 didn't jump to line 115, because the condition on line 113 was never false

114 derivedComponents = self._cls_derivedComponents 

115 if parameters is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false

116 parameters = self._cls_parameters 

117 if delegate is None: 117 ↛ 121line 117 didn't jump to line 121, because the condition on line 117 was never false

118 delegate = self._cls_delegate 

119 

120 # Merge converters with class defaults. 

121 self._converters = {} 

122 if self._cls_converters is not None: 

123 self._converters.update(self._cls_converters) 

124 if converters: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 self._converters.update(converters) 

126 

127 # Version of converters where the python types have been 

128 # Do not try to import anything until needed. 

129 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

130 

131 self.name = name 

132 

133 if pytype is None: 

134 pytype = object 

135 

136 self._pytype: type | None 

137 if not isinstance(pytype, str): 

138 # Already have a type so store it and get the name 

139 self._pytypeName = get_full_type_name(pytype) 

140 self._pytype = pytype 

141 else: 

142 # Store the type name and defer loading of type 

143 self._pytypeName = pytype 

144 self._pytype = None 

145 

146 if components is not None: 

147 if len(components) == 1: 147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true

148 raise ValueError( 

149 f"Composite storage class {name} is not allowed to have" 

150 f" only one component '{next(iter(components))}'." 

151 " Did you mean it to be a derived component?" 

152 ) 

153 self._components = components 

154 else: 

155 self._components = {} 

156 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

157 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

158 # if the delegate is not None also set it and clear the default 

159 # delegate 

160 self._delegate: type | None 

161 self._delegateClassName: str | None 

162 if delegate is not None: 

163 self._delegateClassName = delegate 

164 self._delegate = None 

165 elif components is not None: 165 ↛ 168line 165 didn't jump to line 168, because the condition on line 165 was never true

166 # We set a default delegate for composites so that a class is 

167 # guaranteed to support something if it is a composite. 

168 log.debug("Setting default delegate for %s", self.name) 

169 self._delegate = self.defaultDelegate 

170 self._delegateClassName = self.defaultDelegateName 

171 else: 

172 self._delegate = None 

173 self._delegateClassName = None 

174 

175 @property 

176 def components(self) -> Mapping[str, StorageClass]: 

177 """Return the components associated with this `StorageClass`.""" 

178 return self._components 

179 

180 @property 

181 def derivedComponents(self) -> Mapping[str, StorageClass]: 

182 """Return derived components associated with `StorageClass`.""" 

183 return self._derivedComponents 

184 

185 @property 

186 def converters(self) -> Mapping[str, str]: 

187 """Return the type converters supported by this `StorageClass`.""" 

188 return self._converters 

189 

190 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

191 """Return the type converters as python types.""" 

192 if self._converters_by_type is None: 

193 self._converters_by_type = {} 

194 

195 # Loop over list because the dict can be edited in loop. 

196 for candidate_type_str, converter_str in list(self.converters.items()): 

197 if hasattr(builtins, candidate_type_str): 

198 candidate_type = getattr(builtins, candidate_type_str) 

199 else: 

200 try: 

201 candidate_type = doImportType(candidate_type_str) 

202 except ImportError as e: 

203 log.warning( 

204 "Unable to import type %s associated with storage class %s (%s)", 

205 candidate_type_str, 

206 self.name, 

207 e, 

208 ) 

209 del self._converters[candidate_type_str] 

210 continue 

211 

212 try: 

213 converter = doImportType(converter_str) 

214 except ImportError as e: 

215 log.warning( 

216 "Unable to import conversion function %s associated with storage class %s " 

217 "required to convert type %s (%s)", 

218 converter_str, 

219 self.name, 

220 candidate_type_str, 

221 e, 

222 ) 

223 del self._converters[candidate_type_str] 

224 continue 

225 if not callable(converter): 

226 # doImportType is annotated to return a Type but in actual 

227 # fact it can return Any except ModuleType because package 

228 # variables can be accessed. This make mypy believe it 

229 # is impossible for the return value to not be a callable 

230 # so we must ignore the warning. 

231 log.warning( # type: ignore 

232 "Conversion function %s associated with storage class " 

233 "%s to convert type %s is not a callable.", 

234 converter_str, 

235 self.name, 

236 candidate_type_str, 

237 ) 

238 del self._converters[candidate_type_str] 

239 continue 

240 self._converters_by_type[candidate_type] = converter 

241 return self._converters_by_type 

242 

243 @property 

244 def parameters(self) -> set[str]: 

245 """Return `set` of names of supported parameters.""" 

246 return set(self._parameters) 

247 

248 @property 

249 def pytype(self) -> type: 

250 """Return Python type associated with this `StorageClass`.""" 

251 if self._pytype is not None: 

252 return self._pytype 

253 

254 if hasattr(builtins, self._pytypeName): 

255 pytype = getattr(builtins, self._pytypeName) 

256 else: 

257 pytype = doImportType(self._pytypeName) 

258 self._pytype = pytype 

259 return self._pytype 

260 

261 @property 

262 def delegateClass(self) -> type | None: 

263 """Class to use to delegate type-specific actions.""" 

264 if self._delegate is not None: 

265 return self._delegate 

266 if self._delegateClassName is None: 

267 return None 

268 delegate_class = doImportType(self._delegateClassName) 

269 self._delegate = delegate_class 

270 return self._delegate 

271 

272 def allComponents(self) -> Mapping[str, StorageClass]: 

273 """Return all defined components. 

274 

275 This mapping includes all the derived and read/write components 

276 for the corresponding storage class. 

277 

278 Returns 

279 ------- 

280 comp : `dict` of [`str`, `StorageClass`] 

281 The component name to storage class mapping. 

282 """ 

283 return ChainMap(self._components, self._derivedComponents) 

284 

285 def delegate(self) -> StorageClassDelegate: 

286 """Return an instance of a storage class delegate. 

287 

288 Returns 

289 ------- 

290 delegate : `StorageClassDelegate` 

291 Instance of the delegate associated with this `StorageClass`. 

292 The delegate is constructed with this `StorageClass`. 

293 

294 Raises 

295 ------ 

296 TypeError 

297 This StorageClass has no associated delegate. 

298 """ 

299 cls = self.delegateClass 

300 if cls is None: 

301 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

302 return cls(storageClass=self) 

303 

304 def isComposite(self) -> bool: 

305 """Return Boolean indicating whether this is a composite or not. 

306 

307 Returns 

308 ------- 

309 isComposite : `bool` 

310 `True` if this `StorageClass` is a composite, `False` 

311 otherwise. 

312 """ 

313 if self.components: 

314 return True 

315 return False 

316 

317 def _lookupNames(self) -> tuple[LookupKey, ...]: 

318 """Keys to use when looking up this DatasetRef in a configuration. 

319 

320 The names are returned in order of priority. 

321 

322 Returns 

323 ------- 

324 names : `tuple` of `LookupKey` 

325 Tuple of a `LookupKey` using the `StorageClass` name. 

326 """ 

327 return (LookupKey(name=self.name),) 

328 

329 def knownParameters(self) -> set[str]: 

330 """Return set of all parameters known to this `StorageClass`. 

331 

332 The set includes parameters understood by components of a composite. 

333 

334 Returns 

335 ------- 

336 known : `set` 

337 All parameter keys of this `StorageClass` and the component 

338 storage classes. 

339 """ 

340 known = set(self._parameters) 

341 for sc in self.components.values(): 

342 known.update(sc.knownParameters()) 

343 return known 

344 

345 def validateParameters(self, parameters: Collection | None = None) -> None: 

346 """Check that the parameters are known to this `StorageClass`. 

347 

348 Does not check the values. 

349 

350 Parameters 

351 ---------- 

352 parameters : `~collections.abc.Collection`, optional 

353 Collection containing the parameters. Can be `dict`-like or 

354 `set`-like. The parameter values are not checked. 

355 If no parameters are supplied, always returns without error. 

356 

357 Raises 

358 ------ 

359 KeyError 

360 Some parameters are not understood by this `StorageClass`. 

361 """ 

362 # No parameters is always okay 

363 if not parameters: 

364 return 

365 

366 # Extract the important information into a set. Works for dict and 

367 # list. 

368 external = set(parameters) 

369 

370 diff = external - self.knownParameters() 

371 if diff: 

372 s = "s" if len(diff) > 1 else "" 

373 unknown = "', '".join(diff) 

374 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

375 

376 def filterParameters( 

377 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

378 ) -> Mapping[str, Any]: 

379 """Filter out parameters that are not known to this `StorageClass`. 

380 

381 Parameters 

382 ---------- 

383 parameters : `~collections.abc.Mapping`, optional 

384 Candidate parameters. Can be `None` if no parameters have 

385 been provided. 

386 subset : `~collections.abc.Collection`, optional 

387 Subset of supported parameters that the caller is interested 

388 in using. The subset must be known to the `StorageClass` 

389 if specified. If `None` the supplied parameters will all 

390 be checked, else only the keys in this set will be checked. 

391 

392 Returns 

393 ------- 

394 filtered : `~collections.abc.Mapping` 

395 Valid parameters. Empty `dict` if none are suitable. 

396 

397 Raises 

398 ------ 

399 ValueError 

400 Raised if the provided subset is not a subset of the supported 

401 parameters or if it is an empty set. 

402 """ 

403 if not parameters: 

404 return {} 

405 

406 known = self.knownParameters() 

407 

408 if subset is not None: 

409 if not subset: 

410 raise ValueError("Specified a parameter subset but it was empty") 

411 subset = set(subset) 

412 if not subset.issubset(known): 

413 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

414 wanted = subset 

415 else: 

416 wanted = known 

417 

418 return {k: parameters[k] for k in wanted if k in parameters} 

419 

420 def validateInstance(self, instance: Any) -> bool: 

421 """Check that the supplied Python object has the expected Python type. 

422 

423 Parameters 

424 ---------- 

425 instance : `object` 

426 Object to check. 

427 

428 Returns 

429 ------- 

430 isOk : `bool` 

431 True if the supplied instance object can be handled by this 

432 `StorageClass`, False otherwise. 

433 """ 

434 return isinstance(instance, self.pytype) 

435 

436 def is_type(self, other: type, compare_types: bool = False) -> bool: 

437 """Return Boolean indicating whether the supplied type matches 

438 the type in this `StorageClass`. 

439 

440 Parameters 

441 ---------- 

442 other : `type` 

443 The type to be checked. 

444 compare_types : `bool`, optional 

445 If `True` the python type will be used in the comparison 

446 if the type names do not match. This may trigger an import 

447 of code and so can be slower. 

448 

449 Returns 

450 ------- 

451 match : `bool` 

452 `True` if the types are equal. 

453 

454 Notes 

455 ----- 

456 If this `StorageClass` has not yet imported the Python type the 

457 check is done against the full type name, this prevents an attempt 

458 to import the type when it will likely not match. 

459 """ 

460 if self._pytype: 

461 return self._pytype is other 

462 

463 other_name = get_full_type_name(other) 

464 if self._pytypeName == other_name: 

465 return True 

466 

467 if compare_types: 

468 # Must protect against the import failing. 

469 try: 

470 return self.pytype is other 

471 except Exception: 

472 pass 

473 

474 return False 

475 

476 def can_convert(self, other: StorageClass) -> bool: 

477 """Return `True` if this storage class can convert python types 

478 in the other storage class. 

479 

480 Parameters 

481 ---------- 

482 other : `StorageClass` 

483 The storage class to check. 

484 

485 Returns 

486 ------- 

487 can : `bool` 

488 `True` if this storage class has a registered converter for 

489 the python type associated with the other storage class. That 

490 converter will convert the other python type to the one associated 

491 with this storage class. 

492 """ 

493 if other.name == self.name: 

494 # Identical storage classes are compatible. 

495 return True 

496 

497 # It may be that the storage class being compared is not 

498 # available because the python type can't be imported. In that 

499 # case conversion must be impossible. 

500 try: 

501 other_pytype = other.pytype 

502 except Exception: 

503 return False 

504 

505 # Or even this storage class itself can not have the type imported. 

506 try: 

507 self_pytype = self.pytype 

508 except Exception: 

509 return False 

510 

511 if issubclass(other_pytype, self_pytype): 

512 # Storage classes have different names but the same python type. 

513 return True 

514 

515 for candidate_type in self._get_converters_by_type(): 

516 if issubclass(other_pytype, candidate_type): 

517 return True 

518 return False 

519 

520 def coerce_type(self, incorrect: Any) -> Any: 

521 """Coerce the supplied incorrect instance to the python type 

522 associated with this `StorageClass`. 

523 

524 Parameters 

525 ---------- 

526 incorrect : `object` 

527 An object that might be the incorrect type. 

528 

529 Returns 

530 ------- 

531 correct : `object` 

532 An object that matches the python type of this `StorageClass`. 

533 Can be the same object as given. If `None`, `None` will be 

534 returned. 

535 

536 Raises 

537 ------ 

538 TypeError 

539 Raised if no conversion can be found. 

540 """ 

541 if incorrect is None: 

542 return None 

543 

544 # Possible this is the correct type already. 

545 if self.validateInstance(incorrect): 

546 return incorrect 

547 

548 # Check each registered converter. 

549 for candidate_type, converter in self._get_converters_by_type().items(): 

550 if isinstance(incorrect, candidate_type): 

551 try: 

552 return converter(incorrect) 

553 except Exception: 

554 log.error( 

555 "Converter %s failed to convert type %s", 

556 get_full_type_name(converter), 

557 get_full_type_name(incorrect), 

558 ) 

559 raise 

560 raise TypeError( 

561 "Type does not match and no valid converter found to convert" 

562 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

563 ) 

564 

565 def __eq__(self, other: Any) -> bool: 

566 """Equality checks name, pytype name, delegate name, and components.""" 

567 if not isinstance(other, StorageClass): 

568 return NotImplemented 

569 

570 if self.name != other.name: 

571 return False 

572 

573 # We must compare pytype and delegate by name since we do not want 

574 # to trigger an import of external module code here 

575 if self._delegateClassName != other._delegateClassName: 

576 return False 

577 if self._pytypeName != other._pytypeName: 

578 return False 

579 

580 # Ensure we have the same component keys in each 

581 if set(self.components.keys()) != set(other.components.keys()): 

582 return False 

583 

584 # Same parameters 

585 if self.parameters != other.parameters: 

586 return False 

587 

588 # Ensure that all the components have the same type 

589 for k in self.components: 

590 if self.components[k] != other.components[k]: 

591 return False 

592 

593 # If we got to this point everything checks out 

594 return True 

595 

596 def __hash__(self) -> int: 

597 return hash(self.name) 

598 

599 def __repr__(self) -> str: 

600 optionals: dict[str, Any] = {} 

601 if self._pytypeName != "object": 

602 optionals["pytype"] = self._pytypeName 

603 if self._delegateClassName is not None: 

604 optionals["delegate"] = self._delegateClassName 

605 if self._parameters: 

606 optionals["parameters"] = self._parameters 

607 if self.components: 

608 optionals["components"] = self.components 

609 if self.converters: 

610 optionals["converters"] = self.converters 

611 

612 # order is preserved in the dict 

613 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

614 

615 # Start with mandatory fields 

616 r = f"{self.__class__.__name__}({self.name!r}" 

617 if options: 

618 r = r + ", " + options 

619 r = r + ")" 

620 return r 

621 

622 def __str__(self) -> str: 

623 return self.name 

624 

625 

626class StorageClassFactory(metaclass=Singleton): 

627 """Factory for `StorageClass` instances. 

628 

629 This class is a singleton, with each instance sharing the pool of 

630 StorageClasses. Since code can not know whether it is the first 

631 time the instance has been created, the constructor takes no arguments. 

632 To populate the factory with storage classes, a call to 

633 `~StorageClassFactory.addFromConfig()` should be made. 

634 

635 Parameters 

636 ---------- 

637 config : `StorageClassConfig` or `str`, optional 

638 Load configuration. In a ButlerConfig` the relevant configuration 

639 is located in the ``storageClasses`` section. 

640 """ 

641 

642 def __init__(self, config: StorageClassConfig | str | None = None): 

643 self._storageClasses: dict[str, StorageClass] = {} 

644 self._configs: list[StorageClassConfig] = [] 

645 

646 # Always seed with the default config 

647 self.addFromConfig(StorageClassConfig()) 

648 

649 if config is not None: 649 ↛ 650line 649 didn't jump to line 650, because the condition on line 649 was never true

650 self.addFromConfig(config) 

651 

652 def __str__(self) -> str: 

653 """Return summary of factory. 

654 

655 Returns 

656 ------- 

657 summary : `str` 

658 Summary of the factory status. 

659 """ 

660 sep = "\n" 

661 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

662 

663StorageClasses 

664-------------- 

665{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

666""" 

667 

668 def __contains__(self, storageClassOrName: StorageClass | str) -> bool: 

669 """Indicate whether the storage class exists in the factory. 

670 

671 Parameters 

672 ---------- 

673 storageClassOrName : `str` or `StorageClass` 

674 If `str` is given existence of the named StorageClass 

675 in the factory is checked. If `StorageClass` is given 

676 existence and equality are checked. 

677 

678 Returns 

679 ------- 

680 in : `bool` 

681 True if the supplied string is present, or if the supplied 

682 `StorageClass` is present and identical. 

683 

684 Notes 

685 ----- 

686 The two different checks (one for "key" and one for "value") based on 

687 the type of the given argument mean that it is possible for 

688 StorageClass.name to be in the factory but StorageClass to not be 

689 in the factory. 

690 """ 

691 if isinstance(storageClassOrName, str): 691 ↛ 693line 691 didn't jump to line 693, because the condition on line 691 was never false

692 return storageClassOrName in self._storageClasses 

693 elif isinstance(storageClassOrName, StorageClass): 

694 if storageClassOrName.name in self._storageClasses: 

695 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

696 return False 

697 

698 def __len__(self) -> int: 

699 return len(self._storageClasses) 

700 

701 def __iter__(self) -> Iterator[str]: 

702 return iter(self._storageClasses) 

703 

704 def values(self) -> ValuesView[StorageClass]: 

705 return self._storageClasses.values() 

706 

707 def keys(self) -> KeysView[str]: 

708 return self._storageClasses.keys() 

709 

710 def items(self) -> ItemsView[str, StorageClass]: 

711 return self._storageClasses.items() 

712 

713 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

714 """Add more `StorageClass` definitions from a config file. 

715 

716 Parameters 

717 ---------- 

718 config : `StorageClassConfig`, `Config` or `str` 

719 Storage class configuration. Can contain a ``storageClasses`` 

720 key if part of a global configuration. 

721 """ 

722 sconfig = StorageClassConfig(config) 

723 self._configs.append(sconfig) 

724 

725 # Since we can not assume that we will get definitions of 

726 # components or parents before their classes are defined 

727 # we have a helper function that we can call recursively 

728 # to extract definitions from the configuration. 

729 def processStorageClass(name: str, sconfig: StorageClassConfig, msg: str = "") -> None: 

730 # Maybe we've already processed this through recursion 

731 if name not in sconfig: 

732 return 

733 info = sconfig.pop(name) 

734 

735 # Always create the storage class so we can ensure that 

736 # we are not trying to overwrite with a different definition 

737 components = None 

738 

739 # Extract scalar items from dict that are needed for 

740 # StorageClass Constructor 

741 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

742 

743 if "converters" in info: 

744 storageClassKwargs["converters"] = info["converters"].toDict() 

745 

746 for compName in ("components", "derivedComponents"): 

747 if compName not in info: 

748 continue 

749 components = {} 

750 for cname, ctype in info[compName].items(): 

751 if ctype not in self: 

752 processStorageClass(ctype, sconfig, msg) 

753 components[cname] = self.getStorageClass(ctype) 

754 

755 # Fill in other items 

756 storageClassKwargs[compName] = components 

757 

758 # Create the new storage class and register it 

759 baseClass = None 

760 if "inheritsFrom" in info: 

761 baseName = info["inheritsFrom"] 

762 if baseName not in self: 762 ↛ 763line 762 didn't jump to line 763, because the condition on line 762 was never true

763 processStorageClass(baseName, sconfig, msg) 

764 baseClass = type(self.getStorageClass(baseName)) 

765 

766 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

767 newStorageClass = newStorageClassType() 

768 self.registerStorageClass(newStorageClass, msg=msg) 

769 

770 # In case there is a problem, construct a context message for any 

771 # error reporting. 

772 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

773 context = f"when adding definitions from {', '.join(files)}" if files else "" 

774 log.debug("Adding definitions from config %s", ", ".join(files)) 

775 

776 for name in list(sconfig.keys()): 

777 processStorageClass(name, sconfig, context) 

778 

779 @staticmethod 

780 def makeNewStorageClass( 

781 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any 

782 ) -> type[StorageClass]: 

783 """Create a new Python class as a subclass of `StorageClass`. 

784 

785 Parameters 

786 ---------- 

787 name : `str` 

788 Name to use for this class. 

789 baseClass : `type`, optional 

790 Base class for this `StorageClass`. Must be either `StorageClass` 

791 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

792 be used. 

793 

794 Returns 

795 ------- 

796 newtype : `type` subclass of `StorageClass` 

797 Newly created Python type. 

798 """ 

799 if baseClass is None: 

800 baseClass = StorageClass 

801 if not issubclass(baseClass, StorageClass): 801 ↛ 802line 801 didn't jump to line 802, because the condition on line 801 was never true

802 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

803 

804 # convert the arguments to use different internal names 

805 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

806 clsargs["_cls_name"] = name 

807 

808 # Some container items need to merge with the base class values 

809 # so that a child can inherit but override one bit. 

810 # lists (which you get from configs) are treated as sets for this to 

811 # work consistently. 

812 for k in ("components", "parameters", "derivedComponents", "converters"): 

813 classKey = f"_cls_{k}" 

814 if classKey in clsargs: 

815 baseValue = getattr(baseClass, classKey, None) 

816 if baseValue is not None: 

817 currentValue = clsargs[classKey] 

818 if isinstance(currentValue, dict): 818 ↛ 821line 818 didn't jump to line 821, because the condition on line 818 was never false

819 newValue = baseValue.copy() 

820 else: 

821 newValue = set(baseValue) 

822 newValue.update(currentValue) 

823 clsargs[classKey] = newValue 

824 

825 # If we have parameters they should be a frozen set so that the 

826 # parameters in the class can not be modified. 

827 pk = "_cls_parameters" 

828 if pk in clsargs: 

829 clsargs[pk] = frozenset(clsargs[pk]) 

830 

831 return type(f"StorageClass{name}", (baseClass,), clsargs) 

832 

833 def getStorageClass(self, storageClassName: str) -> StorageClass: 

834 """Get a StorageClass instance associated with the supplied name. 

835 

836 Parameters 

837 ---------- 

838 storageClassName : `str` 

839 Name of the storage class to retrieve. 

840 

841 Returns 

842 ------- 

843 instance : `StorageClass` 

844 Instance of the correct `StorageClass`. 

845 

846 Raises 

847 ------ 

848 KeyError 

849 The requested storage class name is not registered. 

850 """ 

851 return self._storageClasses[storageClassName] 

852 

853 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

854 """Find the storage class associated with this python type. 

855 

856 Parameters 

857 ---------- 

858 pytype : `type` 

859 The Python type to be matched. 

860 compare_types : `bool`, optional 

861 If `False`, the type will be checked against name of the python 

862 type. This comparison is always done first. If `True` and the 

863 string comparison failed, each candidate storage class will be 

864 forced to have its type imported. This can be significantly slower. 

865 

866 Returns 

867 ------- 

868 storageClass : `StorageClass` 

869 The matching storage class. 

870 

871 Raises 

872 ------ 

873 KeyError 

874 Raised if no match could be found. 

875 

876 Notes 

877 ----- 

878 It is possible for a python type to be associated with multiple 

879 storage classes. This method will currently return the first that 

880 matches. 

881 """ 

882 result = self._find_storage_class(pytype, False) 

883 if result: 

884 return result 

885 

886 if compare_types: 

887 # The fast comparison failed and we were asked to try the 

888 # variant that might involve code imports. 

889 result = self._find_storage_class(pytype, True) 

890 if result: 

891 return result 

892 

893 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}") 

894 

895 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

896 """Iterate through all storage classes to find a match. 

897 

898 Parameters 

899 ---------- 

900 pytype : `type` 

901 The Python type to be matched. 

902 compare_types : `bool`, optional 

903 Whether to use type name matching or explicit type matching. 

904 The latter can be slower. 

905 

906 Returns 

907 ------- 

908 storageClass : `StorageClass` or `None` 

909 The matching storage class, or `None` if no match was found. 

910 

911 Notes 

912 ----- 

913 Helper method for ``findStorageClass``. 

914 """ 

915 for storageClass in self.values(): 

916 if storageClass.is_type(pytype, compare_types=compare_types): 

917 return storageClass 

918 return None 

919 

920 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

921 """Store the `StorageClass` in the factory. 

922 

923 Will be indexed by `StorageClass.name` and will return instances 

924 of the supplied `StorageClass`. 

925 

926 Parameters 

927 ---------- 

928 storageClass : `StorageClass` 

929 Type of the Python `StorageClass` to register. 

930 msg : `str`, optional 

931 Additional message string to be included in any error message. 

932 

933 Raises 

934 ------ 

935 ValueError 

936 If a storage class has already been registered with 

937 that storage class name and the previous definition differs. 

938 """ 

939 if storageClass.name in self._storageClasses: 939 ↛ 940line 939 didn't jump to line 940, because the condition on line 939 was never true

940 existing = self.getStorageClass(storageClass.name) 

941 if existing != storageClass: 

942 errmsg = f" {msg}" if msg else "" 

943 raise ValueError( 

944 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

945 f"differs from current definition ({existing!r}){errmsg}" 

946 ) 

947 else: 

948 self._storageClasses[storageClass.name] = storageClass 

949 

950 def _unregisterStorageClass(self, storageClassName: str) -> None: 

951 """Remove the named StorageClass from the factory. 

952 

953 Parameters 

954 ---------- 

955 storageClassName : `str` 

956 Name of storage class to remove. 

957 

958 Raises 

959 ------ 

960 KeyError 

961 The named storage class is not registered. 

962 

963 Notes 

964 ----- 

965 This method is intended to simplify testing of StorageClassFactory 

966 functionality and it is not expected to be required for normal usage. 

967 """ 

968 del self._storageClasses[storageClassName]