Coverage for python/lsst/daf/butler/_storage_class.py: 45%

366 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for Storage Classes.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

33 

34import builtins 

35import itertools 

36import logging 

37from collections import ChainMap 

38from collections.abc import ( 

39 Callable, 

40 Collection, 

41 ItemsView, 

42 Iterator, 

43 KeysView, 

44 Mapping, 

45 Sequence, 

46 Set, 

47 ValuesView, 

48) 

49from typing import Any 

50 

51from lsst.utils import doImportType 

52from lsst.utils.classes import Singleton 

53from lsst.utils.introspection import get_full_type_name 

54 

55from ._config import Config, ConfigSubset 

56from ._config_support import LookupKey 

57from ._storage_class_delegate import StorageClassDelegate 

58 

59log = logging.getLogger(__name__) 

60 

61 

62class StorageClassConfig(ConfigSubset): 

63 """Configuration class for defining Storage Classes.""" 

64 

65 component = "storageClasses" 

66 defaultConfigFile = "storageClasses.yaml" 

67 

68 

69class StorageClass: 

70 """Class describing how a label maps to a particular Python type. 

71 

72 Parameters 

73 ---------- 

74 name : `str` 

75 Name to use for this class. 

76 pytype : `type` or `str` 

77 Python type (or name of type) to associate with the `StorageClass` 

78 components : `dict`, optional 

79 `dict` mapping name of a component to another `StorageClass`. 

80 derivedComponents : `dict`, optional 

81 `dict` mapping name of a derived component to another `StorageClass`. 

82 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

83 Parameters understood by this `StorageClass` that can control 

84 reading of data from datastores. 

85 delegate : `str`, optional 

86 Fully qualified name of class supporting assembly and disassembly 

87 of a `pytype` instance. 

88 converters : `dict` [`str`, `str`], optional 

89 Mapping of python type to function that can be called to convert 

90 that python type to the valid type of this storage class. 

91 """ 

92 

93 _cls_name: str = "BaseStorageClass" 

94 _cls_components: dict[str, StorageClass] | None = None 

95 _cls_derivedComponents: dict[str, StorageClass] | None = None 

96 _cls_parameters: Set[str] | Sequence[str] | None = None 

97 _cls_delegate: str | None = None 

98 _cls_pytype: type | str | None = None 

99 _cls_converters: dict[str, str] | None = None 

100 

101 def __init__( 

102 self, 

103 name: str | None = None, 

104 pytype: type | str | None = None, 

105 components: dict[str, StorageClass] | None = None, 

106 derivedComponents: dict[str, StorageClass] | None = None, 

107 parameters: Sequence[str] | Set[str] | None = None, 

108 delegate: str | None = None, 

109 converters: dict[str, str] | None = None, 

110 ): 

111 if name is None: 

112 name = self._cls_name 

113 if pytype is None: 113 ↛ 115line 113 didn't jump to line 115, because the condition on line 113 was never false

114 pytype = self._cls_pytype 

115 if components is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false

116 components = self._cls_components 

117 if derivedComponents is None: 117 ↛ 119line 117 didn't jump to line 119, because the condition on line 117 was never false

118 derivedComponents = self._cls_derivedComponents 

119 if parameters is None: 119 ↛ 121line 119 didn't jump to line 121, because the condition on line 119 was never false

120 parameters = self._cls_parameters 

121 if delegate is None: 121 ↛ 125line 121 didn't jump to line 125, because the condition on line 121 was never false

122 delegate = self._cls_delegate 

123 

124 # Merge converters with class defaults. 

125 self._converters = {} 

126 if self._cls_converters is not None: 

127 self._converters.update(self._cls_converters) 

128 if converters: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true

129 self._converters.update(converters) 

130 

131 # Version of converters where the python types have been 

132 # Do not try to import anything until needed. 

133 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

134 

135 self.name = name 

136 

137 if pytype is None: 

138 pytype = object 

139 

140 self._pytype: type | None 

141 if not isinstance(pytype, str): 

142 # Already have a type so store it and get the name 

143 self._pytypeName = get_full_type_name(pytype) 

144 self._pytype = pytype 

145 else: 

146 # Store the type name and defer loading of type 

147 self._pytypeName = pytype 

148 self._pytype = None 

149 

150 if components is not None: 

151 if len(components) == 1: 151 ↛ 152line 151 didn't jump to line 152, because the condition on line 151 was never true

152 raise ValueError( 

153 f"Composite storage class {name} is not allowed to have" 

154 f" only one component '{next(iter(components))}'." 

155 " Did you mean it to be a derived component?" 

156 ) 

157 self._components = components 

158 else: 

159 self._components = {} 

160 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

161 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

162 # if the delegate is not None also set it and clear the default 

163 # delegate 

164 self._delegate: type | None 

165 self._delegateClassName: str | None 

166 if delegate is not None: 

167 self._delegateClassName = delegate 

168 self._delegate = None 

169 elif components is not None: 169 ↛ 172line 169 didn't jump to line 172, because the condition on line 169 was never true

170 # We set a default delegate for composites so that a class is 

171 # guaranteed to support something if it is a composite. 

172 log.debug("Setting default delegate for %s", self.name) 

173 self._delegate = StorageClassDelegate 

174 self._delegateClassName = get_full_type_name(self._delegate) 

175 else: 

176 self._delegate = None 

177 self._delegateClassName = None 

178 

179 @property 

180 def components(self) -> Mapping[str, StorageClass]: 

181 """Return the components associated with this `StorageClass`.""" 

182 return self._components 

183 

184 @property 

185 def derivedComponents(self) -> Mapping[str, StorageClass]: 

186 """Return derived components associated with `StorageClass`.""" 

187 return self._derivedComponents 

188 

189 @property 

190 def converters(self) -> Mapping[str, str]: 

191 """Return the type converters supported by this `StorageClass`.""" 

192 return self._converters 

193 

194 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

195 """Return the type converters as python types.""" 

196 if self._converters_by_type is None: 

197 self._converters_by_type = {} 

198 

199 # Loop over list because the dict can be edited in loop. 

200 for candidate_type_str, converter_str in list(self.converters.items()): 

201 if hasattr(builtins, candidate_type_str): 

202 candidate_type = getattr(builtins, candidate_type_str) 

203 else: 

204 try: 

205 candidate_type = doImportType(candidate_type_str) 

206 except ImportError as e: 

207 log.warning( 

208 "Unable to import type %s associated with storage class %s (%s)", 

209 candidate_type_str, 

210 self.name, 

211 e, 

212 ) 

213 del self._converters[candidate_type_str] 

214 continue 

215 

216 try: 

217 converter = doImportType(converter_str) 

218 except ImportError as e: 

219 log.warning( 

220 "Unable to import conversion function %s associated with storage class %s " 

221 "required to convert type %s (%s)", 

222 converter_str, 

223 self.name, 

224 candidate_type_str, 

225 e, 

226 ) 

227 del self._converters[candidate_type_str] 

228 continue 

229 if not callable(converter): 

230 # doImportType is annotated to return a Type but in actual 

231 # fact it can return Any except ModuleType because package 

232 # variables can be accessed. This make mypy believe it 

233 # is impossible for the return value to not be a callable 

234 # so we must ignore the warning. 

235 log.warning( # type: ignore 

236 "Conversion function %s associated with storage class " 

237 "%s to convert type %s is not a callable.", 

238 converter_str, 

239 self.name, 

240 candidate_type_str, 

241 ) 

242 del self._converters[candidate_type_str] 

243 continue 

244 self._converters_by_type[candidate_type] = converter 

245 return self._converters_by_type 

246 

247 @property 

248 def parameters(self) -> set[str]: 

249 """Return `set` of names of supported parameters.""" 

250 return set(self._parameters) 

251 

252 @property 

253 def pytype(self) -> type: 

254 """Return Python type associated with this `StorageClass`.""" 

255 if self._pytype is not None: 

256 return self._pytype 

257 

258 if hasattr(builtins, self._pytypeName): 

259 pytype = getattr(builtins, self._pytypeName) 

260 else: 

261 pytype = doImportType(self._pytypeName) 

262 self._pytype = pytype 

263 return self._pytype 

264 

265 @property 

266 def delegateClass(self) -> type | None: 

267 """Class to use to delegate type-specific actions.""" 

268 if self._delegate is not None: 

269 return self._delegate 

270 if self._delegateClassName is None: 

271 return None 

272 delegate_class = doImportType(self._delegateClassName) 

273 self._delegate = delegate_class 

274 return self._delegate 

275 

276 def allComponents(self) -> Mapping[str, StorageClass]: 

277 """Return all defined components. 

278 

279 This mapping includes all the derived and read/write components 

280 for the corresponding storage class. 

281 

282 Returns 

283 ------- 

284 comp : `dict` of [`str`, `StorageClass`] 

285 The component name to storage class mapping. 

286 """ 

287 return ChainMap(self._components, self._derivedComponents) 

288 

289 def delegate(self) -> StorageClassDelegate: 

290 """Return an instance of a storage class delegate. 

291 

292 Returns 

293 ------- 

294 delegate : `StorageClassDelegate` 

295 Instance of the delegate associated with this `StorageClass`. 

296 The delegate is constructed with this `StorageClass`. 

297 

298 Raises 

299 ------ 

300 TypeError 

301 This StorageClass has no associated delegate. 

302 """ 

303 cls = self.delegateClass 

304 if cls is None: 

305 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

306 return cls(storageClass=self) 

307 

308 def isComposite(self) -> bool: 

309 """Return Boolean indicating whether this is a composite or not. 

310 

311 Returns 

312 ------- 

313 isComposite : `bool` 

314 `True` if this `StorageClass` is a composite, `False` 

315 otherwise. 

316 """ 

317 if self.components: 

318 return True 

319 return False 

320 

321 def _lookupNames(self) -> tuple[LookupKey, ...]: 

322 """Keys to use when looking up this DatasetRef in a configuration. 

323 

324 The names are returned in order of priority. 

325 

326 Returns 

327 ------- 

328 names : `tuple` of `LookupKey` 

329 Tuple of a `LookupKey` using the `StorageClass` name. 

330 """ 

331 return (LookupKey(name=self.name),) 

332 

333 def knownParameters(self) -> set[str]: 

334 """Return set of all parameters known to this `StorageClass`. 

335 

336 The set includes parameters understood by components of a composite. 

337 

338 Returns 

339 ------- 

340 known : `set` 

341 All parameter keys of this `StorageClass` and the component 

342 storage classes. 

343 """ 

344 known = set(self._parameters) 

345 for sc in self.components.values(): 

346 known.update(sc.knownParameters()) 

347 return known 

348 

349 def validateParameters(self, parameters: Collection | None = None) -> None: 

350 """Check that the parameters are known to this `StorageClass`. 

351 

352 Does not check the values. 

353 

354 Parameters 

355 ---------- 

356 parameters : `~collections.abc.Collection`, optional 

357 Collection containing the parameters. Can be `dict`-like or 

358 `set`-like. The parameter values are not checked. 

359 If no parameters are supplied, always returns without error. 

360 

361 Raises 

362 ------ 

363 KeyError 

364 Some parameters are not understood by this `StorageClass`. 

365 """ 

366 # No parameters is always okay 

367 if not parameters: 

368 return 

369 

370 # Extract the important information into a set. Works for dict and 

371 # list. 

372 external = set(parameters) 

373 

374 diff = external - self.knownParameters() 

375 if diff: 

376 s = "s" if len(diff) > 1 else "" 

377 unknown = "', '".join(diff) 

378 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

379 

380 def filterParameters( 

381 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

382 ) -> Mapping[str, Any]: 

383 """Filter out parameters that are not known to this `StorageClass`. 

384 

385 Parameters 

386 ---------- 

387 parameters : `~collections.abc.Mapping`, optional 

388 Candidate parameters. Can be `None` if no parameters have 

389 been provided. 

390 subset : `~collections.abc.Collection`, optional 

391 Subset of supported parameters that the caller is interested 

392 in using. The subset must be known to the `StorageClass` 

393 if specified. If `None` the supplied parameters will all 

394 be checked, else only the keys in this set will be checked. 

395 

396 Returns 

397 ------- 

398 filtered : `~collections.abc.Mapping` 

399 Valid parameters. Empty `dict` if none are suitable. 

400 

401 Raises 

402 ------ 

403 ValueError 

404 Raised if the provided subset is not a subset of the supported 

405 parameters or if it is an empty set. 

406 """ 

407 if not parameters: 

408 return {} 

409 

410 known = self.knownParameters() 

411 

412 if subset is not None: 

413 if not subset: 

414 raise ValueError("Specified a parameter subset but it was empty") 

415 subset = set(subset) 

416 if not subset.issubset(known): 

417 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

418 wanted = subset 

419 else: 

420 wanted = known 

421 

422 return {k: parameters[k] for k in wanted if k in parameters} 

423 

424 def validateInstance(self, instance: Any) -> bool: 

425 """Check that the supplied Python object has the expected Python type. 

426 

427 Parameters 

428 ---------- 

429 instance : `object` 

430 Object to check. 

431 

432 Returns 

433 ------- 

434 isOk : `bool` 

435 True if the supplied instance object can be handled by this 

436 `StorageClass`, False otherwise. 

437 """ 

438 return isinstance(instance, self.pytype) 

439 

440 def is_type(self, other: type, compare_types: bool = False) -> bool: 

441 """Return Boolean indicating whether the supplied type matches 

442 the type in this `StorageClass`. 

443 

444 Parameters 

445 ---------- 

446 other : `type` 

447 The type to be checked. 

448 compare_types : `bool`, optional 

449 If `True` the python type will be used in the comparison 

450 if the type names do not match. This may trigger an import 

451 of code and so can be slower. 

452 

453 Returns 

454 ------- 

455 match : `bool` 

456 `True` if the types are equal. 

457 

458 Notes 

459 ----- 

460 If this `StorageClass` has not yet imported the Python type the 

461 check is done against the full type name, this prevents an attempt 

462 to import the type when it will likely not match. 

463 """ 

464 if self._pytype: 

465 return self._pytype is other 

466 

467 other_name = get_full_type_name(other) 

468 if self._pytypeName == other_name: 

469 return True 

470 

471 if compare_types: 

472 # Must protect against the import failing. 

473 try: 

474 return self.pytype is other 

475 except Exception: 

476 pass 

477 

478 return False 

479 

480 def can_convert(self, other: StorageClass) -> bool: 

481 """Return `True` if this storage class can convert python types 

482 in the other storage class. 

483 

484 Parameters 

485 ---------- 

486 other : `StorageClass` 

487 The storage class to check. 

488 

489 Returns 

490 ------- 

491 can : `bool` 

492 `True` if this storage class has a registered converter for 

493 the python type associated with the other storage class. That 

494 converter will convert the other python type to the one associated 

495 with this storage class. 

496 """ 

497 if other.name == self.name: 

498 # Identical storage classes are compatible. 

499 return True 

500 

501 # It may be that the storage class being compared is not 

502 # available because the python type can't be imported. In that 

503 # case conversion must be impossible. 

504 try: 

505 other_pytype = other.pytype 

506 except Exception: 

507 return False 

508 

509 # Or even this storage class itself can not have the type imported. 

510 try: 

511 self_pytype = self.pytype 

512 except Exception: 

513 return False 

514 

515 if issubclass(other_pytype, self_pytype): 

516 # Storage classes have different names but the same python type. 

517 return True 

518 

519 for candidate_type in self._get_converters_by_type(): 

520 if issubclass(other_pytype, candidate_type): 

521 return True 

522 return False 

523 

524 def coerce_type(self, incorrect: Any) -> Any: 

525 """Coerce the supplied incorrect instance to the python type 

526 associated with this `StorageClass`. 

527 

528 Parameters 

529 ---------- 

530 incorrect : `object` 

531 An object that might be the incorrect type. 

532 

533 Returns 

534 ------- 

535 correct : `object` 

536 An object that matches the python type of this `StorageClass`. 

537 Can be the same object as given. If `None`, `None` will be 

538 returned. 

539 

540 Raises 

541 ------ 

542 TypeError 

543 Raised if no conversion can be found. 

544 """ 

545 if incorrect is None: 

546 return None 

547 

548 # Possible this is the correct type already. 

549 if self.validateInstance(incorrect): 

550 return incorrect 

551 

552 # Check each registered converter. 

553 for candidate_type, converter in self._get_converters_by_type().items(): 

554 if isinstance(incorrect, candidate_type): 

555 try: 

556 return converter(incorrect) 

557 except Exception: 

558 log.error( 

559 "Converter %s failed to convert type %s", 

560 get_full_type_name(converter), 

561 get_full_type_name(incorrect), 

562 ) 

563 raise 

564 raise TypeError( 

565 "Type does not match and no valid converter found to convert" 

566 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

567 ) 

568 

569 def __eq__(self, other: Any) -> bool: 

570 """Equality checks name, pytype name, delegate name, and components.""" 

571 if not isinstance(other, StorageClass): 

572 return NotImplemented 

573 

574 if self.name != other.name: 

575 return False 

576 

577 # We must compare pytype and delegate by name since we do not want 

578 # to trigger an import of external module code here 

579 if self._delegateClassName != other._delegateClassName: 

580 return False 

581 if self._pytypeName != other._pytypeName: 

582 return False 

583 

584 # Ensure we have the same component keys in each 

585 if set(self.components.keys()) != set(other.components.keys()): 

586 return False 

587 

588 # Same parameters 

589 if self.parameters != other.parameters: 

590 return False 

591 

592 # Ensure that all the components have the same type 

593 return all(self.components[k] == other.components[k] for k in self.components) 

594 

595 def __hash__(self) -> int: 

596 return hash(self.name) 

597 

598 def __repr__(self) -> str: 

599 optionals: dict[str, Any] = {} 

600 if self._pytypeName != "object": 

601 optionals["pytype"] = self._pytypeName 

602 if self._delegateClassName is not None: 

603 optionals["delegate"] = self._delegateClassName 

604 if self._parameters: 

605 optionals["parameters"] = self._parameters 

606 if self.components: 

607 optionals["components"] = self.components 

608 if self.converters: 

609 optionals["converters"] = self.converters 

610 

611 # order is preserved in the dict 

612 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

613 

614 # Start with mandatory fields 

615 r = f"{self.__class__.__name__}({self.name!r}" 

616 if options: 

617 r = r + ", " + options 

618 r = r + ")" 

619 return r 

620 

621 def __str__(self) -> str: 

622 return self.name 

623 

624 

625class StorageClassFactory(metaclass=Singleton): 

626 """Factory for `StorageClass` instances. 

627 

628 This class is a singleton, with each instance sharing the pool of 

629 StorageClasses. Since code can not know whether it is the first 

630 time the instance has been created, the constructor takes no arguments. 

631 To populate the factory with storage classes, a call to 

632 `~StorageClassFactory.addFromConfig()` should be made. 

633 

634 Parameters 

635 ---------- 

636 config : `StorageClassConfig` or `str`, optional 

637 Load configuration. In a ButlerConfig` the relevant configuration 

638 is located in the ``storageClasses`` section. 

639 """ 

640 

641 def __init__(self, config: StorageClassConfig | str | None = None): 

642 self._storageClasses: dict[str, StorageClass] = {} 

643 self._configs: list[StorageClassConfig] = [] 

644 

645 # Always seed with the default config 

646 self.addFromConfig(StorageClassConfig()) 

647 

648 if config is not None: 648 ↛ 649line 648 didn't jump to line 649, because the condition on line 648 was never true

649 self.addFromConfig(config) 

650 

651 def __str__(self) -> str: 

652 """Return summary of factory. 

653 

654 Returns 

655 ------- 

656 summary : `str` 

657 Summary of the factory status. 

658 """ 

659 sep = "\n" 

660 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

661 

662StorageClasses 

663-------------- 

664{sep.join(f"{s}: {self._storageClasses[s]!r}" for s in sorted(self._storageClasses))} 

665""" 

666 

667 def __contains__(self, storageClassOrName: StorageClass | str) -> bool: 

668 """Indicate whether the storage class exists in the factory. 

669 

670 Parameters 

671 ---------- 

672 storageClassOrName : `str` or `StorageClass` 

673 If `str` is given existence of the named StorageClass 

674 in the factory is checked. If `StorageClass` is given 

675 existence and equality are checked. 

676 

677 Returns 

678 ------- 

679 in : `bool` 

680 True if the supplied string is present, or if the supplied 

681 `StorageClass` is present and identical. 

682 

683 Notes 

684 ----- 

685 The two different checks (one for "key" and one for "value") based on 

686 the type of the given argument mean that it is possible for 

687 StorageClass.name to be in the factory but StorageClass to not be 

688 in the factory. 

689 """ 

690 if isinstance(storageClassOrName, str): 690 ↛ 692line 690 didn't jump to line 692, because the condition on line 690 was never false

691 return storageClassOrName in self._storageClasses 

692 elif isinstance(storageClassOrName, StorageClass) and storageClassOrName.name in self._storageClasses: 

693 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

694 return False 

695 

696 def __len__(self) -> int: 

697 return len(self._storageClasses) 

698 

699 def __iter__(self) -> Iterator[str]: 

700 return iter(self._storageClasses) 

701 

702 def values(self) -> ValuesView[StorageClass]: 

703 return self._storageClasses.values() 

704 

705 def keys(self) -> KeysView[str]: 

706 return self._storageClasses.keys() 

707 

708 def items(self) -> ItemsView[str, StorageClass]: 

709 return self._storageClasses.items() 

710 

711 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

712 """Add more `StorageClass` definitions from a config file. 

713 

714 Parameters 

715 ---------- 

716 config : `StorageClassConfig`, `Config` or `str` 

717 Storage class configuration. Can contain a ``storageClasses`` 

718 key if part of a global configuration. 

719 """ 

720 sconfig = StorageClassConfig(config) 

721 self._configs.append(sconfig) 

722 

723 # Since we can not assume that we will get definitions of 

724 # components or parents before their classes are defined 

725 # we have a helper function that we can call recursively 

726 # to extract definitions from the configuration. 

727 def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> None: 

728 # Maybe we've already processed this through recursion 

729 if name not in _sconfig: 

730 return 

731 info = _sconfig.pop(name) 

732 

733 # Always create the storage class so we can ensure that 

734 # we are not trying to overwrite with a different definition 

735 components = None 

736 

737 # Extract scalar items from dict that are needed for 

738 # StorageClass Constructor 

739 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

740 

741 if "converters" in info: 

742 storageClassKwargs["converters"] = info["converters"].toDict() 

743 

744 for compName in ("components", "derivedComponents"): 

745 if compName not in info: 

746 continue 

747 components = {} 

748 for cname, ctype in info[compName].items(): 

749 if ctype not in self: 

750 processStorageClass(ctype, sconfig, msg) 

751 components[cname] = self.getStorageClass(ctype) 

752 

753 # Fill in other items 

754 storageClassKwargs[compName] = components 

755 

756 # Create the new storage class and register it 

757 baseClass = None 

758 if "inheritsFrom" in info: 

759 baseName = info["inheritsFrom"] 

760 

761 # The inheritsFrom feature requires that the storage class 

762 # being inherited from is itself a subclass of StorageClass 

763 # that was created with makeNewStorageClass. If it was made 

764 # and registered with a simple StorageClass constructor it 

765 # cannot be used here and we try to recreate it. 

766 if baseName in self: 766 ↛ 778line 766 didn't jump to line 778, because the condition on line 766 was never false

767 baseClass = type(self.getStorageClass(baseName)) 

768 if baseClass is StorageClass: 768 ↛ 769line 768 didn't jump to line 769, because the condition on line 768 was never true

769 log.warning( 

770 "Storage class %s is requested to inherit from %s but that storage class " 

771 "has not been defined to be a subclass of StorageClass and so can not " 

772 "be used. Attempting to recreate parent class from current configuration.", 

773 name, 

774 baseName, 

775 ) 

776 processStorageClass(baseName, sconfig, msg) 

777 else: 

778 processStorageClass(baseName, sconfig, msg) 

779 baseClass = type(self.getStorageClass(baseName)) 

780 if baseClass is StorageClass: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true

781 raise TypeError( 

782 f"Configuration for storage class {name} requests to inherit from " 

783 f" storage class {baseName} but that class is not defined correctly." 

784 ) 

785 

786 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

787 newStorageClass = newStorageClassType() 

788 self.registerStorageClass(newStorageClass, msg=msg) 

789 

790 # In case there is a problem, construct a context message for any 

791 # error reporting. 

792 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

793 context = f"when adding definitions from {', '.join(files)}" if files else "" 

794 log.debug("Adding definitions from config %s", ", ".join(files)) 

795 

796 for name in list(sconfig.keys()): 

797 processStorageClass(name, sconfig, context) 

798 

799 @staticmethod 

800 def makeNewStorageClass( 

801 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any 

802 ) -> type[StorageClass]: 

803 """Create a new Python class as a subclass of `StorageClass`. 

804 

805 Parameters 

806 ---------- 

807 name : `str` 

808 Name to use for this class. 

809 baseClass : `type`, optional 

810 Base class for this `StorageClass`. Must be either `StorageClass` 

811 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

812 be used. 

813 

814 Returns 

815 ------- 

816 newtype : `type` subclass of `StorageClass` 

817 Newly created Python type. 

818 """ 

819 if baseClass is None: 

820 baseClass = StorageClass 

821 if not issubclass(baseClass, StorageClass): 821 ↛ 822line 821 didn't jump to line 822, because the condition on line 821 was never true

822 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

823 

824 # convert the arguments to use different internal names 

825 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

826 clsargs["_cls_name"] = name 

827 

828 # Some container items need to merge with the base class values 

829 # so that a child can inherit but override one bit. 

830 # lists (which you get from configs) are treated as sets for this to 

831 # work consistently. 

832 for k in ("components", "parameters", "derivedComponents", "converters"): 

833 classKey = f"_cls_{k}" 

834 if classKey in clsargs: 

835 baseValue = getattr(baseClass, classKey, None) 

836 if baseValue is not None: 

837 currentValue = clsargs[classKey] 

838 if isinstance(currentValue, dict): 838 ↛ 841line 838 didn't jump to line 841, because the condition on line 838 was never false

839 newValue = baseValue.copy() 

840 else: 

841 newValue = set(baseValue) 

842 newValue.update(currentValue) 

843 clsargs[classKey] = newValue 

844 

845 # If we have parameters they should be a frozen set so that the 

846 # parameters in the class can not be modified. 

847 pk = "_cls_parameters" 

848 if pk in clsargs: 

849 clsargs[pk] = frozenset(clsargs[pk]) 

850 

851 return type(f"StorageClass{name}", (baseClass,), clsargs) 

852 

853 def getStorageClass(self, storageClassName: str) -> StorageClass: 

854 """Get a StorageClass instance associated with the supplied name. 

855 

856 Parameters 

857 ---------- 

858 storageClassName : `str` 

859 Name of the storage class to retrieve. 

860 

861 Returns 

862 ------- 

863 instance : `StorageClass` 

864 Instance of the correct `StorageClass`. 

865 

866 Raises 

867 ------ 

868 KeyError 

869 The requested storage class name is not registered. 

870 """ 

871 return self._storageClasses[storageClassName] 

872 

873 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

874 """Find the storage class associated with this python type. 

875 

876 Parameters 

877 ---------- 

878 pytype : `type` 

879 The Python type to be matched. 

880 compare_types : `bool`, optional 

881 If `False`, the type will be checked against name of the python 

882 type. This comparison is always done first. If `True` and the 

883 string comparison failed, each candidate storage class will be 

884 forced to have its type imported. This can be significantly slower. 

885 

886 Returns 

887 ------- 

888 storageClass : `StorageClass` 

889 The matching storage class. 

890 

891 Raises 

892 ------ 

893 KeyError 

894 Raised if no match could be found. 

895 

896 Notes 

897 ----- 

898 It is possible for a python type to be associated with multiple 

899 storage classes. This method will currently return the first that 

900 matches. 

901 """ 

902 result = self._find_storage_class(pytype, False) 

903 if result: 

904 return result 

905 

906 if compare_types: 

907 # The fast comparison failed and we were asked to try the 

908 # variant that might involve code imports. 

909 result = self._find_storage_class(pytype, True) 

910 if result: 

911 return result 

912 

913 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}") 

914 

915 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

916 """Iterate through all storage classes to find a match. 

917 

918 Parameters 

919 ---------- 

920 pytype : `type` 

921 The Python type to be matched. 

922 compare_types : `bool`, optional 

923 Whether to use type name matching or explicit type matching. 

924 The latter can be slower. 

925 

926 Returns 

927 ------- 

928 storageClass : `StorageClass` or `None` 

929 The matching storage class, or `None` if no match was found. 

930 

931 Notes 

932 ----- 

933 Helper method for ``findStorageClass``. 

934 """ 

935 for storageClass in self.values(): 

936 if storageClass.is_type(pytype, compare_types=compare_types): 

937 return storageClass 

938 return None 

939 

940 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

941 """Store the `StorageClass` in the factory. 

942 

943 Will be indexed by `StorageClass.name` and will return instances 

944 of the supplied `StorageClass`. 

945 

946 Parameters 

947 ---------- 

948 storageClass : `StorageClass` 

949 Type of the Python `StorageClass` to register. 

950 msg : `str`, optional 

951 Additional message string to be included in any error message. 

952 

953 Raises 

954 ------ 

955 ValueError 

956 If a storage class has already been registered with 

957 that storage class name and the previous definition differs. 

958 """ 

959 if storageClass.name in self._storageClasses: 959 ↛ 960line 959 didn't jump to line 960, because the condition on line 959 was never true

960 existing = self.getStorageClass(storageClass.name) 

961 if existing != storageClass: 

962 errmsg = f" {msg}" if msg else "" 

963 raise ValueError( 

964 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

965 f"differs from current definition ({existing!r}){errmsg}" 

966 ) 

967 if type(existing) is StorageClass and type(storageClass) is not StorageClass: 

968 # Replace generic with specialist subclass equivalent. 

969 self._storageClasses[storageClass.name] = storageClass 

970 else: 

971 self._storageClasses[storageClass.name] = storageClass 

972 

973 def _unregisterStorageClass(self, storageClassName: str) -> None: 

974 """Remove the named StorageClass from the factory. 

975 

976 Parameters 

977 ---------- 

978 storageClassName : `str` 

979 Name of storage class to remove. 

980 

981 Raises 

982 ------ 

983 KeyError 

984 The named storage class is not registered. 

985 

986 Notes 

987 ----- 

988 This method is intended to simplify testing of StorageClassFactory 

989 functionality and it is not expected to be required for normal usage. 

990 """ 

991 del self._storageClasses[storageClassName] 

992 

993 def reset(self) -> None: 

994 """Remove all storage class entries from factory and reset to 

995 initial state. 

996 

997 This is useful for test code where a known start state is useful. 

998 """ 

999 self._storageClasses.clear() 

1000 # Seed with the default config. 

1001 self.addFromConfig(StorageClassConfig())