Coverage for python/lsst/daf/butler/_storage_class.py: 45%

366 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for Storage Classes.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

33 

34import builtins 

35import itertools 

36import logging 

37from collections import ChainMap 

38from collections.abc import ( 

39 Callable, 

40 Collection, 

41 ItemsView, 

42 Iterator, 

43 KeysView, 

44 Mapping, 

45 Sequence, 

46 Set, 

47 ValuesView, 

48) 

49from typing import Any 

50 

51from lsst.utils import doImportType 

52from lsst.utils.classes import Singleton 

53from lsst.utils.introspection import get_full_type_name 

54 

55from ._config import Config, ConfigSubset 

56from ._config_support import LookupKey 

57from ._storage_class_delegate import StorageClassDelegate 

58 

59log = logging.getLogger(__name__) 

60 

61 

62class StorageClassConfig(ConfigSubset): 

63 """Configuration class for defining Storage Classes.""" 

64 

65 component = "storageClasses" 

66 defaultConfigFile = "storageClasses.yaml" 

67 

68 

69class StorageClass: 

70 """Class describing how a label maps to a particular Python type. 

71 

72 Parameters 

73 ---------- 

74 name : `str` 

75 Name to use for this class. 

76 pytype : `type` or `str` 

77 Python type (or name of type) to associate with the `StorageClass`. 

78 components : `dict`, optional 

79 `dict` mapping name of a component to another `StorageClass`. 

80 derivedComponents : `dict`, optional 

81 `dict` mapping name of a derived component to another `StorageClass`. 

82 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

83 Parameters understood by this `StorageClass` that can control 

84 reading of data from datastores. 

85 delegate : `str`, optional 

86 Fully qualified name of class supporting assembly and disassembly 

87 of a `pytype` instance. 

88 converters : `dict` [`str`, `str`], optional 

89 Mapping of python type to function that can be called to convert 

90 that python type to the valid type of this storage class. 

91 """ 

92 

93 _cls_name: str = "BaseStorageClass" 

94 _cls_components: dict[str, StorageClass] | None = None 

95 _cls_derivedComponents: dict[str, StorageClass] | None = None 

96 _cls_parameters: Set[str] | Sequence[str] | None = None 

97 _cls_delegate: str | None = None 

98 _cls_pytype: type | str | None = None 

99 _cls_converters: dict[str, str] | None = None 

100 

101 def __init__( 

102 self, 

103 name: str | None = None, 

104 pytype: type | str | None = None, 

105 components: dict[str, StorageClass] | None = None, 

106 derivedComponents: dict[str, StorageClass] | None = None, 

107 parameters: Sequence[str] | Set[str] | None = None, 

108 delegate: str | None = None, 

109 converters: dict[str, str] | None = None, 

110 ): 

111 if name is None: 

112 name = self._cls_name 

113 if pytype is None: 113 ↛ 115line 113 didn't jump to line 115, because the condition on line 113 was never false

114 pytype = self._cls_pytype 

115 if components is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false

116 components = self._cls_components 

117 if derivedComponents is None: 117 ↛ 119line 117 didn't jump to line 119, because the condition on line 117 was never false

118 derivedComponents = self._cls_derivedComponents 

119 if parameters is None: 119 ↛ 121line 119 didn't jump to line 121, because the condition on line 119 was never false

120 parameters = self._cls_parameters 

121 if delegate is None: 121 ↛ 125line 121 didn't jump to line 125, because the condition on line 121 was never false

122 delegate = self._cls_delegate 

123 

124 # Merge converters with class defaults. 

125 self._converters = {} 

126 if self._cls_converters is not None: 

127 self._converters.update(self._cls_converters) 

128 if converters: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true

129 self._converters.update(converters) 

130 

131 # Version of converters where the python types have been 

132 # Do not try to import anything until needed. 

133 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

134 

135 self.name = name 

136 

137 if pytype is None: 

138 pytype = object 

139 

140 self._pytype: type | None 

141 if not isinstance(pytype, str): 

142 # Already have a type so store it and get the name 

143 self._pytypeName = get_full_type_name(pytype) 

144 self._pytype = pytype 

145 else: 

146 # Store the type name and defer loading of type 

147 self._pytypeName = pytype 

148 self._pytype = None 

149 

150 if components is not None: 

151 if len(components) == 1: 151 ↛ 152line 151 didn't jump to line 152, because the condition on line 151 was never true

152 raise ValueError( 

153 f"Composite storage class {name} is not allowed to have" 

154 f" only one component '{next(iter(components))}'." 

155 " Did you mean it to be a derived component?" 

156 ) 

157 self._components = components 

158 else: 

159 self._components = {} 

160 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

161 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

162 # if the delegate is not None also set it and clear the default 

163 # delegate 

164 self._delegate: type | None 

165 self._delegateClassName: str | None 

166 if delegate is not None: 

167 self._delegateClassName = delegate 

168 self._delegate = None 

169 elif components is not None: 169 ↛ 172line 169 didn't jump to line 172, because the condition on line 169 was never true

170 # We set a default delegate for composites so that a class is 

171 # guaranteed to support something if it is a composite. 

172 log.debug("Setting default delegate for %s", self.name) 

173 self._delegate = StorageClassDelegate 

174 self._delegateClassName = get_full_type_name(self._delegate) 

175 else: 

176 self._delegate = None 

177 self._delegateClassName = None 

178 

179 @property 

180 def components(self) -> Mapping[str, StorageClass]: 

181 """Return the components associated with this `StorageClass`.""" 

182 return self._components 

183 

184 @property 

185 def derivedComponents(self) -> Mapping[str, StorageClass]: 

186 """Return derived components associated with `StorageClass`.""" 

187 return self._derivedComponents 

188 

189 @property 

190 def converters(self) -> Mapping[str, str]: 

191 """Return the type converters supported by this `StorageClass`.""" 

192 return self._converters 

193 

194 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

195 """Return the type converters as python types.""" 

196 if self._converters_by_type is None: 

197 self._converters_by_type = {} 

198 

199 # Loop over list because the dict can be edited in loop. 

200 for candidate_type_str, converter_str in list(self.converters.items()): 

201 if hasattr(builtins, candidate_type_str): 

202 candidate_type = getattr(builtins, candidate_type_str) 

203 else: 

204 try: 

205 candidate_type = doImportType(candidate_type_str) 

206 except ImportError as e: 

207 log.warning( 

208 "Unable to import type %s associated with storage class %s (%s)", 

209 candidate_type_str, 

210 self.name, 

211 e, 

212 ) 

213 del self._converters[candidate_type_str] 

214 continue 

215 

216 try: 

217 converter = doImportType(converter_str) 

218 except ImportError as e: 

219 log.warning( 

220 "Unable to import conversion function %s associated with storage class %s " 

221 "required to convert type %s (%s)", 

222 converter_str, 

223 self.name, 

224 candidate_type_str, 

225 e, 

226 ) 

227 del self._converters[candidate_type_str] 

228 continue 

229 if not callable(converter): 

230 # doImportType is annotated to return a Type but in actual 

231 # fact it can return Any except ModuleType because package 

232 # variables can be accessed. This make mypy believe it 

233 # is impossible for the return value to not be a callable 

234 # so we must ignore the warning. 

235 log.warning( # type: ignore 

236 "Conversion function %s associated with storage class " 

237 "%s to convert type %s is not a callable.", 

238 converter_str, 

239 self.name, 

240 candidate_type_str, 

241 ) 

242 del self._converters[candidate_type_str] 

243 continue 

244 self._converters_by_type[candidate_type] = converter 

245 return self._converters_by_type 

246 

247 @property 

248 def parameters(self) -> set[str]: 

249 """Return `set` of names of supported parameters.""" 

250 return set(self._parameters) 

251 

252 @property 

253 def pytype(self) -> type: 

254 """Return Python type associated with this `StorageClass`.""" 

255 if self._pytype is not None: 

256 return self._pytype 

257 

258 if hasattr(builtins, self._pytypeName): 

259 pytype = getattr(builtins, self._pytypeName) 

260 else: 

261 pytype = doImportType(self._pytypeName) 

262 self._pytype = pytype 

263 return self._pytype 

264 

265 @property 

266 def delegateClass(self) -> type | None: 

267 """Class to use to delegate type-specific actions.""" 

268 if self._delegate is not None: 

269 return self._delegate 

270 if self._delegateClassName is None: 

271 return None 

272 delegate_class = doImportType(self._delegateClassName) 

273 self._delegate = delegate_class 

274 return self._delegate 

275 

276 def allComponents(self) -> Mapping[str, StorageClass]: 

277 """Return all defined components. 

278 

279 This mapping includes all the derived and read/write components 

280 for the corresponding storage class. 

281 

282 Returns 

283 ------- 

284 comp : `dict` of [`str`, `StorageClass`] 

285 The component name to storage class mapping. 

286 """ 

287 return ChainMap(self._components, self._derivedComponents) 

288 

289 def delegate(self) -> StorageClassDelegate: 

290 """Return an instance of a storage class delegate. 

291 

292 Returns 

293 ------- 

294 delegate : `StorageClassDelegate` 

295 Instance of the delegate associated with this `StorageClass`. 

296 The delegate is constructed with this `StorageClass`. 

297 

298 Raises 

299 ------ 

300 TypeError 

301 This StorageClass has no associated delegate. 

302 """ 

303 cls = self.delegateClass 

304 if cls is None: 

305 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

306 return cls(storageClass=self) 

307 

308 def isComposite(self) -> bool: 

309 """Return Boolean indicating whether this is a composite or not. 

310 

311 Returns 

312 ------- 

313 isComposite : `bool` 

314 `True` if this `StorageClass` is a composite, `False` 

315 otherwise. 

316 """ 

317 if self.components: 

318 return True 

319 return False 

320 

321 def _lookupNames(self) -> tuple[LookupKey, ...]: 

322 """Keys to use when looking up this DatasetRef in a configuration. 

323 

324 The names are returned in order of priority. 

325 

326 Returns 

327 ------- 

328 names : `tuple` of `LookupKey` 

329 Tuple of a `LookupKey` using the `StorageClass` name. 

330 """ 

331 return (LookupKey(name=self.name),) 

332 

333 def knownParameters(self) -> set[str]: 

334 """Return set of all parameters known to this `StorageClass`. 

335 

336 The set includes parameters understood by components of a composite. 

337 

338 Returns 

339 ------- 

340 known : `set` 

341 All parameter keys of this `StorageClass` and the component 

342 storage classes. 

343 """ 

344 known = set(self._parameters) 

345 for sc in self.components.values(): 

346 known.update(sc.knownParameters()) 

347 return known 

348 

349 def validateParameters(self, parameters: Collection | None = None) -> None: 

350 """Check that the parameters are known to this `StorageClass`. 

351 

352 Does not check the values. 

353 

354 Parameters 

355 ---------- 

356 parameters : `~collections.abc.Collection`, optional 

357 Collection containing the parameters. Can be `dict`-like or 

358 `set`-like. The parameter values are not checked. 

359 If no parameters are supplied, always returns without error. 

360 

361 Raises 

362 ------ 

363 KeyError 

364 Some parameters are not understood by this `StorageClass`. 

365 """ 

366 # No parameters is always okay 

367 if not parameters: 

368 return 

369 

370 # Extract the important information into a set. Works for dict and 

371 # list. 

372 external = set(parameters) 

373 

374 diff = external - self.knownParameters() 

375 if diff: 

376 s = "s" if len(diff) > 1 else "" 

377 unknown = "', '".join(diff) 

378 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

379 

380 def filterParameters( 

381 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

382 ) -> Mapping[str, Any]: 

383 """Filter out parameters that are not known to this `StorageClass`. 

384 

385 Parameters 

386 ---------- 

387 parameters : `~collections.abc.Mapping`, optional 

388 Candidate parameters. Can be `None` if no parameters have 

389 been provided. 

390 subset : `~collections.abc.Collection`, optional 

391 Subset of supported parameters that the caller is interested 

392 in using. The subset must be known to the `StorageClass` 

393 if specified. If `None` the supplied parameters will all 

394 be checked, else only the keys in this set will be checked. 

395 

396 Returns 

397 ------- 

398 filtered : `~collections.abc.Mapping` 

399 Valid parameters. Empty `dict` if none are suitable. 

400 

401 Raises 

402 ------ 

403 ValueError 

404 Raised if the provided subset is not a subset of the supported 

405 parameters or if it is an empty set. 

406 """ 

407 if not parameters: 

408 return {} 

409 

410 known = self.knownParameters() 

411 

412 if subset is not None: 

413 if not subset: 

414 raise ValueError("Specified a parameter subset but it was empty") 

415 subset = set(subset) 

416 if not subset.issubset(known): 

417 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

418 wanted = subset 

419 else: 

420 wanted = known 

421 

422 return {k: parameters[k] for k in wanted if k in parameters} 

423 

424 def validateInstance(self, instance: Any) -> bool: 

425 """Check that the supplied Python object has the expected Python type. 

426 

427 Parameters 

428 ---------- 

429 instance : `object` 

430 Object to check. 

431 

432 Returns 

433 ------- 

434 isOk : `bool` 

435 True if the supplied instance object can be handled by this 

436 `StorageClass`, False otherwise. 

437 """ 

438 return isinstance(instance, self.pytype) 

439 

440 def is_type(self, other: type, compare_types: bool = False) -> bool: 

441 """Return Boolean indicating whether the supplied type matches 

442 the type in this `StorageClass`. 

443 

444 Parameters 

445 ---------- 

446 other : `type` 

447 The type to be checked. 

448 compare_types : `bool`, optional 

449 If `True` the python type will be used in the comparison 

450 if the type names do not match. This may trigger an import 

451 of code and so can be slower. 

452 

453 Returns 

454 ------- 

455 match : `bool` 

456 `True` if the types are equal. 

457 

458 Notes 

459 ----- 

460 If this `StorageClass` has not yet imported the Python type the 

461 check is done against the full type name, this prevents an attempt 

462 to import the type when it will likely not match. 

463 """ 

464 if self._pytype: 

465 return self._pytype is other 

466 

467 other_name = get_full_type_name(other) 

468 if self._pytypeName == other_name: 

469 return True 

470 

471 if compare_types: 

472 # Must protect against the import failing. 

473 try: 

474 return self.pytype is other 

475 except Exception: 

476 pass 

477 

478 return False 

479 

480 def can_convert(self, other: StorageClass) -> bool: 

481 """Return `True` if this storage class can convert python types 

482 in the other storage class. 

483 

484 Parameters 

485 ---------- 

486 other : `StorageClass` 

487 The storage class to check. 

488 

489 Returns 

490 ------- 

491 can : `bool` 

492 `True` if this storage class has a registered converter for 

493 the python type associated with the other storage class. That 

494 converter will convert the other python type to the one associated 

495 with this storage class. 

496 """ 

497 if other.name == self.name: 

498 # Identical storage classes are compatible. 

499 return True 

500 

501 # It may be that the storage class being compared is not 

502 # available because the python type can't be imported. In that 

503 # case conversion must be impossible. 

504 try: 

505 other_pytype = other.pytype 

506 except Exception: 

507 return False 

508 

509 # Or even this storage class itself can not have the type imported. 

510 try: 

511 self_pytype = self.pytype 

512 except Exception: 

513 return False 

514 

515 if issubclass(other_pytype, self_pytype): 

516 # Storage classes have different names but the same python type. 

517 return True 

518 

519 for candidate_type in self._get_converters_by_type(): 

520 if issubclass(other_pytype, candidate_type): 

521 return True 

522 return False 

523 

524 def coerce_type(self, incorrect: Any) -> Any: 

525 """Coerce the supplied incorrect instance to the python type 

526 associated with this `StorageClass`. 

527 

528 Parameters 

529 ---------- 

530 incorrect : `object` 

531 An object that might be the incorrect type. 

532 

533 Returns 

534 ------- 

535 correct : `object` 

536 An object that matches the python type of this `StorageClass`. 

537 Can be the same object as given. If `None`, `None` will be 

538 returned. 

539 

540 Raises 

541 ------ 

542 TypeError 

543 Raised if no conversion can be found. 

544 """ 

545 if incorrect is None: 

546 return None 

547 

548 # Possible this is the correct type already. 

549 if self.validateInstance(incorrect): 

550 return incorrect 

551 

552 # Check each registered converter. 

553 for candidate_type, converter in self._get_converters_by_type().items(): 

554 if isinstance(incorrect, candidate_type): 

555 try: 

556 return converter(incorrect) 

557 except Exception: 

558 log.error( 

559 "Converter %s failed to convert type %s", 

560 get_full_type_name(converter), 

561 get_full_type_name(incorrect), 

562 ) 

563 raise 

564 raise TypeError( 

565 "Type does not match and no valid converter found to convert" 

566 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

567 ) 

568 

569 def __eq__(self, other: Any) -> bool: 

570 """Equality checks name, pytype name, delegate name, and components.""" 

571 if not isinstance(other, StorageClass): 

572 return NotImplemented 

573 

574 if self.name != other.name: 

575 return False 

576 

577 # We must compare pytype and delegate by name since we do not want 

578 # to trigger an import of external module code here 

579 if self._delegateClassName != other._delegateClassName: 

580 return False 

581 if self._pytypeName != other._pytypeName: 

582 return False 

583 

584 # Ensure we have the same component keys in each 

585 if set(self.components.keys()) != set(other.components.keys()): 

586 return False 

587 

588 # Same parameters 

589 if self.parameters != other.parameters: 

590 return False 

591 

592 # Ensure that all the components have the same type 

593 return all(self.components[k] == other.components[k] for k in self.components) 

594 

595 def __hash__(self) -> int: 

596 return hash(self.name) 

597 

598 def __repr__(self) -> str: 

599 optionals: dict[str, Any] = {} 

600 if self._pytypeName != "object": 

601 optionals["pytype"] = self._pytypeName 

602 if self._delegateClassName is not None: 

603 optionals["delegate"] = self._delegateClassName 

604 if self._parameters: 

605 optionals["parameters"] = self._parameters 

606 if self.components: 

607 optionals["components"] = self.components 

608 if self.converters: 

609 optionals["converters"] = self.converters 

610 

611 # order is preserved in the dict 

612 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

613 

614 # Start with mandatory fields 

615 r = f"{self.__class__.__name__}({self.name!r}" 

616 if options: 

617 r = r + ", " + options 

618 r = r + ")" 

619 return r 

620 

621 def __str__(self) -> str: 

622 return self.name 

623 

624 

625class StorageClassFactory(metaclass=Singleton): 

626 """Factory for `StorageClass` instances. 

627 

628 This class is a singleton, with each instance sharing the pool of 

629 StorageClasses. Since code can not know whether it is the first 

630 time the instance has been created, the constructor takes no arguments. 

631 To populate the factory with storage classes, a call to 

632 `~StorageClassFactory.addFromConfig()` should be made. 

633 

634 Parameters 

635 ---------- 

636 config : `StorageClassConfig` or `str`, optional 

637 Load configuration. In a ButlerConfig` the relevant configuration 

638 is located in the ``storageClasses`` section. 

639 """ 

640 

641 def __init__(self, config: StorageClassConfig | str | None = None): 

642 self._storageClasses: dict[str, StorageClass] = {} 

643 self._configs: list[StorageClassConfig] = [] 

644 

645 # Always seed with the default config 

646 self.addFromConfig(StorageClassConfig()) 

647 

648 if config is not None: 648 ↛ 649line 648 didn't jump to line 649, because the condition on line 648 was never true

649 self.addFromConfig(config) 

650 

651 def __str__(self) -> str: 

652 """Return summary of factory. 

653 

654 Returns 

655 ------- 

656 summary : `str` 

657 Summary of the factory status. 

658 """ 

659 sep = "\n" 

660 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

661 

662StorageClasses 

663-------------- 

664{sep.join(f"{s}: {self._storageClasses[s]!r}" for s in sorted(self._storageClasses))} 

665""" 

666 

667 def __contains__(self, storageClassOrName: StorageClass | str) -> bool: 

668 """Indicate whether the storage class exists in the factory. 

669 

670 Parameters 

671 ---------- 

672 storageClassOrName : `str` or `StorageClass` 

673 If `str` is given existence of the named StorageClass 

674 in the factory is checked. If `StorageClass` is given 

675 existence and equality are checked. 

676 

677 Returns 

678 ------- 

679 in : `bool` 

680 True if the supplied string is present, or if the supplied 

681 `StorageClass` is present and identical. 

682 

683 Notes 

684 ----- 

685 The two different checks (one for "key" and one for "value") based on 

686 the type of the given argument mean that it is possible for 

687 StorageClass.name to be in the factory but StorageClass to not be 

688 in the factory. 

689 """ 

690 if isinstance(storageClassOrName, str): 690 ↛ 692line 690 didn't jump to line 692, because the condition on line 690 was never false

691 return storageClassOrName in self._storageClasses 

692 elif isinstance(storageClassOrName, StorageClass) and storageClassOrName.name in self._storageClasses: 

693 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

694 return False 

695 

696 def __len__(self) -> int: 

697 return len(self._storageClasses) 

698 

699 def __iter__(self) -> Iterator[str]: 

700 return iter(self._storageClasses) 

701 

702 def values(self) -> ValuesView[StorageClass]: 

703 return self._storageClasses.values() 

704 

705 def keys(self) -> KeysView[str]: 

706 return self._storageClasses.keys() 

707 

708 def items(self) -> ItemsView[str, StorageClass]: 

709 return self._storageClasses.items() 

710 

711 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

712 """Add more `StorageClass` definitions from a config file. 

713 

714 Parameters 

715 ---------- 

716 config : `StorageClassConfig`, `Config` or `str` 

717 Storage class configuration. Can contain a ``storageClasses`` 

718 key if part of a global configuration. 

719 """ 

720 sconfig = StorageClassConfig(config) 

721 self._configs.append(sconfig) 

722 

723 # Since we can not assume that we will get definitions of 

724 # components or parents before their classes are defined 

725 # we have a helper function that we can call recursively 

726 # to extract definitions from the configuration. 

727 def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> None: 

728 # Maybe we've already processed this through recursion 

729 if name not in _sconfig: 

730 return 

731 info = _sconfig.pop(name) 

732 

733 # Always create the storage class so we can ensure that 

734 # we are not trying to overwrite with a different definition 

735 components = None 

736 

737 # Extract scalar items from dict that are needed for 

738 # StorageClass Constructor 

739 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

740 

741 if "converters" in info: 

742 storageClassKwargs["converters"] = info["converters"].toDict() 

743 

744 for compName in ("components", "derivedComponents"): 

745 if compName not in info: 

746 continue 

747 components = {} 

748 for cname, ctype in info[compName].items(): 

749 if ctype not in self: 

750 processStorageClass(ctype, sconfig, msg) 

751 components[cname] = self.getStorageClass(ctype) 

752 

753 # Fill in other items 

754 storageClassKwargs[compName] = components 

755 

756 # Create the new storage class and register it 

757 baseClass = None 

758 if "inheritsFrom" in info: 

759 baseName = info["inheritsFrom"] 

760 

761 # The inheritsFrom feature requires that the storage class 

762 # being inherited from is itself a subclass of StorageClass 

763 # that was created with makeNewStorageClass. If it was made 

764 # and registered with a simple StorageClass constructor it 

765 # cannot be used here and we try to recreate it. 

766 if baseName in self: 766 ↛ 778line 766 didn't jump to line 778, because the condition on line 766 was never false

767 baseClass = type(self.getStorageClass(baseName)) 

768 if baseClass is StorageClass: 768 ↛ 769line 768 didn't jump to line 769, because the condition on line 768 was never true

769 log.warning( 

770 "Storage class %s is requested to inherit from %s but that storage class " 

771 "has not been defined to be a subclass of StorageClass and so can not " 

772 "be used. Attempting to recreate parent class from current configuration.", 

773 name, 

774 baseName, 

775 ) 

776 processStorageClass(baseName, sconfig, msg) 

777 else: 

778 processStorageClass(baseName, sconfig, msg) 

779 baseClass = type(self.getStorageClass(baseName)) 

780 if baseClass is StorageClass: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true

781 raise TypeError( 

782 f"Configuration for storage class {name} requests to inherit from " 

783 f" storage class {baseName} but that class is not defined correctly." 

784 ) 

785 

786 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

787 newStorageClass = newStorageClassType() 

788 self.registerStorageClass(newStorageClass, msg=msg) 

789 

790 # In case there is a problem, construct a context message for any 

791 # error reporting. 

792 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

793 context = f"when adding definitions from {', '.join(files)}" if files else "" 

794 log.debug("Adding definitions from config %s", ", ".join(files)) 

795 

796 for name in list(sconfig.keys()): 

797 processStorageClass(name, sconfig, context) 

798 

799 @staticmethod 

800 def makeNewStorageClass( 

801 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any 

802 ) -> type[StorageClass]: 

803 """Create a new Python class as a subclass of `StorageClass`. 

804 

805 Parameters 

806 ---------- 

807 name : `str` 

808 Name to use for this class. 

809 baseClass : `type`, optional 

810 Base class for this `StorageClass`. Must be either `StorageClass` 

811 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

812 be used. 

813 **kwargs 

814 Additional parameter values to use as defaults for this class. 

815 This can include ``components``, ``parameters``, 

816 ``derivedComponents``, and ``converters``. 

817 

818 Returns 

819 ------- 

820 newtype : `type` subclass of `StorageClass` 

821 Newly created Python type. 

822 """ 

823 if baseClass is None: 

824 baseClass = StorageClass 

825 if not issubclass(baseClass, StorageClass): 825 ↛ 826line 825 didn't jump to line 826, because the condition on line 825 was never true

826 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

827 

828 # convert the arguments to use different internal names 

829 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

830 clsargs["_cls_name"] = name 

831 

832 # Some container items need to merge with the base class values 

833 # so that a child can inherit but override one bit. 

834 # lists (which you get from configs) are treated as sets for this to 

835 # work consistently. 

836 for k in ("components", "parameters", "derivedComponents", "converters"): 

837 classKey = f"_cls_{k}" 

838 if classKey in clsargs: 

839 baseValue = getattr(baseClass, classKey, None) 

840 if baseValue is not None: 

841 currentValue = clsargs[classKey] 

842 if isinstance(currentValue, dict): 842 ↛ 845line 842 didn't jump to line 845, because the condition on line 842 was never false

843 newValue = baseValue.copy() 

844 else: 

845 newValue = set(baseValue) 

846 newValue.update(currentValue) 

847 clsargs[classKey] = newValue 

848 

849 # If we have parameters they should be a frozen set so that the 

850 # parameters in the class can not be modified. 

851 pk = "_cls_parameters" 

852 if pk in clsargs: 

853 clsargs[pk] = frozenset(clsargs[pk]) 

854 

855 return type(f"StorageClass{name}", (baseClass,), clsargs) 

856 

857 def getStorageClass(self, storageClassName: str) -> StorageClass: 

858 """Get a StorageClass instance associated with the supplied name. 

859 

860 Parameters 

861 ---------- 

862 storageClassName : `str` 

863 Name of the storage class to retrieve. 

864 

865 Returns 

866 ------- 

867 instance : `StorageClass` 

868 Instance of the correct `StorageClass`. 

869 

870 Raises 

871 ------ 

872 KeyError 

873 The requested storage class name is not registered. 

874 """ 

875 return self._storageClasses[storageClassName] 

876 

877 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

878 """Find the storage class associated with this python type. 

879 

880 Parameters 

881 ---------- 

882 pytype : `type` 

883 The Python type to be matched. 

884 compare_types : `bool`, optional 

885 If `False`, the type will be checked against name of the python 

886 type. This comparison is always done first. If `True` and the 

887 string comparison failed, each candidate storage class will be 

888 forced to have its type imported. This can be significantly slower. 

889 

890 Returns 

891 ------- 

892 storageClass : `StorageClass` 

893 The matching storage class. 

894 

895 Raises 

896 ------ 

897 KeyError 

898 Raised if no match could be found. 

899 

900 Notes 

901 ----- 

902 It is possible for a python type to be associated with multiple 

903 storage classes. This method will currently return the first that 

904 matches. 

905 """ 

906 result = self._find_storage_class(pytype, False) 

907 if result: 

908 return result 

909 

910 if compare_types: 

911 # The fast comparison failed and we were asked to try the 

912 # variant that might involve code imports. 

913 result = self._find_storage_class(pytype, True) 

914 if result: 

915 return result 

916 

917 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}") 

918 

919 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

920 """Iterate through all storage classes to find a match. 

921 

922 Parameters 

923 ---------- 

924 pytype : `type` 

925 The Python type to be matched. 

926 compare_types : `bool`, optional 

927 Whether to use type name matching or explicit type matching. 

928 The latter can be slower. 

929 

930 Returns 

931 ------- 

932 storageClass : `StorageClass` or `None` 

933 The matching storage class, or `None` if no match was found. 

934 

935 Notes 

936 ----- 

937 Helper method for ``findStorageClass``. 

938 """ 

939 for storageClass in self.values(): 

940 if storageClass.is_type(pytype, compare_types=compare_types): 

941 return storageClass 

942 return None 

943 

944 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

945 """Store the `StorageClass` in the factory. 

946 

947 Will be indexed by `StorageClass.name` and will return instances 

948 of the supplied `StorageClass`. 

949 

950 Parameters 

951 ---------- 

952 storageClass : `StorageClass` 

953 Type of the Python `StorageClass` to register. 

954 msg : `str`, optional 

955 Additional message string to be included in any error message. 

956 

957 Raises 

958 ------ 

959 ValueError 

960 If a storage class has already been registered with 

961 that storage class name and the previous definition differs. 

962 """ 

963 if storageClass.name in self._storageClasses: 963 ↛ 964line 963 didn't jump to line 964, because the condition on line 963 was never true

964 existing = self.getStorageClass(storageClass.name) 

965 if existing != storageClass: 

966 errmsg = f" {msg}" if msg else "" 

967 raise ValueError( 

968 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

969 f"differs from current definition ({existing!r}){errmsg}" 

970 ) 

971 if type(existing) is StorageClass and type(storageClass) is not StorageClass: 

972 # Replace generic with specialist subclass equivalent. 

973 self._storageClasses[storageClass.name] = storageClass 

974 else: 

975 self._storageClasses[storageClass.name] = storageClass 

976 

977 def _unregisterStorageClass(self, storageClassName: str) -> None: 

978 """Remove the named StorageClass from the factory. 

979 

980 Parameters 

981 ---------- 

982 storageClassName : `str` 

983 Name of storage class to remove. 

984 

985 Raises 

986 ------ 

987 KeyError 

988 The named storage class is not registered. 

989 

990 Notes 

991 ----- 

992 This method is intended to simplify testing of StorageClassFactory 

993 functionality and it is not expected to be required for normal usage. 

994 """ 

995 del self._storageClasses[storageClassName] 

996 

997 def reset(self) -> None: 

998 """Remove all storage class entries from factory and reset to 

999 initial state. 

1000 

1001 This is useful for test code where a known start state is useful. 

1002 """ 

1003 self._storageClasses.clear() 

1004 # Seed with the default config. 

1005 self.addFromConfig(StorageClassConfig())