Coverage for python/lsst/daf/butler/_storage_class.py: 45%

367 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 10:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for Storage Classes.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

33 

34import builtins 

35import itertools 

36import logging 

37from collections import ChainMap 

38from collections.abc import Callable, Collection, Mapping, Sequence, Set 

39from threading import RLock 

40from typing import Any 

41 

42from lsst.utils import doImportType 

43from lsst.utils.classes import Singleton 

44from lsst.utils.introspection import get_full_type_name 

45 

46from ._config import Config, ConfigSubset 

47from ._config_support import LookupKey 

48from ._storage_class_delegate import StorageClassDelegate 

49 

50log = logging.getLogger(__name__) 

51 

52 

53class StorageClassConfig(ConfigSubset): 

54 """Configuration class for defining Storage Classes.""" 

55 

56 component = "storageClasses" 

57 defaultConfigFile = "storageClasses.yaml" 

58 

59 

60class StorageClass: 

61 """Class describing how a label maps to a particular Python type. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name to use for this class. 

67 pytype : `type` or `str` 

68 Python type (or name of type) to associate with the `StorageClass`. 

69 components : `dict`, optional 

70 `dict` mapping name of a component to another `StorageClass`. 

71 derivedComponents : `dict`, optional 

72 `dict` mapping name of a derived component to another `StorageClass`. 

73 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

74 Parameters understood by this `StorageClass` that can control 

75 reading of data from datastores. 

76 delegate : `str`, optional 

77 Fully qualified name of class supporting assembly and disassembly 

78 of a `pytype` instance. 

79 converters : `dict` [`str`, `str`], optional 

80 Mapping of python type to function that can be called to convert 

81 that python type to the valid type of this storage class. 

82 """ 

83 

84 _cls_name: str = "BaseStorageClass" 

85 _cls_components: dict[str, StorageClass] | None = None 

86 _cls_derivedComponents: dict[str, StorageClass] | None = None 

87 _cls_parameters: Set[str] | Sequence[str] | None = None 

88 _cls_delegate: str | None = None 

89 _cls_pytype: type | str | None = None 

90 _cls_converters: dict[str, str] | None = None 

91 

92 def __init__( 

93 self, 

94 name: str | None = None, 

95 pytype: type | str | None = None, 

96 components: dict[str, StorageClass] | None = None, 

97 derivedComponents: dict[str, StorageClass] | None = None, 

98 parameters: Sequence[str] | Set[str] | None = None, 

99 delegate: str | None = None, 

100 converters: dict[str, str] | None = None, 

101 ): 

102 if name is None: 

103 name = self._cls_name 

104 if pytype is None: 104 ↛ 106line 104 didn't jump to line 106, because the condition on line 104 was never false

105 pytype = self._cls_pytype 

106 if components is None: 106 ↛ 108line 106 didn't jump to line 108, because the condition on line 106 was never false

107 components = self._cls_components 

108 if derivedComponents is None: 108 ↛ 110line 108 didn't jump to line 110, because the condition on line 108 was never false

109 derivedComponents = self._cls_derivedComponents 

110 if parameters is None: 110 ↛ 112line 110 didn't jump to line 112, because the condition on line 110 was never false

111 parameters = self._cls_parameters 

112 if delegate is None: 112 ↛ 116line 112 didn't jump to line 116, because the condition on line 112 was never false

113 delegate = self._cls_delegate 

114 

115 # Merge converters with class defaults. 

116 self._converters = {} 

117 if self._cls_converters is not None: 

118 self._converters.update(self._cls_converters) 

119 if converters: 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 self._converters.update(converters) 

121 

122 # Version of converters where the python types have been 

123 # Do not try to import anything until needed. 

124 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None 

125 

126 self.name = name 

127 

128 if pytype is None: 

129 pytype = object 

130 

131 self._pytype: type | None 

132 if not isinstance(pytype, str): 

133 # Already have a type so store it and get the name 

134 self._pytypeName = get_full_type_name(pytype) 

135 self._pytype = pytype 

136 else: 

137 # Store the type name and defer loading of type 

138 self._pytypeName = pytype 

139 self._pytype = None 

140 

141 if components is not None: 

142 if len(components) == 1: 142 ↛ 143line 142 didn't jump to line 143, because the condition on line 142 was never true

143 raise ValueError( 

144 f"Composite storage class {name} is not allowed to have" 

145 f" only one component '{next(iter(components))}'." 

146 " Did you mean it to be a derived component?" 

147 ) 

148 self._components = components 

149 else: 

150 self._components = {} 

151 self._derivedComponents = derivedComponents if derivedComponents is not None else {} 

152 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

153 # if the delegate is not None also set it and clear the default 

154 # delegate 

155 self._delegate: type | None 

156 self._delegateClassName: str | None 

157 if delegate is not None: 

158 self._delegateClassName = delegate 

159 self._delegate = None 

160 elif components is not None: 160 ↛ 163line 160 didn't jump to line 163, because the condition on line 160 was never true

161 # We set a default delegate for composites so that a class is 

162 # guaranteed to support something if it is a composite. 

163 log.debug("Setting default delegate for %s", self.name) 

164 self._delegate = StorageClassDelegate 

165 self._delegateClassName = get_full_type_name(self._delegate) 

166 else: 

167 self._delegate = None 

168 self._delegateClassName = None 

169 

170 @property 

171 def components(self) -> Mapping[str, StorageClass]: 

172 """Return the components associated with this `StorageClass`.""" 

173 return self._components 

174 

175 @property 

176 def derivedComponents(self) -> Mapping[str, StorageClass]: 

177 """Return derived components associated with `StorageClass`.""" 

178 return self._derivedComponents 

179 

180 @property 

181 def converters(self) -> Mapping[str, str]: 

182 """Return the type converters supported by this `StorageClass`.""" 

183 return self._converters 

184 

185 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]: 

186 """Return the type converters as python types.""" 

187 if self._converters_by_type is None: 

188 self._converters_by_type = {} 

189 

190 # Loop over list because the dict can be edited in loop. 

191 for candidate_type_str, converter_str in list(self.converters.items()): 

192 if hasattr(builtins, candidate_type_str): 

193 candidate_type = getattr(builtins, candidate_type_str) 

194 else: 

195 try: 

196 candidate_type = doImportType(candidate_type_str) 

197 except ImportError as e: 

198 log.warning( 

199 "Unable to import type %s associated with storage class %s (%s)", 

200 candidate_type_str, 

201 self.name, 

202 e, 

203 ) 

204 del self._converters[candidate_type_str] 

205 continue 

206 

207 try: 

208 converter = doImportType(converter_str) 

209 except ImportError as e: 

210 log.warning( 

211 "Unable to import conversion function %s associated with storage class %s " 

212 "required to convert type %s (%s)", 

213 converter_str, 

214 self.name, 

215 candidate_type_str, 

216 e, 

217 ) 

218 del self._converters[candidate_type_str] 

219 continue 

220 if not callable(converter): 

221 # doImportType is annotated to return a Type but in actual 

222 # fact it can return Any except ModuleType because package 

223 # variables can be accessed. This make mypy believe it 

224 # is impossible for the return value to not be a callable 

225 # so we must ignore the warning. 

226 log.warning( # type: ignore 

227 "Conversion function %s associated with storage class " 

228 "%s to convert type %s is not a callable.", 

229 converter_str, 

230 self.name, 

231 candidate_type_str, 

232 ) 

233 del self._converters[candidate_type_str] 

234 continue 

235 self._converters_by_type[candidate_type] = converter 

236 return self._converters_by_type 

237 

238 @property 

239 def parameters(self) -> set[str]: 

240 """Return `set` of names of supported parameters.""" 

241 return set(self._parameters) 

242 

243 @property 

244 def pytype(self) -> type: 

245 """Return Python type associated with this `StorageClass`.""" 

246 if self._pytype is not None: 

247 return self._pytype 

248 

249 if hasattr(builtins, self._pytypeName): 

250 pytype = getattr(builtins, self._pytypeName) 

251 else: 

252 pytype = doImportType(self._pytypeName) 

253 self._pytype = pytype 

254 return self._pytype 

255 

256 @property 

257 def delegateClass(self) -> type | None: 

258 """Class to use to delegate type-specific actions.""" 

259 if self._delegate is not None: 

260 return self._delegate 

261 if self._delegateClassName is None: 

262 return None 

263 delegate_class = doImportType(self._delegateClassName) 

264 self._delegate = delegate_class 

265 return self._delegate 

266 

267 def allComponents(self) -> Mapping[str, StorageClass]: 

268 """Return all defined components. 

269 

270 This mapping includes all the derived and read/write components 

271 for the corresponding storage class. 

272 

273 Returns 

274 ------- 

275 comp : `dict` of [`str`, `StorageClass`] 

276 The component name to storage class mapping. 

277 """ 

278 return ChainMap(self._components, self._derivedComponents) 

279 

280 def delegate(self) -> StorageClassDelegate: 

281 """Return an instance of a storage class delegate. 

282 

283 Returns 

284 ------- 

285 delegate : `StorageClassDelegate` 

286 Instance of the delegate associated with this `StorageClass`. 

287 The delegate is constructed with this `StorageClass`. 

288 

289 Raises 

290 ------ 

291 TypeError 

292 This StorageClass has no associated delegate. 

293 """ 

294 cls = self.delegateClass 

295 if cls is None: 

296 raise TypeError(f"No delegate class is associated with StorageClass {self.name}") 

297 return cls(storageClass=self) 

298 

299 def isComposite(self) -> bool: 

300 """Return Boolean indicating whether this is a composite or not. 

301 

302 Returns 

303 ------- 

304 isComposite : `bool` 

305 `True` if this `StorageClass` is a composite, `False` 

306 otherwise. 

307 """ 

308 if self.components: 

309 return True 

310 return False 

311 

312 def _lookupNames(self) -> tuple[LookupKey, ...]: 

313 """Keys to use when looking up this DatasetRef in a configuration. 

314 

315 The names are returned in order of priority. 

316 

317 Returns 

318 ------- 

319 names : `tuple` of `LookupKey` 

320 Tuple of a `LookupKey` using the `StorageClass` name. 

321 """ 

322 return (LookupKey(name=self.name),) 

323 

324 def knownParameters(self) -> set[str]: 

325 """Return set of all parameters known to this `StorageClass`. 

326 

327 The set includes parameters understood by components of a composite. 

328 

329 Returns 

330 ------- 

331 known : `set` 

332 All parameter keys of this `StorageClass` and the component 

333 storage classes. 

334 """ 

335 known = set(self._parameters) 

336 for sc in self.components.values(): 

337 known.update(sc.knownParameters()) 

338 return known 

339 

340 def validateParameters(self, parameters: Collection | None = None) -> None: 

341 """Check that the parameters are known to this `StorageClass`. 

342 

343 Does not check the values. 

344 

345 Parameters 

346 ---------- 

347 parameters : `~collections.abc.Collection`, optional 

348 Collection containing the parameters. Can be `dict`-like or 

349 `set`-like. The parameter values are not checked. 

350 If no parameters are supplied, always returns without error. 

351 

352 Raises 

353 ------ 

354 KeyError 

355 Some parameters are not understood by this `StorageClass`. 

356 """ 

357 # No parameters is always okay 

358 if not parameters: 

359 return 

360 

361 # Extract the important information into a set. Works for dict and 

362 # list. 

363 external = set(parameters) 

364 

365 diff = external - self.knownParameters() 

366 if diff: 

367 s = "s" if len(diff) > 1 else "" 

368 unknown = "', '".join(diff) 

369 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

370 

371 def filterParameters( 

372 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None 

373 ) -> Mapping[str, Any]: 

374 """Filter out parameters that are not known to this `StorageClass`. 

375 

376 Parameters 

377 ---------- 

378 parameters : `~collections.abc.Mapping`, optional 

379 Candidate parameters. Can be `None` if no parameters have 

380 been provided. 

381 subset : `~collections.abc.Collection`, optional 

382 Subset of supported parameters that the caller is interested 

383 in using. The subset must be known to the `StorageClass` 

384 if specified. If `None` the supplied parameters will all 

385 be checked, else only the keys in this set will be checked. 

386 

387 Returns 

388 ------- 

389 filtered : `~collections.abc.Mapping` 

390 Valid parameters. Empty `dict` if none are suitable. 

391 

392 Raises 

393 ------ 

394 ValueError 

395 Raised if the provided subset is not a subset of the supported 

396 parameters or if it is an empty set. 

397 """ 

398 if not parameters: 

399 return {} 

400 

401 known = self.knownParameters() 

402 

403 if subset is not None: 

404 if not subset: 

405 raise ValueError("Specified a parameter subset but it was empty") 

406 subset = set(subset) 

407 if not subset.issubset(known): 

408 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})") 

409 wanted = subset 

410 else: 

411 wanted = known 

412 

413 return {k: parameters[k] for k in wanted if k in parameters} 

414 

415 def validateInstance(self, instance: Any) -> bool: 

416 """Check that the supplied Python object has the expected Python type. 

417 

418 Parameters 

419 ---------- 

420 instance : `object` 

421 Object to check. 

422 

423 Returns 

424 ------- 

425 isOk : `bool` 

426 True if the supplied instance object can be handled by this 

427 `StorageClass`, False otherwise. 

428 """ 

429 return isinstance(instance, self.pytype) 

430 

431 def is_type(self, other: type, compare_types: bool = False) -> bool: 

432 """Return Boolean indicating whether the supplied type matches 

433 the type in this `StorageClass`. 

434 

435 Parameters 

436 ---------- 

437 other : `type` 

438 The type to be checked. 

439 compare_types : `bool`, optional 

440 If `True` the python type will be used in the comparison 

441 if the type names do not match. This may trigger an import 

442 of code and so can be slower. 

443 

444 Returns 

445 ------- 

446 match : `bool` 

447 `True` if the types are equal. 

448 

449 Notes 

450 ----- 

451 If this `StorageClass` has not yet imported the Python type the 

452 check is done against the full type name, this prevents an attempt 

453 to import the type when it will likely not match. 

454 """ 

455 if self._pytype: 

456 return self._pytype is other 

457 

458 other_name = get_full_type_name(other) 

459 if self._pytypeName == other_name: 

460 return True 

461 

462 if compare_types: 

463 # Must protect against the import failing. 

464 try: 

465 return self.pytype is other 

466 except Exception: 

467 pass 

468 

469 return False 

470 

471 def can_convert(self, other: StorageClass) -> bool: 

472 """Return `True` if this storage class can convert python types 

473 in the other storage class. 

474 

475 Parameters 

476 ---------- 

477 other : `StorageClass` 

478 The storage class to check. 

479 

480 Returns 

481 ------- 

482 can : `bool` 

483 `True` if this storage class has a registered converter for 

484 the python type associated with the other storage class. That 

485 converter will convert the other python type to the one associated 

486 with this storage class. 

487 """ 

488 if other.name == self.name: 

489 # Identical storage classes are compatible. 

490 return True 

491 

492 # It may be that the storage class being compared is not 

493 # available because the python type can't be imported. In that 

494 # case conversion must be impossible. 

495 try: 

496 other_pytype = other.pytype 

497 except Exception: 

498 return False 

499 

500 # Or even this storage class itself can not have the type imported. 

501 try: 

502 self_pytype = self.pytype 

503 except Exception: 

504 return False 

505 

506 if issubclass(other_pytype, self_pytype): 

507 # Storage classes have different names but the same python type. 

508 return True 

509 

510 for candidate_type in self._get_converters_by_type(): 

511 if issubclass(other_pytype, candidate_type): 

512 return True 

513 return False 

514 

515 def coerce_type(self, incorrect: Any) -> Any: 

516 """Coerce the supplied incorrect instance to the python type 

517 associated with this `StorageClass`. 

518 

519 Parameters 

520 ---------- 

521 incorrect : `object` 

522 An object that might be the incorrect type. 

523 

524 Returns 

525 ------- 

526 correct : `object` 

527 An object that matches the python type of this `StorageClass`. 

528 Can be the same object as given. If `None`, `None` will be 

529 returned. 

530 

531 Raises 

532 ------ 

533 TypeError 

534 Raised if no conversion can be found. 

535 """ 

536 if incorrect is None: 

537 return None 

538 

539 # Possible this is the correct type already. 

540 if self.validateInstance(incorrect): 

541 return incorrect 

542 

543 # Check each registered converter. 

544 for candidate_type, converter in self._get_converters_by_type().items(): 

545 if isinstance(incorrect, candidate_type): 

546 try: 

547 return converter(incorrect) 

548 except Exception: 

549 log.error( 

550 "Converter %s failed to convert type %s", 

551 get_full_type_name(converter), 

552 get_full_type_name(incorrect), 

553 ) 

554 raise 

555 raise TypeError( 

556 "Type does not match and no valid converter found to convert" 

557 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'" 

558 ) 

559 

560 def __eq__(self, other: Any) -> bool: 

561 """Equality checks name, pytype name, delegate name, and components.""" 

562 if not isinstance(other, StorageClass): 

563 return NotImplemented 

564 

565 if self.name != other.name: 

566 return False 

567 

568 # We must compare pytype and delegate by name since we do not want 

569 # to trigger an import of external module code here 

570 if self._delegateClassName != other._delegateClassName: 

571 return False 

572 if self._pytypeName != other._pytypeName: 

573 return False 

574 

575 # Ensure we have the same component keys in each 

576 if set(self.components.keys()) != set(other.components.keys()): 

577 return False 

578 

579 # Same parameters 

580 if self.parameters != other.parameters: 

581 return False 

582 

583 # Ensure that all the components have the same type 

584 return all(self.components[k] == other.components[k] for k in self.components) 

585 

586 def __hash__(self) -> int: 

587 return hash(self.name) 

588 

589 def __repr__(self) -> str: 

590 optionals: dict[str, Any] = {} 

591 if self._pytypeName != "object": 

592 optionals["pytype"] = self._pytypeName 

593 if self._delegateClassName is not None: 

594 optionals["delegate"] = self._delegateClassName 

595 if self._parameters: 

596 optionals["parameters"] = self._parameters 

597 if self.components: 

598 optionals["components"] = self.components 

599 if self.converters: 

600 optionals["converters"] = self.converters 

601 

602 # order is preserved in the dict 

603 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

604 

605 # Start with mandatory fields 

606 r = f"{self.__class__.__name__}({self.name!r}" 

607 if options: 

608 r = r + ", " + options 

609 r = r + ")" 

610 return r 

611 

612 def __str__(self) -> str: 

613 return self.name 

614 

615 

616class StorageClassFactory(metaclass=Singleton): 

617 """Factory for `StorageClass` instances. 

618 

619 This class is a singleton, with each instance sharing the pool of 

620 StorageClasses. Since code can not know whether it is the first 

621 time the instance has been created, the constructor takes no arguments. 

622 To populate the factory with storage classes, a call to 

623 `~StorageClassFactory.addFromConfig()` should be made. 

624 

625 Parameters 

626 ---------- 

627 config : `StorageClassConfig` or `str`, optional 

628 Load configuration. In a ButlerConfig` the relevant configuration 

629 is located in the ``storageClasses`` section. 

630 """ 

631 

632 def __init__(self, config: StorageClassConfig | str | None = None): 

633 self._storageClasses: dict[str, StorageClass] = {} 

634 self._configs: list[StorageClassConfig] = [] 

635 self._lock = RLock() 

636 

637 # Always seed with the default config 

638 self.addFromConfig(StorageClassConfig()) 

639 

640 if config is not None: 640 ↛ 641line 640 didn't jump to line 641, because the condition on line 640 was never true

641 self.addFromConfig(config) 

642 

643 def __str__(self) -> str: 

644 """Return summary of factory. 

645 

646 Returns 

647 ------- 

648 summary : `str` 

649 Summary of the factory status. 

650 """ 

651 with self._lock: 

652 sep = "\n" 

653 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

654 

655StorageClasses 

656-------------- 

657{sep.join(f"{s}: {self._storageClasses[s]!r}" for s in sorted(self._storageClasses))} 

658""" 

659 

660 def __contains__(self, storageClassOrName: StorageClass | str) -> bool: 

661 """Indicate whether the storage class exists in the factory. 

662 

663 Parameters 

664 ---------- 

665 storageClassOrName : `str` or `StorageClass` 

666 If `str` is given existence of the named StorageClass 

667 in the factory is checked. If `StorageClass` is given 

668 existence and equality are checked. 

669 

670 Returns 

671 ------- 

672 in : `bool` 

673 True if the supplied string is present, or if the supplied 

674 `StorageClass` is present and identical. 

675 

676 Notes 

677 ----- 

678 The two different checks (one for "key" and one for "value") based on 

679 the type of the given argument mean that it is possible for 

680 StorageClass.name to be in the factory but StorageClass to not be 

681 in the factory. 

682 """ 

683 with self._lock: 

684 if isinstance(storageClassOrName, str): 684 ↛ 686line 684 didn't jump to line 686, because the condition on line 684 was never false

685 return storageClassOrName in self._storageClasses 

686 elif ( 

687 isinstance(storageClassOrName, StorageClass) 

688 and storageClassOrName.name in self._storageClasses 

689 ): 

690 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

691 return False 

692 

693 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None: 

694 """Add more `StorageClass` definitions from a config file. 

695 

696 Parameters 

697 ---------- 

698 config : `StorageClassConfig`, `Config` or `str` 

699 Storage class configuration. Can contain a ``storageClasses`` 

700 key if part of a global configuration. 

701 """ 

702 sconfig = StorageClassConfig(config) 

703 

704 # Since we can not assume that we will get definitions of 

705 # components or parents before their classes are defined 

706 # we have a helper function that we can call recursively 

707 # to extract definitions from the configuration. 

708 def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> None: 

709 # Maybe we've already processed this through recursion 

710 if name not in _sconfig: 

711 return 

712 info = _sconfig.pop(name) 

713 

714 # Always create the storage class so we can ensure that 

715 # we are not trying to overwrite with a different definition 

716 components = None 

717 

718 # Extract scalar items from dict that are needed for 

719 # StorageClass Constructor 

720 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info} 

721 

722 if "converters" in info: 

723 storageClassKwargs["converters"] = info["converters"].toDict() 

724 

725 for compName in ("components", "derivedComponents"): 

726 if compName not in info: 

727 continue 

728 components = {} 

729 for cname, ctype in info[compName].items(): 

730 if ctype not in self: 

731 processStorageClass(ctype, sconfig, msg) 

732 components[cname] = self.getStorageClass(ctype) 

733 

734 # Fill in other items 

735 storageClassKwargs[compName] = components 

736 

737 # Create the new storage class and register it 

738 baseClass = None 

739 if "inheritsFrom" in info: 

740 baseName = info["inheritsFrom"] 

741 

742 # The inheritsFrom feature requires that the storage class 

743 # being inherited from is itself a subclass of StorageClass 

744 # that was created with makeNewStorageClass. If it was made 

745 # and registered with a simple StorageClass constructor it 

746 # cannot be used here and we try to recreate it. 

747 if baseName in self: 747 ↛ 759line 747 didn't jump to line 759, because the condition on line 747 was never false

748 baseClass = type(self.getStorageClass(baseName)) 

749 if baseClass is StorageClass: 749 ↛ 750line 749 didn't jump to line 750, because the condition on line 749 was never true

750 log.warning( 

751 "Storage class %s is requested to inherit from %s but that storage class " 

752 "has not been defined to be a subclass of StorageClass and so can not " 

753 "be used. Attempting to recreate parent class from current configuration.", 

754 name, 

755 baseName, 

756 ) 

757 processStorageClass(baseName, sconfig, msg) 

758 else: 

759 processStorageClass(baseName, sconfig, msg) 

760 baseClass = type(self.getStorageClass(baseName)) 

761 if baseClass is StorageClass: 761 ↛ 762line 761 didn't jump to line 762, because the condition on line 761 was never true

762 raise TypeError( 

763 f"Configuration for storage class {name} requests to inherit from " 

764 f" storage class {baseName} but that class is not defined correctly." 

765 ) 

766 

767 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

768 newStorageClass = newStorageClassType() 

769 self.registerStorageClass(newStorageClass, msg=msg) 

770 

771 # In case there is a problem, construct a context message for any 

772 # error reporting. 

773 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f] 

774 context = f"when adding definitions from {', '.join(files)}" if files else "" 

775 log.debug("Adding definitions from config %s", ", ".join(files)) 

776 

777 with self._lock: 

778 self._configs.append(sconfig) 

779 for name in list(sconfig.keys()): 

780 processStorageClass(name, sconfig, context) 

781 

782 @staticmethod 

783 def makeNewStorageClass( 

784 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any 

785 ) -> type[StorageClass]: 

786 """Create a new Python class as a subclass of `StorageClass`. 

787 

788 Parameters 

789 ---------- 

790 name : `str` 

791 Name to use for this class. 

792 baseClass : `type`, optional 

793 Base class for this `StorageClass`. Must be either `StorageClass` 

794 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

795 be used. 

796 **kwargs 

797 Additional parameter values to use as defaults for this class. 

798 This can include ``components``, ``parameters``, 

799 ``derivedComponents``, and ``converters``. 

800 

801 Returns 

802 ------- 

803 newtype : `type` subclass of `StorageClass` 

804 Newly created Python type. 

805 """ 

806 if baseClass is None: 

807 baseClass = StorageClass 

808 if not issubclass(baseClass, StorageClass): 808 ↛ 809line 808 didn't jump to line 809, because the condition on line 808 was never true

809 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

810 

811 # convert the arguments to use different internal names 

812 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

813 clsargs["_cls_name"] = name 

814 

815 # Some container items need to merge with the base class values 

816 # so that a child can inherit but override one bit. 

817 # lists (which you get from configs) are treated as sets for this to 

818 # work consistently. 

819 for k in ("components", "parameters", "derivedComponents", "converters"): 

820 classKey = f"_cls_{k}" 

821 if classKey in clsargs: 

822 baseValue = getattr(baseClass, classKey, None) 

823 if baseValue is not None: 

824 currentValue = clsargs[classKey] 

825 if isinstance(currentValue, dict): 825 ↛ 828line 825 didn't jump to line 828, because the condition on line 825 was never false

826 newValue = baseValue.copy() 

827 else: 

828 newValue = set(baseValue) 

829 newValue.update(currentValue) 

830 clsargs[classKey] = newValue 

831 

832 # If we have parameters they should be a frozen set so that the 

833 # parameters in the class can not be modified. 

834 pk = "_cls_parameters" 

835 if pk in clsargs: 

836 clsargs[pk] = frozenset(clsargs[pk]) 

837 

838 return type(f"StorageClass{name}", (baseClass,), clsargs) 

839 

840 def getStorageClass(self, storageClassName: str) -> StorageClass: 

841 """Get a StorageClass instance associated with the supplied name. 

842 

843 Parameters 

844 ---------- 

845 storageClassName : `str` 

846 Name of the storage class to retrieve. 

847 

848 Returns 

849 ------- 

850 instance : `StorageClass` 

851 Instance of the correct `StorageClass`. 

852 

853 Raises 

854 ------ 

855 KeyError 

856 The requested storage class name is not registered. 

857 """ 

858 with self._lock: 

859 return self._storageClasses[storageClassName] 

860 

861 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass: 

862 """Find the storage class associated with this python type. 

863 

864 Parameters 

865 ---------- 

866 pytype : `type` 

867 The Python type to be matched. 

868 compare_types : `bool`, optional 

869 If `False`, the type will be checked against name of the python 

870 type. This comparison is always done first. If `True` and the 

871 string comparison failed, each candidate storage class will be 

872 forced to have its type imported. This can be significantly slower. 

873 

874 Returns 

875 ------- 

876 storageClass : `StorageClass` 

877 The matching storage class. 

878 

879 Raises 

880 ------ 

881 KeyError 

882 Raised if no match could be found. 

883 

884 Notes 

885 ----- 

886 It is possible for a python type to be associated with multiple 

887 storage classes. This method will currently return the first that 

888 matches. 

889 """ 

890 with self._lock: 

891 result = self._find_storage_class(pytype, False) 

892 if result: 

893 return result 

894 

895 if compare_types: 

896 # The fast comparison failed and we were asked to try the 

897 # variant that might involve code imports. 

898 result = self._find_storage_class(pytype, True) 

899 if result: 

900 return result 

901 

902 raise KeyError( 

903 f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}" 

904 ) 

905 

906 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None: 

907 """Iterate through all storage classes to find a match. 

908 

909 Parameters 

910 ---------- 

911 pytype : `type` 

912 The Python type to be matched. 

913 compare_types : `bool`, optional 

914 Whether to use type name matching or explicit type matching. 

915 The latter can be slower. 

916 

917 Returns 

918 ------- 

919 storageClass : `StorageClass` or `None` 

920 The matching storage class, or `None` if no match was found. 

921 

922 Notes 

923 ----- 

924 Helper method for ``findStorageClass``. 

925 """ 

926 with self._lock: 

927 for storageClass in self._storageClasses.values(): 

928 if storageClass.is_type(pytype, compare_types=compare_types): 

929 return storageClass 

930 return None 

931 

932 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None: 

933 """Store the `StorageClass` in the factory. 

934 

935 Will be indexed by `StorageClass.name` and will return instances 

936 of the supplied `StorageClass`. 

937 

938 Parameters 

939 ---------- 

940 storageClass : `StorageClass` 

941 Type of the Python `StorageClass` to register. 

942 msg : `str`, optional 

943 Additional message string to be included in any error message. 

944 

945 Raises 

946 ------ 

947 ValueError 

948 If a storage class has already been registered with 

949 that storage class name and the previous definition differs. 

950 """ 

951 with self._lock: 

952 if storageClass.name in self._storageClasses: 952 ↛ 953line 952 didn't jump to line 953, because the condition on line 952 was never true

953 existing = self.getStorageClass(storageClass.name) 

954 if existing != storageClass: 

955 errmsg = f" {msg}" if msg else "" 

956 raise ValueError( 

957 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) " 

958 f"differs from current definition ({existing!r}){errmsg}" 

959 ) 

960 if type(existing) is StorageClass and type(storageClass) is not StorageClass: 

961 # Replace generic with specialist subclass equivalent. 

962 self._storageClasses[storageClass.name] = storageClass 

963 else: 

964 self._storageClasses[storageClass.name] = storageClass 

965 

966 def _unregisterStorageClass(self, storageClassName: str) -> None: 

967 """Remove the named StorageClass from the factory. 

968 

969 Parameters 

970 ---------- 

971 storageClassName : `str` 

972 Name of storage class to remove. 

973 

974 Raises 

975 ------ 

976 KeyError 

977 The named storage class is not registered. 

978 

979 Notes 

980 ----- 

981 This method is intended to simplify testing of StorageClassFactory 

982 functionality and it is not expected to be required for normal usage. 

983 """ 

984 with self._lock: 

985 del self._storageClasses[storageClassName] 

986 

987 def reset(self) -> None: 

988 """Remove all storage class entries from factory and reset to 

989 initial state. 

990 

991 This is useful for test code where a known start state is useful. 

992 """ 

993 with self._lock: 

994 self._storageClasses.clear() 

995 # Seed with the default config. 

996 self.addFromConfig(StorageClassConfig())