Coverage for python/lsst/daf/butler/_storage_class.py: 45%
366 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Support for Storage Classes."""
30from __future__ import annotations
32__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig")
34import builtins
35import itertools
36import logging
37from collections import ChainMap
38from collections.abc import (
39 Callable,
40 Collection,
41 ItemsView,
42 Iterator,
43 KeysView,
44 Mapping,
45 Sequence,
46 Set,
47 ValuesView,
48)
49from typing import Any
51from lsst.utils import doImportType
52from lsst.utils.classes import Singleton
53from lsst.utils.introspection import get_full_type_name
55from ._config import Config, ConfigSubset
56from ._config_support import LookupKey
57from ._storage_class_delegate import StorageClassDelegate
59log = logging.getLogger(__name__)
62class StorageClassConfig(ConfigSubset):
63 """Configuration class for defining Storage Classes."""
65 component = "storageClasses"
66 defaultConfigFile = "storageClasses.yaml"
69class StorageClass:
70 """Class describing how a label maps to a particular Python type.
72 Parameters
73 ----------
74 name : `str`
75 Name to use for this class.
76 pytype : `type` or `str`
77 Python type (or name of type) to associate with the `StorageClass`
78 components : `dict`, optional
79 `dict` mapping name of a component to another `StorageClass`.
80 derivedComponents : `dict`, optional
81 `dict` mapping name of a derived component to another `StorageClass`.
82 parameters : `~collections.abc.Sequence` or `~collections.abc.Set`
83 Parameters understood by this `StorageClass` that can control
84 reading of data from datastores.
85 delegate : `str`, optional
86 Fully qualified name of class supporting assembly and disassembly
87 of a `pytype` instance.
88 converters : `dict` [`str`, `str`], optional
89 Mapping of python type to function that can be called to convert
90 that python type to the valid type of this storage class.
91 """
93 _cls_name: str = "BaseStorageClass"
94 _cls_components: dict[str, StorageClass] | None = None
95 _cls_derivedComponents: dict[str, StorageClass] | None = None
96 _cls_parameters: Set[str] | Sequence[str] | None = None
97 _cls_delegate: str | None = None
98 _cls_pytype: type | str | None = None
99 _cls_converters: dict[str, str] | None = None
101 def __init__(
102 self,
103 name: str | None = None,
104 pytype: type | str | None = None,
105 components: dict[str, StorageClass] | None = None,
106 derivedComponents: dict[str, StorageClass] | None = None,
107 parameters: Sequence[str] | Set[str] | None = None,
108 delegate: str | None = None,
109 converters: dict[str, str] | None = None,
110 ):
111 if name is None:
112 name = self._cls_name
113 if pytype is None: 113 ↛ 115line 113 didn't jump to line 115, because the condition on line 113 was never false
114 pytype = self._cls_pytype
115 if components is None: 115 ↛ 117line 115 didn't jump to line 117, because the condition on line 115 was never false
116 components = self._cls_components
117 if derivedComponents is None: 117 ↛ 119line 117 didn't jump to line 119, because the condition on line 117 was never false
118 derivedComponents = self._cls_derivedComponents
119 if parameters is None: 119 ↛ 121line 119 didn't jump to line 121, because the condition on line 119 was never false
120 parameters = self._cls_parameters
121 if delegate is None: 121 ↛ 125line 121 didn't jump to line 125, because the condition on line 121 was never false
122 delegate = self._cls_delegate
124 # Merge converters with class defaults.
125 self._converters = {}
126 if self._cls_converters is not None:
127 self._converters.update(self._cls_converters)
128 if converters: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true
129 self._converters.update(converters)
131 # Version of converters where the python types have been
132 # Do not try to import anything until needed.
133 self._converters_by_type: dict[type, Callable[[Any], Any]] | None = None
135 self.name = name
137 if pytype is None:
138 pytype = object
140 self._pytype: type | None
141 if not isinstance(pytype, str):
142 # Already have a type so store it and get the name
143 self._pytypeName = get_full_type_name(pytype)
144 self._pytype = pytype
145 else:
146 # Store the type name and defer loading of type
147 self._pytypeName = pytype
148 self._pytype = None
150 if components is not None:
151 if len(components) == 1: 151 ↛ 152line 151 didn't jump to line 152, because the condition on line 151 was never true
152 raise ValueError(
153 f"Composite storage class {name} is not allowed to have"
154 f" only one component '{next(iter(components))}'."
155 " Did you mean it to be a derived component?"
156 )
157 self._components = components
158 else:
159 self._components = {}
160 self._derivedComponents = derivedComponents if derivedComponents is not None else {}
161 self._parameters = frozenset(parameters) if parameters is not None else frozenset()
162 # if the delegate is not None also set it and clear the default
163 # delegate
164 self._delegate: type | None
165 self._delegateClassName: str | None
166 if delegate is not None:
167 self._delegateClassName = delegate
168 self._delegate = None
169 elif components is not None: 169 ↛ 172line 169 didn't jump to line 172, because the condition on line 169 was never true
170 # We set a default delegate for composites so that a class is
171 # guaranteed to support something if it is a composite.
172 log.debug("Setting default delegate for %s", self.name)
173 self._delegate = StorageClassDelegate
174 self._delegateClassName = get_full_type_name(self._delegate)
175 else:
176 self._delegate = None
177 self._delegateClassName = None
179 @property
180 def components(self) -> Mapping[str, StorageClass]:
181 """Return the components associated with this `StorageClass`."""
182 return self._components
184 @property
185 def derivedComponents(self) -> Mapping[str, StorageClass]:
186 """Return derived components associated with `StorageClass`."""
187 return self._derivedComponents
189 @property
190 def converters(self) -> Mapping[str, str]:
191 """Return the type converters supported by this `StorageClass`."""
192 return self._converters
194 def _get_converters_by_type(self) -> Mapping[type, Callable[[Any], Any]]:
195 """Return the type converters as python types."""
196 if self._converters_by_type is None:
197 self._converters_by_type = {}
199 # Loop over list because the dict can be edited in loop.
200 for candidate_type_str, converter_str in list(self.converters.items()):
201 if hasattr(builtins, candidate_type_str):
202 candidate_type = getattr(builtins, candidate_type_str)
203 else:
204 try:
205 candidate_type = doImportType(candidate_type_str)
206 except ImportError as e:
207 log.warning(
208 "Unable to import type %s associated with storage class %s (%s)",
209 candidate_type_str,
210 self.name,
211 e,
212 )
213 del self._converters[candidate_type_str]
214 continue
216 try:
217 converter = doImportType(converter_str)
218 except ImportError as e:
219 log.warning(
220 "Unable to import conversion function %s associated with storage class %s "
221 "required to convert type %s (%s)",
222 converter_str,
223 self.name,
224 candidate_type_str,
225 e,
226 )
227 del self._converters[candidate_type_str]
228 continue
229 if not callable(converter):
230 # doImportType is annotated to return a Type but in actual
231 # fact it can return Any except ModuleType because package
232 # variables can be accessed. This make mypy believe it
233 # is impossible for the return value to not be a callable
234 # so we must ignore the warning.
235 log.warning( # type: ignore
236 "Conversion function %s associated with storage class "
237 "%s to convert type %s is not a callable.",
238 converter_str,
239 self.name,
240 candidate_type_str,
241 )
242 del self._converters[candidate_type_str]
243 continue
244 self._converters_by_type[candidate_type] = converter
245 return self._converters_by_type
247 @property
248 def parameters(self) -> set[str]:
249 """Return `set` of names of supported parameters."""
250 return set(self._parameters)
252 @property
253 def pytype(self) -> type:
254 """Return Python type associated with this `StorageClass`."""
255 if self._pytype is not None:
256 return self._pytype
258 if hasattr(builtins, self._pytypeName):
259 pytype = getattr(builtins, self._pytypeName)
260 else:
261 pytype = doImportType(self._pytypeName)
262 self._pytype = pytype
263 return self._pytype
265 @property
266 def delegateClass(self) -> type | None:
267 """Class to use to delegate type-specific actions."""
268 if self._delegate is not None:
269 return self._delegate
270 if self._delegateClassName is None:
271 return None
272 delegate_class = doImportType(self._delegateClassName)
273 self._delegate = delegate_class
274 return self._delegate
276 def allComponents(self) -> Mapping[str, StorageClass]:
277 """Return all defined components.
279 This mapping includes all the derived and read/write components
280 for the corresponding storage class.
282 Returns
283 -------
284 comp : `dict` of [`str`, `StorageClass`]
285 The component name to storage class mapping.
286 """
287 return ChainMap(self._components, self._derivedComponents)
289 def delegate(self) -> StorageClassDelegate:
290 """Return an instance of a storage class delegate.
292 Returns
293 -------
294 delegate : `StorageClassDelegate`
295 Instance of the delegate associated with this `StorageClass`.
296 The delegate is constructed with this `StorageClass`.
298 Raises
299 ------
300 TypeError
301 This StorageClass has no associated delegate.
302 """
303 cls = self.delegateClass
304 if cls is None:
305 raise TypeError(f"No delegate class is associated with StorageClass {self.name}")
306 return cls(storageClass=self)
308 def isComposite(self) -> bool:
309 """Return Boolean indicating whether this is a composite or not.
311 Returns
312 -------
313 isComposite : `bool`
314 `True` if this `StorageClass` is a composite, `False`
315 otherwise.
316 """
317 if self.components:
318 return True
319 return False
321 def _lookupNames(self) -> tuple[LookupKey, ...]:
322 """Keys to use when looking up this DatasetRef in a configuration.
324 The names are returned in order of priority.
326 Returns
327 -------
328 names : `tuple` of `LookupKey`
329 Tuple of a `LookupKey` using the `StorageClass` name.
330 """
331 return (LookupKey(name=self.name),)
333 def knownParameters(self) -> set[str]:
334 """Return set of all parameters known to this `StorageClass`.
336 The set includes parameters understood by components of a composite.
338 Returns
339 -------
340 known : `set`
341 All parameter keys of this `StorageClass` and the component
342 storage classes.
343 """
344 known = set(self._parameters)
345 for sc in self.components.values():
346 known.update(sc.knownParameters())
347 return known
349 def validateParameters(self, parameters: Collection | None = None) -> None:
350 """Check that the parameters are known to this `StorageClass`.
352 Does not check the values.
354 Parameters
355 ----------
356 parameters : `~collections.abc.Collection`, optional
357 Collection containing the parameters. Can be `dict`-like or
358 `set`-like. The parameter values are not checked.
359 If no parameters are supplied, always returns without error.
361 Raises
362 ------
363 KeyError
364 Some parameters are not understood by this `StorageClass`.
365 """
366 # No parameters is always okay
367 if not parameters:
368 return
370 # Extract the important information into a set. Works for dict and
371 # list.
372 external = set(parameters)
374 diff = external - self.knownParameters()
375 if diff:
376 s = "s" if len(diff) > 1 else ""
377 unknown = "', '".join(diff)
378 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}")
380 def filterParameters(
381 self, parameters: Mapping[str, Any] | None, subset: Collection | None = None
382 ) -> Mapping[str, Any]:
383 """Filter out parameters that are not known to this `StorageClass`.
385 Parameters
386 ----------
387 parameters : `~collections.abc.Mapping`, optional
388 Candidate parameters. Can be `None` if no parameters have
389 been provided.
390 subset : `~collections.abc.Collection`, optional
391 Subset of supported parameters that the caller is interested
392 in using. The subset must be known to the `StorageClass`
393 if specified. If `None` the supplied parameters will all
394 be checked, else only the keys in this set will be checked.
396 Returns
397 -------
398 filtered : `~collections.abc.Mapping`
399 Valid parameters. Empty `dict` if none are suitable.
401 Raises
402 ------
403 ValueError
404 Raised if the provided subset is not a subset of the supported
405 parameters or if it is an empty set.
406 """
407 if not parameters:
408 return {}
410 known = self.knownParameters()
412 if subset is not None:
413 if not subset:
414 raise ValueError("Specified a parameter subset but it was empty")
415 subset = set(subset)
416 if not subset.issubset(known):
417 raise ValueError(f"Requested subset ({subset}) is not a subset of known parameters ({known})")
418 wanted = subset
419 else:
420 wanted = known
422 return {k: parameters[k] for k in wanted if k in parameters}
424 def validateInstance(self, instance: Any) -> bool:
425 """Check that the supplied Python object has the expected Python type.
427 Parameters
428 ----------
429 instance : `object`
430 Object to check.
432 Returns
433 -------
434 isOk : `bool`
435 True if the supplied instance object can be handled by this
436 `StorageClass`, False otherwise.
437 """
438 return isinstance(instance, self.pytype)
440 def is_type(self, other: type, compare_types: bool = False) -> bool:
441 """Return Boolean indicating whether the supplied type matches
442 the type in this `StorageClass`.
444 Parameters
445 ----------
446 other : `type`
447 The type to be checked.
448 compare_types : `bool`, optional
449 If `True` the python type will be used in the comparison
450 if the type names do not match. This may trigger an import
451 of code and so can be slower.
453 Returns
454 -------
455 match : `bool`
456 `True` if the types are equal.
458 Notes
459 -----
460 If this `StorageClass` has not yet imported the Python type the
461 check is done against the full type name, this prevents an attempt
462 to import the type when it will likely not match.
463 """
464 if self._pytype:
465 return self._pytype is other
467 other_name = get_full_type_name(other)
468 if self._pytypeName == other_name:
469 return True
471 if compare_types:
472 # Must protect against the import failing.
473 try:
474 return self.pytype is other
475 except Exception:
476 pass
478 return False
480 def can_convert(self, other: StorageClass) -> bool:
481 """Return `True` if this storage class can convert python types
482 in the other storage class.
484 Parameters
485 ----------
486 other : `StorageClass`
487 The storage class to check.
489 Returns
490 -------
491 can : `bool`
492 `True` if this storage class has a registered converter for
493 the python type associated with the other storage class. That
494 converter will convert the other python type to the one associated
495 with this storage class.
496 """
497 if other.name == self.name:
498 # Identical storage classes are compatible.
499 return True
501 # It may be that the storage class being compared is not
502 # available because the python type can't be imported. In that
503 # case conversion must be impossible.
504 try:
505 other_pytype = other.pytype
506 except Exception:
507 return False
509 # Or even this storage class itself can not have the type imported.
510 try:
511 self_pytype = self.pytype
512 except Exception:
513 return False
515 if issubclass(other_pytype, self_pytype):
516 # Storage classes have different names but the same python type.
517 return True
519 for candidate_type in self._get_converters_by_type():
520 if issubclass(other_pytype, candidate_type):
521 return True
522 return False
524 def coerce_type(self, incorrect: Any) -> Any:
525 """Coerce the supplied incorrect instance to the python type
526 associated with this `StorageClass`.
528 Parameters
529 ----------
530 incorrect : `object`
531 An object that might be the incorrect type.
533 Returns
534 -------
535 correct : `object`
536 An object that matches the python type of this `StorageClass`.
537 Can be the same object as given. If `None`, `None` will be
538 returned.
540 Raises
541 ------
542 TypeError
543 Raised if no conversion can be found.
544 """
545 if incorrect is None:
546 return None
548 # Possible this is the correct type already.
549 if self.validateInstance(incorrect):
550 return incorrect
552 # Check each registered converter.
553 for candidate_type, converter in self._get_converters_by_type().items():
554 if isinstance(incorrect, candidate_type):
555 try:
556 return converter(incorrect)
557 except Exception:
558 log.error(
559 "Converter %s failed to convert type %s",
560 get_full_type_name(converter),
561 get_full_type_name(incorrect),
562 )
563 raise
564 raise TypeError(
565 "Type does not match and no valid converter found to convert"
566 f" '{get_full_type_name(incorrect)}' to '{get_full_type_name(self.pytype)}'"
567 )
569 def __eq__(self, other: Any) -> bool:
570 """Equality checks name, pytype name, delegate name, and components."""
571 if not isinstance(other, StorageClass):
572 return NotImplemented
574 if self.name != other.name:
575 return False
577 # We must compare pytype and delegate by name since we do not want
578 # to trigger an import of external module code here
579 if self._delegateClassName != other._delegateClassName:
580 return False
581 if self._pytypeName != other._pytypeName:
582 return False
584 # Ensure we have the same component keys in each
585 if set(self.components.keys()) != set(other.components.keys()):
586 return False
588 # Same parameters
589 if self.parameters != other.parameters:
590 return False
592 # Ensure that all the components have the same type
593 return all(self.components[k] == other.components[k] for k in self.components)
595 def __hash__(self) -> int:
596 return hash(self.name)
598 def __repr__(self) -> str:
599 optionals: dict[str, Any] = {}
600 if self._pytypeName != "object":
601 optionals["pytype"] = self._pytypeName
602 if self._delegateClassName is not None:
603 optionals["delegate"] = self._delegateClassName
604 if self._parameters:
605 optionals["parameters"] = self._parameters
606 if self.components:
607 optionals["components"] = self.components
608 if self.converters:
609 optionals["converters"] = self.converters
611 # order is preserved in the dict
612 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items())
614 # Start with mandatory fields
615 r = f"{self.__class__.__name__}({self.name!r}"
616 if options:
617 r = r + ", " + options
618 r = r + ")"
619 return r
621 def __str__(self) -> str:
622 return self.name
625class StorageClassFactory(metaclass=Singleton):
626 """Factory for `StorageClass` instances.
628 This class is a singleton, with each instance sharing the pool of
629 StorageClasses. Since code can not know whether it is the first
630 time the instance has been created, the constructor takes no arguments.
631 To populate the factory with storage classes, a call to
632 `~StorageClassFactory.addFromConfig()` should be made.
634 Parameters
635 ----------
636 config : `StorageClassConfig` or `str`, optional
637 Load configuration. In a ButlerConfig` the relevant configuration
638 is located in the ``storageClasses`` section.
639 """
641 def __init__(self, config: StorageClassConfig | str | None = None):
642 self._storageClasses: dict[str, StorageClass] = {}
643 self._configs: list[StorageClassConfig] = []
645 # Always seed with the default config
646 self.addFromConfig(StorageClassConfig())
648 if config is not None: 648 ↛ 649line 648 didn't jump to line 649, because the condition on line 648 was never true
649 self.addFromConfig(config)
651 def __str__(self) -> str:
652 """Return summary of factory.
654 Returns
655 -------
656 summary : `str`
657 Summary of the factory status.
658 """
659 sep = "\n"
660 return f"""Number of registered StorageClasses: {len(self._storageClasses)}
662StorageClasses
663--------------
664{sep.join(f"{s}: {self._storageClasses[s]!r}" for s in sorted(self._storageClasses))}
665"""
667 def __contains__(self, storageClassOrName: StorageClass | str) -> bool:
668 """Indicate whether the storage class exists in the factory.
670 Parameters
671 ----------
672 storageClassOrName : `str` or `StorageClass`
673 If `str` is given existence of the named StorageClass
674 in the factory is checked. If `StorageClass` is given
675 existence and equality are checked.
677 Returns
678 -------
679 in : `bool`
680 True if the supplied string is present, or if the supplied
681 `StorageClass` is present and identical.
683 Notes
684 -----
685 The two different checks (one for "key" and one for "value") based on
686 the type of the given argument mean that it is possible for
687 StorageClass.name to be in the factory but StorageClass to not be
688 in the factory.
689 """
690 if isinstance(storageClassOrName, str): 690 ↛ 692line 690 didn't jump to line 692, because the condition on line 690 was never false
691 return storageClassOrName in self._storageClasses
692 elif isinstance(storageClassOrName, StorageClass) and storageClassOrName.name in self._storageClasses:
693 return storageClassOrName == self._storageClasses[storageClassOrName.name]
694 return False
696 def __len__(self) -> int:
697 return len(self._storageClasses)
699 def __iter__(self) -> Iterator[str]:
700 return iter(self._storageClasses)
702 def values(self) -> ValuesView[StorageClass]:
703 return self._storageClasses.values()
705 def keys(self) -> KeysView[str]:
706 return self._storageClasses.keys()
708 def items(self) -> ItemsView[str, StorageClass]:
709 return self._storageClasses.items()
711 def addFromConfig(self, config: StorageClassConfig | Config | str) -> None:
712 """Add more `StorageClass` definitions from a config file.
714 Parameters
715 ----------
716 config : `StorageClassConfig`, `Config` or `str`
717 Storage class configuration. Can contain a ``storageClasses``
718 key if part of a global configuration.
719 """
720 sconfig = StorageClassConfig(config)
721 self._configs.append(sconfig)
723 # Since we can not assume that we will get definitions of
724 # components or parents before their classes are defined
725 # we have a helper function that we can call recursively
726 # to extract definitions from the configuration.
727 def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> None:
728 # Maybe we've already processed this through recursion
729 if name not in _sconfig:
730 return
731 info = _sconfig.pop(name)
733 # Always create the storage class so we can ensure that
734 # we are not trying to overwrite with a different definition
735 components = None
737 # Extract scalar items from dict that are needed for
738 # StorageClass Constructor
739 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info}
741 if "converters" in info:
742 storageClassKwargs["converters"] = info["converters"].toDict()
744 for compName in ("components", "derivedComponents"):
745 if compName not in info:
746 continue
747 components = {}
748 for cname, ctype in info[compName].items():
749 if ctype not in self:
750 processStorageClass(ctype, sconfig, msg)
751 components[cname] = self.getStorageClass(ctype)
753 # Fill in other items
754 storageClassKwargs[compName] = components
756 # Create the new storage class and register it
757 baseClass = None
758 if "inheritsFrom" in info:
759 baseName = info["inheritsFrom"]
761 # The inheritsFrom feature requires that the storage class
762 # being inherited from is itself a subclass of StorageClass
763 # that was created with makeNewStorageClass. If it was made
764 # and registered with a simple StorageClass constructor it
765 # cannot be used here and we try to recreate it.
766 if baseName in self: 766 ↛ 778line 766 didn't jump to line 778, because the condition on line 766 was never false
767 baseClass = type(self.getStorageClass(baseName))
768 if baseClass is StorageClass: 768 ↛ 769line 768 didn't jump to line 769, because the condition on line 768 was never true
769 log.warning(
770 "Storage class %s is requested to inherit from %s but that storage class "
771 "has not been defined to be a subclass of StorageClass and so can not "
772 "be used. Attempting to recreate parent class from current configuration.",
773 name,
774 baseName,
775 )
776 processStorageClass(baseName, sconfig, msg)
777 else:
778 processStorageClass(baseName, sconfig, msg)
779 baseClass = type(self.getStorageClass(baseName))
780 if baseClass is StorageClass: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true
781 raise TypeError(
782 f"Configuration for storage class {name} requests to inherit from "
783 f" storage class {baseName} but that class is not defined correctly."
784 )
786 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs)
787 newStorageClass = newStorageClassType()
788 self.registerStorageClass(newStorageClass, msg=msg)
790 # In case there is a problem, construct a context message for any
791 # error reporting.
792 files = [str(f) for f in itertools.chain([sconfig.configFile], sconfig.filesRead) if f]
793 context = f"when adding definitions from {', '.join(files)}" if files else ""
794 log.debug("Adding definitions from config %s", ", ".join(files))
796 for name in list(sconfig.keys()):
797 processStorageClass(name, sconfig, context)
799 @staticmethod
800 def makeNewStorageClass(
801 name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any
802 ) -> type[StorageClass]:
803 """Create a new Python class as a subclass of `StorageClass`.
805 Parameters
806 ----------
807 name : `str`
808 Name to use for this class.
809 baseClass : `type`, optional
810 Base class for this `StorageClass`. Must be either `StorageClass`
811 or a subclass of `StorageClass`. If `None`, `StorageClass` will
812 be used.
814 Returns
815 -------
816 newtype : `type` subclass of `StorageClass`
817 Newly created Python type.
818 """
819 if baseClass is None:
820 baseClass = StorageClass
821 if not issubclass(baseClass, StorageClass): 821 ↛ 822line 821 didn't jump to line 822, because the condition on line 821 was never true
822 raise ValueError(f"Base class must be a StorageClass not {baseClass}")
824 # convert the arguments to use different internal names
825 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None}
826 clsargs["_cls_name"] = name
828 # Some container items need to merge with the base class values
829 # so that a child can inherit but override one bit.
830 # lists (which you get from configs) are treated as sets for this to
831 # work consistently.
832 for k in ("components", "parameters", "derivedComponents", "converters"):
833 classKey = f"_cls_{k}"
834 if classKey in clsargs:
835 baseValue = getattr(baseClass, classKey, None)
836 if baseValue is not None:
837 currentValue = clsargs[classKey]
838 if isinstance(currentValue, dict): 838 ↛ 841line 838 didn't jump to line 841, because the condition on line 838 was never false
839 newValue = baseValue.copy()
840 else:
841 newValue = set(baseValue)
842 newValue.update(currentValue)
843 clsargs[classKey] = newValue
845 # If we have parameters they should be a frozen set so that the
846 # parameters in the class can not be modified.
847 pk = "_cls_parameters"
848 if pk in clsargs:
849 clsargs[pk] = frozenset(clsargs[pk])
851 return type(f"StorageClass{name}", (baseClass,), clsargs)
853 def getStorageClass(self, storageClassName: str) -> StorageClass:
854 """Get a StorageClass instance associated with the supplied name.
856 Parameters
857 ----------
858 storageClassName : `str`
859 Name of the storage class to retrieve.
861 Returns
862 -------
863 instance : `StorageClass`
864 Instance of the correct `StorageClass`.
866 Raises
867 ------
868 KeyError
869 The requested storage class name is not registered.
870 """
871 return self._storageClasses[storageClassName]
873 def findStorageClass(self, pytype: type, compare_types: bool = False) -> StorageClass:
874 """Find the storage class associated with this python type.
876 Parameters
877 ----------
878 pytype : `type`
879 The Python type to be matched.
880 compare_types : `bool`, optional
881 If `False`, the type will be checked against name of the python
882 type. This comparison is always done first. If `True` and the
883 string comparison failed, each candidate storage class will be
884 forced to have its type imported. This can be significantly slower.
886 Returns
887 -------
888 storageClass : `StorageClass`
889 The matching storage class.
891 Raises
892 ------
893 KeyError
894 Raised if no match could be found.
896 Notes
897 -----
898 It is possible for a python type to be associated with multiple
899 storage classes. This method will currently return the first that
900 matches.
901 """
902 result = self._find_storage_class(pytype, False)
903 if result:
904 return result
906 if compare_types:
907 # The fast comparison failed and we were asked to try the
908 # variant that might involve code imports.
909 result = self._find_storage_class(pytype, True)
910 if result:
911 return result
913 raise KeyError(f"Unable to find a StorageClass associated with type {get_full_type_name(pytype)!r}")
915 def _find_storage_class(self, pytype: type, compare_types: bool) -> StorageClass | None:
916 """Iterate through all storage classes to find a match.
918 Parameters
919 ----------
920 pytype : `type`
921 The Python type to be matched.
922 compare_types : `bool`, optional
923 Whether to use type name matching or explicit type matching.
924 The latter can be slower.
926 Returns
927 -------
928 storageClass : `StorageClass` or `None`
929 The matching storage class, or `None` if no match was found.
931 Notes
932 -----
933 Helper method for ``findStorageClass``.
934 """
935 for storageClass in self.values():
936 if storageClass.is_type(pytype, compare_types=compare_types):
937 return storageClass
938 return None
940 def registerStorageClass(self, storageClass: StorageClass, msg: str | None = None) -> None:
941 """Store the `StorageClass` in the factory.
943 Will be indexed by `StorageClass.name` and will return instances
944 of the supplied `StorageClass`.
946 Parameters
947 ----------
948 storageClass : `StorageClass`
949 Type of the Python `StorageClass` to register.
950 msg : `str`, optional
951 Additional message string to be included in any error message.
953 Raises
954 ------
955 ValueError
956 If a storage class has already been registered with
957 that storage class name and the previous definition differs.
958 """
959 if storageClass.name in self._storageClasses: 959 ↛ 960line 959 didn't jump to line 960, because the condition on line 959 was never true
960 existing = self.getStorageClass(storageClass.name)
961 if existing != storageClass:
962 errmsg = f" {msg}" if msg else ""
963 raise ValueError(
964 f"New definition for StorageClass {storageClass.name} ({storageClass!r}) "
965 f"differs from current definition ({existing!r}){errmsg}"
966 )
967 if type(existing) is StorageClass and type(storageClass) is not StorageClass:
968 # Replace generic with specialist subclass equivalent.
969 self._storageClasses[storageClass.name] = storageClass
970 else:
971 self._storageClasses[storageClass.name] = storageClass
973 def _unregisterStorageClass(self, storageClassName: str) -> None:
974 """Remove the named StorageClass from the factory.
976 Parameters
977 ----------
978 storageClassName : `str`
979 Name of storage class to remove.
981 Raises
982 ------
983 KeyError
984 The named storage class is not registered.
986 Notes
987 -----
988 This method is intended to simplify testing of StorageClassFactory
989 functionality and it is not expected to be required for normal usage.
990 """
991 del self._storageClasses[storageClassName]
993 def reset(self) -> None:
994 """Remove all storage class entries from factory and reset to
995 initial state.
997 This is useful for test code where a known start state is useful.
998 """
999 self._storageClasses.clear()
1000 # Seed with the default config.
1001 self.addFromConfig(StorageClassConfig())