Coverage for python/lsst/daf/butler/core/storageClass.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Support for Storage Classes."""
26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig")
28import builtins
29import copy
30import logging
32from typing import (
33 Any,
34 Collection,
35 Dict,
36 List,
37 Mapping,
38 Optional,
39 Set,
40 Sequence,
41 Tuple,
42 Type,
43 Union,
44)
46from lsst.utils import doImport
47from .utils import Singleton, getFullTypeName
48from .storageClassDelegate import StorageClassDelegate
49from .config import ConfigSubset, Config
50from .configSupport import LookupKey
52log = logging.getLogger(__name__)
55class StorageClassConfig(ConfigSubset):
56 component = "storageClasses"
57 defaultConfigFile = "storageClasses.yaml"
60class StorageClass:
61 """Class describing how a label maps to a particular Python type.
63 Parameters
64 ----------
65 name : `str`
66 Name to use for this class.
67 pytype : `type` or `str`
68 Python type (or name of type) to associate with the `StorageClass`
69 components : `dict`, optional
70 `dict` mapping name of a component to another `StorageClass`.
71 derivedComponents : `dict`, optional
72 `dict` mapping name of a derived component to another `StorageClass`.
73 parameters : `~collections.abc.Sequence` or `~collections.abc.Set`
74 Parameters understood by this `StorageClass` that can control
75 reading of data from datastores.
76 delegate : `str`, optional
77 Fully qualified name of class supporting assembly and disassembly
78 of a `pytype` instance.
79 """
80 _cls_name: str = "BaseStorageClass"
81 _cls_components: Optional[Dict[str, StorageClass]] = None
82 _cls_derivedComponents: Optional[Dict[str, StorageClass]] = None
83 _cls_parameters: Optional[Union[Set[str], Sequence[str]]] = None
84 _cls_delegate: Optional[str] = None
85 _cls_pytype: Optional[Union[Type, str]] = None
86 defaultDelegate: Type = StorageClassDelegate
87 defaultDelegateName: str = getFullTypeName(defaultDelegate)
89 def __init__(self, name: Optional[str] = None,
90 pytype: Optional[Union[Type, str]] = None,
91 components: Optional[Dict[str, StorageClass]] = None,
92 derivedComponents: Optional[Dict[str, StorageClass]] = None,
93 parameters: Optional[Union[Sequence, Set]] = None,
94 delegate: Optional[str] = None):
95 if name is None: 95 ↛ 96line 95 didn't jump to line 96, because the condition on line 95 was never true
96 name = self._cls_name
97 if pytype is None: 97 ↛ 99line 97 didn't jump to line 99, because the condition on line 97 was never false
98 pytype = self._cls_pytype
99 if components is None: 99 ↛ 101line 99 didn't jump to line 101, because the condition on line 99 was never false
100 components = self._cls_components
101 if derivedComponents is None: 101 ↛ 103line 101 didn't jump to line 103, because the condition on line 101 was never false
102 derivedComponents = self._cls_derivedComponents
103 if parameters is None: 103 ↛ 105line 103 didn't jump to line 105, because the condition on line 103 was never false
104 parameters = self._cls_parameters
105 if delegate is None: 105 ↛ 107line 105 didn't jump to line 107, because the condition on line 105 was never false
106 delegate = self._cls_delegate
107 self.name = name
109 if pytype is None: 109 ↛ 112line 109 didn't jump to line 112, because the condition on line 109 was never false
110 pytype = object
112 self._pytype: Optional[Type]
113 if not isinstance(pytype, str): 113 ↛ 119line 113 didn't jump to line 119, because the condition on line 113 was never false
114 # Already have a type so store it and get the name
115 self._pytypeName = getFullTypeName(pytype)
116 self._pytype = pytype
117 else:
118 # Store the type name and defer loading of type
119 self._pytypeName = pytype
120 self._pytype = None
122 self._components = components if components is not None else {}
123 self._derivedComponents = derivedComponents if derivedComponents is not None else {}
124 self._parameters = frozenset(parameters) if parameters is not None else frozenset()
125 # if the delegate is not None also set it and clear the default
126 # delegate
127 self._delegate: Optional[Type]
128 self._delegateClassName: Optional[str]
129 if delegate is not None: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true
130 self._delegateClassName = delegate
131 self._delegate = None
132 elif components is not None: 132 ↛ 135line 132 didn't jump to line 135, because the condition on line 132 was never true
133 # We set a default delegate for composites so that a class is
134 # guaranteed to support something if it is a composite.
135 log.debug("Setting default delegate for %s", self.name)
136 self._delegate = self.defaultDelegate
137 self._delegateClassName = self.defaultDelegateName
138 else:
139 self._delegate = None
140 self._delegateClassName = None
142 @property
143 def components(self) -> Dict[str, StorageClass]:
144 """Component names mapped to associated `StorageClass`
145 """
146 return self._components
148 @property
149 def derivedComponents(self) -> Dict[str, StorageClass]:
150 """Derived component names mapped to associated `StorageClass`
151 """
152 return self._derivedComponents
154 @property
155 def parameters(self) -> Set[str]:
156 """`set` of names of parameters supported by this `StorageClass`
157 """
158 return set(self._parameters)
160 @property
161 def pytype(self) -> Type:
162 """Python type associated with this `StorageClass`."""
163 if self._pytype is not None:
164 return self._pytype
166 if hasattr(builtins, self._pytypeName):
167 pytype = getattr(builtins, self._pytypeName)
168 else:
169 pytype = doImport(self._pytypeName)
170 self._pytype = pytype
171 return self._pytype
173 @property
174 def delegateClass(self) -> Optional[Type]:
175 """Class to use to delegate type-specific actions."""
176 if self._delegate is not None:
177 return self._delegate
178 if self._delegateClassName is None:
179 return None
180 self._delegate = doImport(self._delegateClassName)
181 return self._delegate
183 def allComponents(self) -> Mapping[str, StorageClass]:
184 """Return a mapping of all the derived and read/write components
185 to the corresponding storage class.
187 Returns
188 -------
189 comp : `dict` of [`str`, `StorageClass`]
190 The component name to storage class mapping.
191 """
192 components = copy.copy(self.components)
193 components.update(self.derivedComponents)
194 return components
196 def delegate(self) -> StorageClassDelegate:
197 """Return an instance of a storage class delegate.
199 Returns
200 -------
201 delegate : `StorageClassDelegate`
202 Instance of the delegate associated with this `StorageClass`.
203 The delegate is constructed with this `StorageClass`.
205 Raises
206 ------
207 TypeError
208 This StorageClass has no associated delegate.
209 """
210 cls = self.delegateClass
211 if cls is None:
212 raise TypeError(f"No delegate class is associated with StorageClass {self.name}")
213 return cls(storageClass=self)
215 def isComposite(self) -> bool:
216 """Boolean indicating whether this `StorageClass` is a composite
217 or not.
219 Returns
220 -------
221 isComposite : `bool`
222 `True` if this `StorageClass` is a composite, `False`
223 otherwise.
224 """
225 if self.components:
226 return True
227 return False
229 def _lookupNames(self) -> Tuple[LookupKey, ...]:
230 """Keys to use when looking up this DatasetRef in a configuration.
232 The names are returned in order of priority.
234 Returns
235 -------
236 names : `tuple` of `LookupKey`
237 Tuple of a `LookupKey` using the `StorageClass` name.
238 """
239 return (LookupKey(name=self.name), )
241 def knownParameters(self) -> Set[str]:
242 """Return set of all parameters known to this `StorageClass`
244 The set includes parameters understood by components of a composite.
246 Returns
247 -------
248 known : `set`
249 All parameter keys of this `StorageClass` and the component
250 storage classes.
251 """
252 known = set(self._parameters)
253 for sc in self.components.values():
254 known.update(sc.knownParameters())
255 return known
257 def validateParameters(self, parameters: Collection = None) -> None:
258 """Check that the parameters are known to this `StorageClass`
260 Does not check the values.
262 Parameters
263 ----------
264 parameters : `~collections.abc.Collection`, optional
265 Collection containing the parameters. Can be `dict`-like or
266 `set`-like. The parameter values are not checked.
267 If no parameters are supplied, always returns without error.
269 Raises
270 ------
271 KeyError
272 Some parameters are not understood by this `StorageClass`.
273 """
274 # No parameters is always okay
275 if not parameters:
276 return
278 # Extract the important information into a set. Works for dict and
279 # list.
280 external = set(parameters)
282 diff = external - self.knownParameters()
283 if diff:
284 s = "s" if len(diff) > 1 else ""
285 unknown = '\', \''.join(diff)
286 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}")
288 def filterParameters(self, parameters: Dict[str, Any],
289 subset: Collection = None) -> Dict[str, Any]:
290 """Filter out parameters that are not known to this StorageClass
292 Parameters
293 ----------
294 parameters : `dict`, optional
295 Candidate parameters. Can be `None` if no parameters have
296 been provided.
297 subset : `~collections.abc.Collection`, optional
298 Subset of supported parameters that the caller is interested
299 in using. The subset must be known to the `StorageClass`
300 if specified. If `None` the supplied parameters will all
301 be checked, else only the keys in this set will be checked.
303 Returns
304 -------
305 filtered : `dict`
306 Valid parameters. Empty `dict` if none are suitable.
308 Raises
309 ------
310 ValueError
311 Raised if the provided subset is not a subset of the supported
312 parameters or if it is an empty set.
313 """
314 if not parameters:
315 return {}
317 known = self.knownParameters()
319 if subset is not None:
320 if not subset:
321 raise ValueError("Specified a parameter subset but it was empty")
322 subset = set(subset)
323 if not subset.issubset(known):
324 raise ValueError(f"Requested subset ({subset}) is not a subset of"
325 f" known parameters ({known})")
326 wanted = subset
327 else:
328 wanted = known
330 return {k: parameters[k] for k in wanted if k in parameters}
332 def validateInstance(self, instance: Any) -> bool:
333 """Check that the supplied Python object has the expected Python type
335 Parameters
336 ----------
337 instance : `object`
338 Object to check.
340 Returns
341 -------
342 isOk : `bool`
343 True if the supplied instance object can be handled by this
344 `StorageClass`, False otherwise.
345 """
346 return isinstance(instance, self.pytype)
348 def __eq__(self, other: Any) -> bool:
349 """Equality checks name, pytype name, delegate name, and components"""
351 if not isinstance(other, StorageClass):
352 return False
354 if self.name != other.name:
355 return False
357 # We must compare pytype and delegate by name since we do not want
358 # to trigger an import of external module code here
359 if self._delegateClassName != other._delegateClassName:
360 return False
361 if self._pytypeName != other._pytypeName:
362 return False
364 # Ensure we have the same component keys in each
365 if set(self.components.keys()) != set(other.components.keys()):
366 return False
368 # Same parameters
369 if self.parameters != other.parameters:
370 return False
372 # Ensure that all the components have the same type
373 for k in self.components:
374 if self.components[k] != other.components[k]:
375 return False
377 # If we got to this point everything checks out
378 return True
380 def __hash__(self) -> int:
381 return hash(self.name)
383 def __repr__(self) -> str:
384 optionals: Dict[str, Any] = {}
385 if self._pytypeName != "object":
386 optionals["pytype"] = self._pytypeName
387 if self._delegateClassName is not None:
388 optionals["delegate"] = self._delegateClassName
389 if self._parameters:
390 optionals["parameters"] = self._parameters
391 if self.components:
392 optionals["components"] = self.components
394 # order is preserved in the dict
395 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items())
397 # Start with mandatory fields
398 r = f"{self.__class__.__name__}({self.name!r}"
399 if options:
400 r = r + ", " + options
401 r = r + ")"
402 return r
404 def __str__(self) -> str:
405 return self.name
408class StorageClassFactory(metaclass=Singleton):
409 """Factory for `StorageClass` instances.
411 This class is a singleton, with each instance sharing the pool of
412 StorageClasses. Since code can not know whether it is the first
413 time the instance has been created, the constructor takes no arguments.
414 To populate the factory with storage classes, a call to
415 `~StorageClassFactory.addFromConfig()` should be made.
417 Parameters
418 ----------
419 config : `StorageClassConfig` or `str`, optional
420 Load configuration. In a ButlerConfig` the relevant configuration
421 is located in the ``storageClasses`` section.
422 """
424 def __init__(self, config: Optional[Union[StorageClassConfig, str]] = None):
425 self._storageClasses: Dict[str, StorageClass] = {}
426 self._configs: List[StorageClassConfig] = []
428 # Always seed with the default config
429 self.addFromConfig(StorageClassConfig())
431 if config is not None:
432 self.addFromConfig(config)
434 def __str__(self) -> str:
435 """Return summary of factory.
437 Returns
438 -------
439 summary : `str`
440 Summary of the factory status.
441 """
442 sep = "\n"
443 return f"""Number of registered StorageClasses: {len(self._storageClasses)}
445StorageClasses
446--------------
447{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)}
448"""
450 def __contains__(self, storageClassOrName: Union[StorageClass, str]) -> bool:
451 """Indicates whether the storage class exists in the factory.
453 Parameters
454 ----------
455 storageClassOrName : `str` or `StorageClass`
456 If `str` is given existence of the named StorageClass
457 in the factory is checked. If `StorageClass` is given
458 existence and equality are checked.
460 Returns
461 -------
462 in : `bool`
463 True if the supplied string is present, or if the supplied
464 `StorageClass` is present and identical.
466 Notes
467 -----
468 The two different checks (one for "key" and one for "value") based on
469 the type of the given argument mean that it is possible for
470 StorageClass.name to be in the factory but StorageClass to not be
471 in the factory.
472 """
473 if isinstance(storageClassOrName, str):
474 return storageClassOrName in self._storageClasses
475 elif isinstance(storageClassOrName, StorageClass):
476 if storageClassOrName.name in self._storageClasses:
477 return storageClassOrName == self._storageClasses[storageClassOrName.name]
478 return False
480 def addFromConfig(self, config: Union[StorageClassConfig, Config, str]) -> None:
481 """Add more `StorageClass` definitions from a config file.
483 Parameters
484 ----------
485 config : `StorageClassConfig`, `Config` or `str`
486 Storage class configuration. Can contain a ``storageClasses``
487 key if part of a global configuration.
488 """
489 sconfig = StorageClassConfig(config)
490 self._configs.append(sconfig)
492 # Since we can not assume that we will get definitions of
493 # components or parents before their classes are defined
494 # we have a helper function that we can call recursively
495 # to extract definitions from the configuration.
496 def processStorageClass(name: str, sconfig: StorageClassConfig) -> None:
497 # Maybe we've already processed this through recursion
498 if name not in sconfig:
499 return
500 info = sconfig.pop(name)
502 # Always create the storage class so we can ensure that
503 # we are not trying to overwrite with a different definition
504 components = None
506 # Extract scalar items from dict that are needed for
507 # StorageClass Constructor
508 storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info}
510 for compName in ("components", "derivedComponents"):
511 if compName not in info:
512 continue
513 components = {}
514 for cname, ctype in info[compName].items():
515 if ctype not in self:
516 processStorageClass(ctype, sconfig)
517 components[cname] = self.getStorageClass(ctype)
519 # Fill in other items
520 storageClassKwargs[compName] = components
522 # Create the new storage class and register it
523 baseClass = None
524 if "inheritsFrom" in info:
525 baseName = info["inheritsFrom"]
526 if baseName not in self:
527 processStorageClass(baseName, sconfig)
528 baseClass = type(self.getStorageClass(baseName))
530 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs)
531 newStorageClass = newStorageClassType()
532 self.registerStorageClass(newStorageClass)
534 for name in list(sconfig.keys()):
535 processStorageClass(name, sconfig)
537 @staticmethod
538 def makeNewStorageClass(name: str,
539 baseClass: Optional[Type[StorageClass]] = StorageClass,
540 **kwargs: Any) -> Type[StorageClass]:
541 """Create a new Python class as a subclass of `StorageClass`.
543 Parameters
544 ----------
545 name : `str`
546 Name to use for this class.
547 baseClass : `type`, optional
548 Base class for this `StorageClass`. Must be either `StorageClass`
549 or a subclass of `StorageClass`. If `None`, `StorageClass` will
550 be used.
552 Returns
553 -------
554 newtype : `type` subclass of `StorageClass`
555 Newly created Python type.
556 """
558 if baseClass is None:
559 baseClass = StorageClass
560 if not issubclass(baseClass, StorageClass):
561 raise ValueError(f"Base class must be a StorageClass not {baseClass}")
563 # convert the arguments to use different internal names
564 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None}
565 clsargs["_cls_name"] = name
567 # Some container items need to merge with the base class values
568 # so that a child can inherit but override one bit.
569 # lists (which you get from configs) are treated as sets for this to
570 # work consistently.
571 for k in ("components", "parameters", "derivedComponents"):
572 classKey = f"_cls_{k}"
573 if classKey in clsargs:
574 baseValue = getattr(baseClass, classKey, None)
575 if baseValue is not None:
576 currentValue = clsargs[classKey]
577 if isinstance(currentValue, dict):
578 newValue = baseValue.copy()
579 else:
580 newValue = set(baseValue)
581 newValue.update(currentValue)
582 clsargs[classKey] = newValue
584 # If we have parameters they should be a frozen set so that the
585 # parameters in the class can not be modified.
586 pk = "_cls_parameters"
587 if pk in clsargs:
588 clsargs[pk] = frozenset(clsargs[pk])
590 return type(f"StorageClass{name}", (baseClass,), clsargs)
592 def getStorageClass(self, storageClassName: str) -> StorageClass:
593 """Get a StorageClass instance associated with the supplied name.
595 Parameters
596 ----------
597 storageClassName : `str`
598 Name of the storage class to retrieve.
600 Returns
601 -------
602 instance : `StorageClass`
603 Instance of the correct `StorageClass`.
605 Raises
606 ------
607 KeyError
608 The requested storage class name is not registered.
609 """
610 return self._storageClasses[storageClassName]
612 def registerStorageClass(self, storageClass: StorageClass) -> None:
613 """Store the `StorageClass` in the factory.
615 Will be indexed by `StorageClass.name` and will return instances
616 of the supplied `StorageClass`.
618 Parameters
619 ----------
620 storageClass : `StorageClass`
621 Type of the Python `StorageClass` to register.
623 Raises
624 ------
625 ValueError
626 If a storage class has already been registered with
627 storageClassName and the previous definition differs.
628 """
629 if storageClass.name in self._storageClasses:
630 existing = self.getStorageClass(storageClass.name)
631 if existing != storageClass:
632 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) "
633 f"differs from current definition ({existing})")
634 else:
635 self._storageClasses[storageClass.name] = storageClass
637 def _unregisterStorageClass(self, storageClassName: str) -> None:
638 """Remove the named StorageClass from the factory.
640 Parameters
641 ----------
642 storageClassName : `str`
643 Name of storage class to remove.
645 Raises
646 ------
647 KeyError
648 The named storage class is not registered.
650 Notes
651 -----
652 This method is intended to simplify testing of StorageClassFactory
653 functionality and it is not expected to be required for normal usage.
654 """
655 del self._storageClasses[storageClassName]