Coverage for python/lsst/daf/butler/core/storageClass.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Support for Storage Classes."""
26__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig")
28import builtins
29import logging
31from typing import (
32 Any,
33 Collection,
34 Dict,
35 List,
36 Optional,
37 Set,
38 Sequence,
39 Tuple,
40 Type,
41 Union,
42)
44from lsst.utils import doImport
45from .utils import Singleton, getFullTypeName
46from .assembler import CompositeAssembler
47from .config import ConfigSubset, Config
48from .configSupport import LookupKey
50log = logging.getLogger(__name__)
53class StorageClassConfig(ConfigSubset):
54 component = "storageClasses"
55 defaultConfigFile = "storageClasses.yaml"
58class StorageClass:
59 """Class describing how a label maps to a particular Python type.
61 Parameters
62 ----------
63 name : `str`
64 Name to use for this class.
65 pytype : `type` or `str`
66 Python type (or name of type) to associate with the `StorageClass`
67 components : `dict`, optional
68 `dict` mapping name of a component to another `StorageClass`.
69 parameters : `~collections.abc.Sequence` or `~collections.abc.Set`
70 Parameters understood by this `StorageClass`.
71 assembler : `str`, optional
72 Fully qualified name of class supporting assembly and disassembly
73 of a `pytype` instance.
74 """
75 _cls_name: str = "BaseStorageClass"
76 _cls_components: Optional[Dict[str, StorageClass]] = None
77 _cls_parameters: Optional[Union[Set[str], Sequence[str]]] = None
78 _cls_assembler: Optional[str] = None
79 _cls_pytype: Optional[Union[Type, str]] = None
80 defaultAssembler: Type = CompositeAssembler
81 defaultAssemblerName: str = getFullTypeName(defaultAssembler)
83 def __init__(self, name: Optional[str] = None,
84 pytype: Optional[Union[Type, str]] = None,
85 components: Optional[Dict[str, StorageClass]] = None,
86 parameters: Optional[Union[Sequence, Set]] = None,
87 assembler: Optional[str] = None):
88 if name is None:
89 name = self._cls_name
90 if pytype is None:
91 pytype = self._cls_pytype
92 if components is None:
93 components = self._cls_components
94 if parameters is None:
95 parameters = self._cls_parameters
96 if assembler is None:
97 assembler = self._cls_assembler
98 self.name = name
100 if pytype is None:
101 pytype = object
103 self._pytype: Optional[Type]
104 if not isinstance(pytype, str):
105 # Already have a type so store it and get the name
106 self._pytypeName = getFullTypeName(pytype)
107 self._pytype = pytype
108 else:
109 # Store the type name and defer loading of type
110 self._pytypeName = pytype
111 self._pytype = None
113 self._components = components if components is not None else {}
114 self._parameters = frozenset(parameters) if parameters is not None else frozenset()
115 # if the assembler is not None also set it and clear the default
116 # assembler
117 self._assembler: Optional[Type]
118 self._assemblerClassName: Optional[str]
119 if assembler is not None:
120 self._assemblerClassName = assembler
121 self._assembler = None
122 elif components is not None:
123 # We set a default assembler for composites so that a class is
124 # guaranteed to support something if it is a composite.
125 log.debug("Setting default assembler for %s", self.name)
126 self._assembler = self.defaultAssembler
127 self._assemblerClassName = self.defaultAssemblerName
128 else:
129 self._assembler = None
130 self._assemblerClassName = None
132 @property
133 def components(self) -> Dict[str, StorageClass]:
134 """Component names mapped to associated `StorageClass`
135 """
136 return self._components
138 @property
139 def parameters(self) -> Set[str]:
140 """`set` of names of parameters supported by this `StorageClass`
141 """
142 return set(self._parameters)
144 @property
145 def pytype(self) -> Type:
146 """Python type associated with this `StorageClass`."""
147 if self._pytype is not None:
148 return self._pytype
150 if hasattr(builtins, self._pytypeName):
151 pytype = getattr(builtins, self._pytypeName)
152 else:
153 pytype = doImport(self._pytypeName)
154 self._pytype = pytype
155 return self._pytype
157 @property
158 def assemblerClass(self) -> Optional[Type]:
159 """Class to use to (dis)assemble an object from components."""
160 if self._assembler is not None:
161 return self._assembler
162 if self._assemblerClassName is None:
163 return None
164 self._assembler = doImport(self._assemblerClassName)
165 return self._assembler
167 def assembler(self) -> CompositeAssembler:
168 """Return an instance of an assembler.
170 Returns
171 -------
172 assembler : `CompositeAssembler`
173 Instance of the assembler associated with this `StorageClass`.
174 Assembler is constructed with this `StorageClass`.
176 Raises
177 ------
178 TypeError
179 This StorageClass has no associated assembler.
180 """
181 cls = self.assemblerClass
182 if cls is None:
183 raise TypeError(f"No assembler class is associated with StorageClass {self.name}")
184 return cls(storageClass=self)
186 def isComposite(self) -> bool:
187 """Boolean indicating whether this `StorageClass` is a composite
188 or not.
190 Returns
191 -------
192 isComposite : `bool`
193 `True` if this `StorageClass` is a composite, `False`
194 otherwise.
195 """
196 if self.components:
197 return True
198 return False
200 def _lookupNames(self) -> Tuple[LookupKey, ...]:
201 """Keys to use when looking up this DatasetRef in a configuration.
203 The names are returned in order of priority.
205 Returns
206 -------
207 names : `tuple` of `LookupKey`
208 Tuple of a `LookupKey` using the `StorageClass` name.
209 """
210 return (LookupKey(name=self.name), )
212 def knownParameters(self) -> Set[str]:
213 """Return set of all parameters known to this `StorageClass`
215 The set includes parameters understood by components of a composite.
217 Returns
218 -------
219 known : `set`
220 All parameter keys of this `StorageClass` and the component
221 storage classes.
222 """
223 known = set(self._parameters)
224 for sc in self.components.values():
225 known.update(sc.knownParameters())
226 return known
228 def validateParameters(self, parameters: Collection = None) -> None:
229 """Check that the parameters are known to this `StorageClass`
231 Does not check the values.
233 Parameters
234 ----------
235 parameters : `~collections.abc.Collection`, optional
236 Collection containing the parameters. Can be `dict`-like or
237 `set`-like. The parameter values are not checked.
238 If no parameters are supplied, always returns without error.
240 Raises
241 ------
242 KeyError
243 Some parameters are not understood by this `StorageClass`.
244 """
245 # No parameters is always okay
246 if not parameters:
247 return
249 # Extract the important information into a set. Works for dict and
250 # list.
251 external = set(parameters)
253 diff = external - self.knownParameters()
254 if diff:
255 s = "s" if len(diff) > 1 else ""
256 unknown = '\', \''.join(diff)
257 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}")
259 def filterParameters(self, parameters: Dict[str, Any],
260 subset: Collection = None) -> Dict[str, Any]:
261 """Filter out parameters that are not known to this StorageClass
263 Parameters
264 ----------
265 parameters : `dict`, optional
266 Candidate parameters. Can be `None` if no parameters have
267 been provided.
268 subset : `~collections.abc.Collection`, optional
269 Subset of supported parameters that the caller is interested
270 in using. The subset must be known to the `StorageClass`
271 if specified.
273 Returns
274 -------
275 filtered : `dict`
276 Valid parameters. Empty `dict` if none are suitable.
277 """
278 if not parameters:
279 return {}
280 subset = set(subset) if subset is not None else set()
281 known = self.knownParameters()
282 if not subset.issubset(known):
283 raise ValueError(f"Requested subset ({subset}) is not a subset of"
284 f" known parameters ({known})")
285 return {k: parameters[k] for k in known if k in parameters}
287 def validateInstance(self, instance: Any) -> bool:
288 """Check that the supplied Python object has the expected Python type
290 Parameters
291 ----------
292 instance : `object`
293 Object to check.
295 Returns
296 -------
297 isOk : `bool`
298 True if the supplied instance object can be handled by this
299 `StorageClass`, False otherwise.
300 """
301 return isinstance(instance, self.pytype)
303 def __eq__(self, other: Any) -> bool:
304 """Equality checks name, pytype name, assembler name, and components"""
306 if not isinstance(other, StorageClass):
307 return False
309 if self.name != other.name:
310 return False
312 # We must compare pytype and assembler by name since we do not want
313 # to trigger an import of external module code here
314 if self._assemblerClassName != other._assemblerClassName:
315 return False
316 if self._pytypeName != other._pytypeName:
317 return False
319 # Ensure we have the same component keys in each
320 if set(self.components.keys()) != set(other.components.keys()):
321 return False
323 # Same parameters
324 if self.parameters != other.parameters:
325 return False
327 # Ensure that all the components have the same type
328 for k in self.components:
329 if self.components[k] != other.components[k]:
330 return False
332 # If we got to this point everything checks out
333 return True
335 def __hash__(self) -> int:
336 return hash(self.name)
338 def __repr__(self) -> str:
339 optionals: Dict[str, Any] = {}
340 if self._pytypeName != "object":
341 optionals["pytype"] = self._pytypeName
342 if self._assemblerClassName is not None:
343 optionals["assembler"] = self._assemblerClassName
344 if self._parameters:
345 optionals["parameters"] = self._parameters
346 if self.components:
347 optionals["components"] = self.components
349 # order is preserved in the dict
350 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items())
352 # Start with mandatory fields
353 r = f"{self.__class__.__name__}({self.name!r}"
354 if options:
355 r = r + ", " + options
356 r = r + ")"
357 return r
359 def __str__(self) -> str:
360 return self.name
363class StorageClassFactory(metaclass=Singleton):
364 """Factory for `StorageClass` instances.
366 This class is a singleton, with each instance sharing the pool of
367 StorageClasses. Since code can not know whether it is the first
368 time the instance has been created, the constructor takes no arguments.
369 To populate the factory with storage classes, a call to
370 `~StorageClassFactory.addFromConfig()` should be made.
372 Parameters
373 ----------
374 config : `StorageClassConfig` or `str`, optional
375 Load configuration. In a ButlerConfig` the relevant configuration
376 is located in the ``storageClasses`` section.
377 """
379 def __init__(self, config: Optional[Union[StorageClassConfig, str]] = None):
380 self._storageClasses: Dict[str, StorageClass] = {}
381 self._configs: List[StorageClassConfig] = []
383 # Always seed with the default config
384 self.addFromConfig(StorageClassConfig())
386 if config is not None:
387 self.addFromConfig(config)
389 def __str__(self) -> str:
390 """Return summary of factory.
392 Returns
393 -------
394 summary : `str`
395 Summary of the factory status.
396 """
397 sep = "\n"
398 return f"""Number of registered StorageClasses: {len(self._storageClasses)}
400StorageClasses
401--------------
402{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)}
403"""
405 def __contains__(self, storageClassOrName: Union[StorageClass, str]) -> bool:
406 """Indicates whether the storage class exists in the factory.
408 Parameters
409 ----------
410 storageClassOrName : `str` or `StorageClass`
411 If `str` is given existence of the named StorageClass
412 in the factory is checked. If `StorageClass` is given
413 existence and equality are checked.
415 Returns
416 -------
417 in : `bool`
418 True if the supplied string is present, or if the supplied
419 `StorageClass` is present and identical.
421 Notes
422 -----
423 The two different checks (one for "key" and one for "value") based on
424 the type of the given argument mean that it is possible for
425 StorageClass.name to be in the factory but StorageClass to not be
426 in the factory.
427 """
428 if isinstance(storageClassOrName, str):
429 return storageClassOrName in self._storageClasses
430 elif isinstance(storageClassOrName, StorageClass):
431 if storageClassOrName.name in self._storageClasses:
432 return storageClassOrName == self._storageClasses[storageClassOrName.name]
433 return False
435 def addFromConfig(self, config: Union[StorageClassConfig, Config, str]) -> None:
436 """Add more `StorageClass` definitions from a config file.
438 Parameters
439 ----------
440 config : `StorageClassConfig`, `Config` or `str`
441 Storage class configuration. Can contain a ``storageClasses``
442 key if part of a global configuration.
443 """
444 sconfig = StorageClassConfig(config)
445 self._configs.append(sconfig)
447 # Since we can not assume that we will get definitions of
448 # components or parents before their classes are defined
449 # we have a helper function that we can call recursively
450 # to extract definitions from the configuration.
451 def processStorageClass(name: str, sconfig: StorageClassConfig) -> None:
452 # Maybe we've already processed this through recursion
453 if name not in sconfig:
454 return
455 info = sconfig.pop(name)
457 # Always create the storage class so we can ensure that
458 # we are not trying to overwrite with a different definition
459 components = None
460 if "components" in info:
461 components = {}
462 for cname, ctype in info["components"].items():
463 if ctype not in self:
464 processStorageClass(ctype, sconfig)
465 components[cname] = self.getStorageClass(ctype)
467 # Extract scalar items from dict that are needed for
468 # StorageClass Constructor
469 storageClassKwargs = {k: info[k] for k in ("pytype", "assembler", "parameters") if k in info}
471 # Fill in other items
472 storageClassKwargs["components"] = components
474 # Create the new storage class and register it
475 baseClass = None
476 if "inheritsFrom" in info:
477 baseName = info["inheritsFrom"]
478 if baseName not in self:
479 processStorageClass(baseName, sconfig)
480 baseClass = type(self.getStorageClass(baseName))
482 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs)
483 newStorageClass = newStorageClassType()
484 self.registerStorageClass(newStorageClass)
486 for name in list(sconfig.keys()):
487 processStorageClass(name, sconfig)
489 @staticmethod
490 def makeNewStorageClass(name: str,
491 baseClass: Optional[Type[StorageClass]] = StorageClass,
492 **kwargs: Any) -> Type[StorageClass]:
493 """Create a new Python class as a subclass of `StorageClass`.
495 Parameters
496 ----------
497 name : `str`
498 Name to use for this class.
499 baseClass : `type`, optional
500 Base class for this `StorageClass`. Must be either `StorageClass`
501 or a subclass of `StorageClass`. If `None`, `StorageClass` will
502 be used.
504 Returns
505 -------
506 newtype : `type` subclass of `StorageClass`
507 Newly created Python type.
508 """
510 if baseClass is None:
511 baseClass = StorageClass
512 if not issubclass(baseClass, StorageClass):
513 raise ValueError(f"Base class must be a StorageClass not {baseClass}")
515 # convert the arguments to use different internal names
516 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None}
517 clsargs["_cls_name"] = name
519 # Some container items need to merge with the base class values
520 # so that a child can inherit but override one bit.
521 # lists (which you get from configs) are treated as sets for this to
522 # work consistently.
523 for k in ("components", "parameters"):
524 classKey = f"_cls_{k}"
525 if classKey in clsargs:
526 baseValue = getattr(baseClass, classKey, None)
527 if baseValue is not None:
528 currentValue = clsargs[classKey]
529 if isinstance(currentValue, dict):
530 newValue = baseValue.copy()
531 else:
532 newValue = set(baseValue)
533 newValue.update(currentValue)
534 clsargs[classKey] = newValue
536 # If we have parameters they should be a frozen set so that the
537 # parameters in the class can not be modified.
538 pk = "_cls_parameters"
539 if pk in clsargs:
540 clsargs[pk] = frozenset(clsargs[pk])
542 return type(f"StorageClass{name}", (baseClass,), clsargs)
544 def getStorageClass(self, storageClassName: str) -> StorageClass:
545 """Get a StorageClass instance associated with the supplied name.
547 Parameters
548 ----------
549 storageClassName : `str`
550 Name of the storage class to retrieve.
552 Returns
553 -------
554 instance : `StorageClass`
555 Instance of the correct `StorageClass`.
557 Raises
558 ------
559 KeyError
560 The requested storage class name is not registered.
561 """
562 return self._storageClasses[storageClassName]
564 def registerStorageClass(self, storageClass: StorageClass) -> None:
565 """Store the `StorageClass` in the factory.
567 Will be indexed by `StorageClass.name` and will return instances
568 of the supplied `StorageClass`.
570 Parameters
571 ----------
572 storageClass : `StorageClass`
573 Type of the Python `StorageClass` to register.
575 Raises
576 ------
577 ValueError
578 If a storage class has already been registered with
579 storageClassName and the previous definition differs.
580 """
581 if storageClass.name in self._storageClasses:
582 existing = self.getStorageClass(storageClass.name)
583 if existing != storageClass:
584 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) "
585 f"differs from current definition ({existing})")
586 else:
587 self._storageClasses[storageClass.name] = storageClass
589 def _unregisterStorageClass(self, storageClassName: str) -> None:
590 """Remove the named StorageClass from the factory.
592 Parameters
593 ----------
594 storageClassName : `str`
595 Name of storage class to remove.
597 Raises
598 ------
599 KeyError
600 The named storage class is not registered.
602 Notes
603 -----
604 This method is intended to simplify testing of StorageClassFactory
605 functionality and it is not expected to be required for normal usage.
606 """
607 del self._storageClasses[storageClassName]