Coverage for python/lsst/daf/butler/core/storageClass.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Support for Storage Classes."""
24__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig")
26import builtins
27import logging
29from lsst.utils import doImport
30from .utils import Singleton, getFullTypeName
31from .assembler import CompositeAssembler
32from .config import ConfigSubset
33from .configSupport import LookupKey
35log = logging.getLogger(__name__)
38class StorageClassConfig(ConfigSubset):
39 component = "storageClasses"
40 defaultConfigFile = "storageClasses.yaml"
43class StorageClass:
44 """Class describing how a label maps to a particular Python type.
46 Parameters
47 ----------
48 name : `str`
49 Name to use for this class.
50 pytype : `type`
51 Python type (or name of type) to associate with the `StorageClass`
52 components : `dict`, optional
53 `dict` mapping name of a component to another `StorageClass`.
54 parameters : `~collections.abc.Sequence` or `~collections.abc.Set`
55 Parameters understood by this `StorageClass`.
56 assembler : `str`, optional
57 Fully qualified name of class supporting assembly and disassembly
58 of a `pytype` instance.
59 """
60 _cls_name = "BaseStorageClass"
61 _cls_components = None
62 _cls_parameters = None
63 _cls_assembler = None
64 _cls_pytype = None
65 defaultAssembler = CompositeAssembler
66 defaultAssemblerName = getFullTypeName(defaultAssembler)
68 def __init__(self, name=None, pytype=None, components=None, parameters=None, assembler=None):
69 if name is None:
70 name = self._cls_name
71 if pytype is None:
72 pytype = self._cls_pytype
73 if components is None:
74 components = self._cls_components
75 if parameters is None:
76 parameters = self._cls_parameters
77 if assembler is None:
78 assembler = self._cls_assembler
79 self.name = name
81 if pytype is None:
82 pytype = object
84 if not isinstance(pytype, str):
85 # Already have a type so store it and get the name
86 self._pytypeName = getFullTypeName(pytype)
87 self._pytype = pytype
88 else:
89 # Store the type name and defer loading of type
90 self._pytypeName = pytype
92 self._components = components if components is not None else {}
93 self._parameters = frozenset(parameters) if parameters is not None else frozenset()
94 # if the assembler is not None also set it and clear the default
95 # assembler
96 if assembler is not None:
97 self._assemblerClassName = assembler
98 self._assembler = None
99 elif components is not None:
100 # We set a default assembler for composites so that a class is
101 # guaranteed to support something if it is a composite.
102 log.debug("Setting default assembler for %s", self.name)
103 self._assembler = self.defaultAssembler
104 self._assemblerClassName = self.defaultAssemblerName
105 else:
106 self._assembler = None
107 self._assemblerClassName = None
108 # The types are created on demand and cached
109 self._pytype = None
111 @property
112 def components(self):
113 """Component names mapped to associated `StorageClass`
114 """
115 return self._components
117 @property
118 def parameters(self):
119 """`set` of names of parameters supported by this `StorageClass`
120 """
121 return set(self._parameters)
123 @property
124 def pytype(self):
125 """Python type associated with this `StorageClass`."""
126 if self._pytype is not None:
127 return self._pytype
129 if hasattr(builtins, self._pytypeName):
130 pytype = getattr(builtins, self._pytypeName)
131 else:
132 pytype = doImport(self._pytypeName)
133 self._pytype = pytype
134 return self._pytype
136 @property
137 def assemblerClass(self):
138 """Class to use to (dis)assemble an object from components."""
139 if self._assembler is not None:
140 return self._assembler
141 if self._assemblerClassName is None:
142 return None
143 self._assembler = doImport(self._assemblerClassName)
144 return self._assembler
146 def assembler(self):
147 """Return an instance of an assembler.
149 Returns
150 -------
151 assembler : `CompositeAssembler`
152 Instance of the assembler associated with this `StorageClass`.
153 Assembler is constructed with this `StorageClass`.
155 Raises
156 ------
157 TypeError
158 This StorageClass has no associated assembler.
159 """
160 cls = self.assemblerClass
161 if cls is None:
162 raise TypeError(f"No assembler class is associated with StorageClass {self.name}")
163 return cls(storageClass=self)
165 def isComposite(self):
166 """Boolean indicating whether this `StorageClass` is a composite
167 or not.
169 Returns
170 -------
171 isComposite : `bool`
172 `True` if this `StorageClass` is a composite, `False`
173 otherwise.
174 """
175 if self.components:
176 return True
177 return False
179 def _lookupNames(self):
180 """Keys to use when looking up this DatasetRef in a configuration.
182 The names are returned in order of priority.
184 Returns
185 -------
186 names : `tuple` of `LookupKey`
187 Tuple of a `LookupKey` using the `StorageClass` name.
188 """
189 return (LookupKey(name=self.name), )
191 def knownParameters(self):
192 """Return set of all parameters known to this `StorageClass`
194 The set includes parameters understood by components of a composite.
196 Returns
197 -------
198 known : `set`
199 All parameter keys of this `StorageClass` and the component
200 storage classes.
201 """
202 known = set(self._parameters)
203 for sc in self.components.values():
204 known.update(sc.knownParameters())
205 return known
207 def validateParameters(self, parameters=None):
208 """Check that the parameters are known to this `StorageClass`
210 Does not check the values.
212 Parameters
213 ----------
214 parameters : `~collections.abc.Collection`, optional
215 Collection containing the parameters. Can be `dict`-like or
216 `set`-like. The parameter values are not checked.
217 If no parameters are supplied, always returns without error.
219 Raises
220 ------
221 KeyError
222 Some parameters are not understood by this `StorageClass`.
223 """
224 # No parameters is always okay
225 if not parameters:
226 return
228 # Extract the important information into a set. Works for dict and
229 # list.
230 external = set(parameters)
232 diff = external - self.knownParameters()
233 if diff:
234 s = "s" if len(diff) > 1 else ""
235 unknown = '\', \''.join(diff)
236 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}")
238 def filterParameters(self, parameters, subset=None):
239 """Filter out parameters that are not known to this StorageClass
241 Parameters
242 ----------
243 parameters : `dict`, optional
244 Candidate parameters. Can be `None` if no parameters have
245 been provided.
246 subset : `~collections.abc.Collection`, optional
247 Subset of supported parameters that the caller is interested
248 in using. The subset must be known to the `StorageClass`
249 if specified.
251 Returns
252 -------
253 filtered : `dict`
254 Valid parameters. Empty `dict` if none are suitable.
255 """
256 if not parameters:
257 return {}
258 subset = set(subset) if subset is not None else set()
259 known = self.knownParameters()
260 if not subset.issubset(known):
261 raise ValueError(f"Requested subset ({subset}) is not a subset of"
262 f" known parameters ({known})")
263 return {k: parameters[k] for k in known if k in parameters}
265 def validateInstance(self, instance):
266 """Check that the supplied Python object has the expected Python type
268 Parameters
269 ----------
270 instance : `object`
271 Object to check.
273 Returns
274 -------
275 isOk : `bool`
276 True if the supplied instance object can be handled by this
277 `StorageClass`, False otherwise.
278 """
279 return isinstance(instance, self.pytype)
281 def __eq__(self, other):
282 """Equality checks name, pytype name, assembler name, and components"""
283 if self.name != other.name:
284 return False
286 if not isinstance(other, StorageClass):
287 return False
289 # We must compare pytype and assembler by name since we do not want
290 # to trigger an import of external module code here
291 if self._assemblerClassName != other._assemblerClassName:
292 return False
293 if self._pytypeName != other._pytypeName:
294 return False
296 # Ensure we have the same component keys in each
297 if set(self.components.keys()) != set(other.components.keys()):
298 return False
300 # Same parameters
301 if self.parameters != other.parameters:
302 return False
304 # Ensure that all the components have the same type
305 for k in self.components:
306 if self.components[k] != other.components[k]:
307 return False
309 # If we got to this point everything checks out
310 return True
312 def __hash__(self):
313 return hash(self.name)
315 def __repr__(self):
316 optionals = {}
317 if self._pytypeName != "object":
318 optionals["pytype"] = self._pytypeName
319 if self._assemblerClassName is not None:
320 optionals["assembler"] = self._assemblerClassName
321 if self._parameters:
322 optionals["parameters"] = self._parameters
323 if self.components:
324 optionals["components"] = self.components
326 # order is preserved in the dict
327 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items())
329 # Start with mandatory fields
330 r = f"{self.__class__.__name__}({self.name!r}"
331 if options:
332 r = r + ", " + options
333 r = r + ")"
334 return r
336 def __str__(self):
337 return self.name
340class StorageClassFactory(metaclass=Singleton):
341 """Factory for `StorageClass` instances.
343 This class is a singleton, with each instance sharing the pool of
344 StorageClasses. Since code can not know whether it is the first
345 time the instance has been created, the constructor takes no arguments.
346 To populate the factory with storage classes, a call to
347 `~StorageClassFactory.addFromConfig()` should be made.
349 Parameters
350 ----------
351 config : `StorageClassConfig` or `str`, optional
352 Load configuration. In a ButlerConfig` the relevant configuration
353 is located in the ``storageClasses`` section.
354 """
356 def __init__(self, config=None):
357 self._storageClasses = {}
358 self._configs = []
360 # Always seed with the default config
361 self.addFromConfig(StorageClassConfig())
363 if config is not None:
364 self.addFromConfig(config)
366 def __str__(self):
367 """Return summary of factory.
369 Returns
370 -------
371 summary : `str`
372 Summary of the factory status.
373 """
374 sep = "\n"
375 return f"""Number of registered StorageClasses: {len(self._storageClasses)}
377StorageClasses
378--------------
379{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)}
380"""
382 def __contains__(self, storageClassOrName):
383 """Indicates whether the storage class exists in the factory.
385 Parameters
386 ----------
387 storageClassOrName : `str` or `StorageClass`
388 If `str` is given existence of the named StorageClass
389 in the factory is checked. If `StorageClass` is given
390 existence and equality are checked.
392 Returns
393 -------
394 in : `bool`
395 True if the supplied string is present, or if the supplied
396 `StorageClass` is present and identical.
398 Notes
399 -----
400 The two different checks (one for "key" and one for "value") based on
401 the type of the given argument mean that it is possible for
402 StorageClass.name to be in the factory but StorageClass to not be
403 in the factory.
404 """
405 if isinstance(storageClassOrName, str):
406 return storageClassOrName in self._storageClasses
407 elif isinstance(storageClassOrName, StorageClass):
408 if storageClassOrName.name in self._storageClasses:
409 return storageClassOrName == self._storageClasses[storageClassOrName.name]
410 return False
412 def addFromConfig(self, config):
413 """Add more `StorageClass` definitions from a config file.
415 Parameters
416 ----------
417 config : `StorageClassConfig`, `Config` or `str`
418 Storage class configuration. Can contain a ``storageClasses``
419 key if part of a global configuration.
420 """
421 sconfig = StorageClassConfig(config)
422 self._configs.append(sconfig)
424 # Since we can not assume that we will get definitions of
425 # components or parents before their classes are defined
426 # we have a helper function that we can call recursively
427 # to extract definitions from the configuration.
428 def processStorageClass(name, sconfig):
429 # Maybe we've already processed this through recursion
430 if name not in sconfig:
431 return
432 info = sconfig.pop(name)
434 # Always create the storage class so we can ensure that
435 # we are not trying to overwrite with a different definition
436 components = None
437 if "components" in info:
438 components = {}
439 for cname, ctype in info["components"].items():
440 if ctype not in self:
441 processStorageClass(ctype, sconfig)
442 components[cname] = self.getStorageClass(ctype)
444 # Extract scalar items from dict that are needed for
445 # StorageClass Constructor
446 storageClassKwargs = {k: info[k] for k in ("pytype", "assembler", "parameters") if k in info}
448 # Fill in other items
449 storageClassKwargs["components"] = components
451 # Create the new storage class and register it
452 baseClass = None
453 if "inheritsFrom" in info:
454 baseName = info["inheritsFrom"]
455 if baseName not in self:
456 processStorageClass(baseName, sconfig)
457 baseClass = type(self.getStorageClass(baseName))
459 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs)
460 newStorageClass = newStorageClassType()
461 self.registerStorageClass(newStorageClass)
463 for name in list(sconfig.keys()):
464 processStorageClass(name, sconfig)
466 @staticmethod
467 def makeNewStorageClass(name, baseClass=StorageClass, **kwargs):
468 """Create a new Python class as a subclass of `StorageClass`.
470 Parameters
471 ----------
472 name : `str`
473 Name to use for this class.
474 baseClass : `type`, optional
475 Base class for this `StorageClass`. Must be either `StorageClass`
476 or a subclass of `StorageClass`. If `None`, `StorageClass` will
477 be used.
479 Returns
480 -------
481 newtype : `type` subclass of `StorageClass`
482 Newly created Python type.
483 """
485 if baseClass is None:
486 baseClass = StorageClass
487 if not issubclass(baseClass, StorageClass):
488 raise ValueError(f"Base class must be a StorageClass not {baseClass}")
490 # convert the arguments to use different internal names
491 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None}
492 clsargs["_cls_name"] = name
494 # Some container items need to merge with the base class values
495 # so that a child can inherit but override one bit.
496 # lists (which you get from configs) are treated as sets for this to
497 # work consistently.
498 for k in ("components", "parameters"):
499 classKey = f"_cls_{k}"
500 if classKey in clsargs:
501 baseValue = getattr(baseClass, classKey, None)
502 if baseValue is not None:
503 currentValue = clsargs[classKey]
504 if isinstance(currentValue, dict):
505 newValue = baseValue.copy()
506 else:
507 newValue = set(baseValue)
508 newValue.update(currentValue)
509 clsargs[classKey] = newValue
511 # If we have parameters they should be a frozen set so that the
512 # parameters in the class can not be modified.
513 pk = f"_cls_parameters"
514 if pk in clsargs:
515 clsargs[pk] = frozenset(clsargs[pk])
517 return type(f"StorageClass{name}", (baseClass,), clsargs)
519 def getStorageClass(self, storageClassName):
520 """Get a StorageClass instance associated with the supplied name.
522 Parameters
523 ----------
524 storageClassName : `str`
525 Name of the storage class to retrieve.
527 Returns
528 -------
529 instance : `StorageClass`
530 Instance of the correct `StorageClass`.
532 Raises
533 ------
534 KeyError
535 The requested storage class name is not registered.
536 """
537 return self._storageClasses[storageClassName]
539 def registerStorageClass(self, storageClass):
540 """Store the `StorageClass` in the factory.
542 Will be indexed by `StorageClass.name` and will return instances
543 of the supplied `StorageClass`.
545 Parameters
546 ----------
547 storageClass : `StorageClass`
548 Type of the Python `StorageClass` to register.
550 Raises
551 ------
552 ValueError
553 If a storage class has already been registered with
554 storageClassName and the previous definition differs.
555 """
556 if storageClass.name in self._storageClasses:
557 existing = self.getStorageClass(storageClass.name)
558 if existing != storageClass:
559 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) "
560 f"differs from current definition ({existing})")
561 else:
562 self._storageClasses[storageClass.name] = storageClass
564 def _unregisterStorageClass(self, storageClassName):
565 """Remove the named StorageClass from the factory.
567 Parameters
568 ----------
569 storageClassName : `str`
570 Name of storage class to remove.
572 Raises
573 ------
574 KeyError
575 The named storage class is not registered.
577 Notes
578 -----
579 This method is intended to simplify testing of StorageClassFactory
580 functionality and it is not expected to be required for normal usage.
581 """
582 del self._storageClasses[storageClassName]