Coverage for python/lsst/daf/butler/core/storageClassDelegate.py: 16%
118 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Support for reading and writing composite objects."""
24from __future__ import annotations
26__all__ = ("DatasetComponent", "StorageClassDelegate")
28import collections.abc
29from dataclasses import dataclass
30import logging
31from typing import (
32 Any,
33 Dict,
34 Iterable,
35 Mapping,
36 Optional,
37 Set,
38 Tuple,
39 Type,
40 TYPE_CHECKING,
41)
43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .storageClass import StorageClass
46log = logging.getLogger(__name__)
49@dataclass
50class DatasetComponent:
51 """Component of a dataset and associated information."""
53 name: str
54 """Name of the component.
55 """
57 storageClass: StorageClass
58 """StorageClass to be used when reading or writing this component.
59 """
61 component: Any
62 """Component extracted from the composite object.
63 """
66class StorageClassDelegate:
67 """Delegate class for StorageClass components and parameters.
69 This class delegates the handling of components and parameters for the
70 python type associated with a particular `StorageClass`.
72 A delegate is required for any storage class that defines components
73 (derived or otherwise) or support read parameters. It is used for
74 composite disassembly and assembly.
76 Attributes
77 ----------
78 storageClass : `StorageClass`
80 Parameters
81 ----------
82 storageClass : `StorageClass`
83 `StorageClass` to be used with this delegate.
84 """
86 def __init__(self, storageClass: StorageClass):
87 assert storageClass is not None
88 self.storageClass = storageClass
90 @staticmethod
91 def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]:
92 """Return list of suitable attribute names to attempt to use.
94 Parameters
95 ----------
96 componentName : `str`
97 Name of component/attribute to look for.
98 getter : `bool`
99 If true, return getters, else return setters.
101 Returns
102 -------
103 attrs : `tuple(str)`
104 Tuple of strings to attempt.
105 """
106 root = "get" if getter else "set"
108 # Capitalized name for getXxx must only capitalize first letter and not
109 # downcase the rest. getVisitInfo and not getVisitinfo
110 first = componentName[0].upper()
111 if len(componentName) > 1:
112 tail = componentName[1:]
113 else:
114 tail = ""
115 capitalized = "{}{}{}".format(root, first, tail)
116 return (componentName, "{}_{}".format(root, componentName), capitalized)
118 def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any:
119 """Construct an object from components based on storageClass.
121 This generic implementation assumes that instances of objects
122 can be created either by passing all the components to a constructor
123 or by calling setter methods with the name.
125 Parameters
126 ----------
127 components : `dict`
128 Collection of components from which to assemble a new composite
129 object. Keys correspond to composite names in the `StorageClass`.
130 pytype : `type`, optional
131 Override the type from the
132 :attr:`StorageClassDelegate.storageClass`
133 to use when assembling the final object.
135 Returns
136 -------
137 composite : `object`
138 New composite object assembled from components.
140 Raises
141 ------
142 ValueError
143 Some components could not be used to create the object or,
144 alternatively, some components were not defined in the associated
145 StorageClass.
146 """
147 if pytype is not None:
148 cls = pytype
149 else:
150 cls = self.storageClass.pytype
152 # Check that the storage class components are consistent
153 understood = set(self.storageClass.components)
154 requested = set(components.keys())
155 unknown = requested - understood
156 if unknown:
157 raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown))
159 # First try to create an instance directly using keyword args
160 try:
161 obj = cls(**components)
162 except TypeError:
163 obj = None
165 # Now try to use setters if direct instantiation didn't work
166 if not obj:
167 obj = cls()
169 failed = []
170 for name, component in components.items():
171 if component is None:
172 continue
173 for attr in self._attrNames(name, getter=False):
174 if hasattr(obj, attr):
175 if attr == name: # Real attribute
176 setattr(obj, attr, component)
177 else:
178 setter = getattr(obj, attr)
179 setter(component)
180 break
181 else:
182 failed.append(name)
184 if failed:
185 raise ValueError("Unhandled components during assembly ({})".format(failed))
187 return obj
189 def getValidComponents(self, composite: Any) -> Dict[str, Any]:
190 """Extract all non-None components from a composite.
192 Parameters
193 ----------
194 composite : `object`
195 Composite from which to extract components.
197 Returns
198 -------
199 comps : `dict`
200 Non-None components extracted from the composite, indexed by the
201 component name as derived from the
202 `StorageClassDelegate.storageClass`.
203 """
204 components = {}
205 if self.storageClass.isComposite():
206 for c in self.storageClass.components:
207 if isinstance(composite, collections.abc.Mapping):
208 comp = composite[c]
209 else:
210 try:
211 comp = self.getComponent(composite, c)
212 except AttributeError:
213 pass
214 else:
215 if comp is not None:
216 components[c] = comp
217 return components
219 def getComponent(self, composite: Any, componentName: str) -> Any:
220 """Attempt to retrieve component from composite object by heuristic.
222 Will attempt a direct attribute retrieval, or else getter methods of
223 the form "get_componentName" and "getComponentName".
225 Parameters
226 ----------
227 composite : `object`
228 Item to query for the component.
229 componentName : `str`
230 Name of component to retrieve.
232 Returns
233 -------
234 component : `object`
235 Component extracted from composite.
237 Raises
238 ------
239 AttributeError
240 The attribute could not be read from the composite.
241 """
242 component = None
244 if hasattr(composite, "__contains__") and componentName in composite:
245 component = composite[componentName]
246 return component
248 for attr in self._attrNames(componentName, getter=True):
249 if hasattr(composite, attr):
250 component = getattr(composite, attr)
251 if attr != componentName: # We have a method
252 component = component()
253 break
254 else:
255 raise AttributeError("Unable to get component {}".format(componentName))
256 return component
258 def disassemble(self, composite: Any, subset: Optional[Iterable] = None,
259 override: bool = None) -> Dict[str, Any]:
260 """Disassembler a composite.
262 This is a generic implementation of a disassembler.
263 This implementation attempts to extract components from the parent
264 by looking for attributes of the same name or getter methods derived
265 from the component name.
267 Parameters
268 ----------
269 composite : `object`
270 Parent composite object consisting of components to be extracted.
271 subset : iterable, optional
272 Iterable containing subset of components to extract from composite.
273 Must be a subset of those defined in
274 `StorageClassDelegate.storageClass`.
275 override : `object`, optional
276 Object to use for disassembly instead of parent. This can be useful
277 when called from subclasses that have composites in a hierarchy.
279 Returns
280 -------
281 components : `dict`
282 `dict` with keys matching the components defined in
283 `StorageClassDelegate.storageClass`
284 and values being `DatasetComponent` instances describing the
285 component. Returns None if this is not a composite
286 `StorageClassDelegate.storageClass`.
288 Raises
289 ------
290 ValueError
291 A requested component can not be found in the parent using generic
292 lookups.
293 TypeError
294 The parent object does not match the supplied
295 `StorageClassDelegate.storageClass`.
296 """
297 if not self.storageClass.isComposite():
298 raise TypeError("Can not disassemble something that is not a composite"
299 f" (storage class={self.storageClass})")
301 if not self.storageClass.validateInstance(composite):
302 raise TypeError("Unexpected type mismatch between parent and StorageClass"
303 " ({} != {})".format(type(composite), self.storageClass.pytype))
305 requested = set(self.storageClass.components)
307 if subset is not None:
308 subset = set(subset)
309 diff = subset - requested
310 if diff:
311 raise ValueError("Requested subset is not a subset of supported components: {}".format(diff))
312 requested = subset
314 if override is not None:
315 composite = override
317 components = {}
318 for c in list(requested):
319 # Try three different ways to get a value associated with the
320 # component name.
321 try:
322 component = self.getComponent(composite, c)
323 except AttributeError:
324 # Defer complaining so we get an idea of how many problems
325 # we have
326 pass
327 else:
328 # If we found a match store it in the results dict and remove
329 # it from the list of components we are still looking for.
330 if component is not None:
331 components[c] = DatasetComponent(c, self.storageClass.components[c], component)
332 requested.remove(c)
334 if requested:
335 raise ValueError("Unhandled components during disassembly ({})".format(requested))
337 return components
339 def handleParameters(self, inMemoryDataset: Any, parameters: Optional[Mapping[str, Any]] = None) -> Any:
340 """Modify the in-memory dataset using the supplied parameters.
342 Can return a possibly new object.
344 For safety, if any parameters are given to this method an
345 exception will be raised. This is to protect the user from
346 thinking that parameters have been applied when they have not been
347 applied.
349 Parameters
350 ----------
351 inMemoryDataset : `object`
352 Object to modify based on the parameters.
353 parameters : `dict`
354 Parameters to apply. Values are specific to the parameter.
355 Supported parameters are defined in the associated
356 `StorageClass`. If no relevant parameters are specified the
357 inMemoryDataset will be return unchanged.
359 Returns
360 -------
361 inMemoryDataset : `object`
362 Updated form of supplied in-memory dataset, after parameters
363 have been used.
365 Raises
366 ------
367 ValueError
368 Parameters have been provided to this default implementation.
369 """
370 if parameters:
371 raise ValueError(f"Parameters ({parameters}) provided to default implementation.")
373 return inMemoryDataset
375 @classmethod
376 def selectResponsibleComponent(cls, derivedComponent: str, fromComponents: Set[Optional[str]]) -> str:
377 """Select the best component for calcluating a derived component.
379 Given a possible set of components to choose from, return the
380 component that should be used to calculate the requested derived
381 component.
383 Parameters
384 ----------
385 derivedComponent : `str`
386 The derived component that is being requested.
387 fromComponents : `set` of `str`
388 The available set of component options from which that derived
389 component can be computed. `None` can be included but should
390 be ignored.
392 Returns
393 -------
394 required : `str`
395 The component that should be used.
397 Raises
398 ------
399 NotImplementedError
400 Raised if this delegate refuses to answer the question.
401 ValueError
402 Raised if this delegate can not determine a relevant component
403 from the supplied options.
404 """
405 raise NotImplementedError("This delegate does not support derived components")