Coverage for python/lsst/daf/butler/tests/_examplePythonTypes.py: 37%
109 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""
29Python classes that can be used to test datastores without requiring
30large external dependencies on python classes such as afw or serialization
31formats such as FITS or HDF5.
32"""
34from __future__ import annotations
36__all__ = (
37 "ListDelegate",
38 "MetricsDelegate",
39 "MetricsExample",
40 "registerMetricsExample",
41 "MetricsExampleModel",
42 "MetricsExampleDataclass",
43)
46import copy
47import dataclasses
48import types
49from collections.abc import Mapping
50from typing import TYPE_CHECKING, Any
52from lsst.daf.butler import StorageClass, StorageClassDelegate
53from pydantic import BaseModel
55if TYPE_CHECKING:
56 from lsst.daf.butler import Butler, Datastore, FormatterFactory
59def registerMetricsExample(butler: Butler) -> None:
60 """Modify a repository to support reading and writing
61 `MetricsExample` objects.
63 This method allows `MetricsExample` to be used with test repositories
64 in any package without needing to provide a custom configuration there.
66 Parameters
67 ----------
68 butler : `lsst.daf.butler.Butler`
69 The repository that needs to support `MetricsExample`.
71 Notes
72 -----
73 This method enables the following storage classes:
75 ``StructuredData``
76 A `MetricsExample` whose ``summary``, ``output``, and ``data`` members
77 can be retrieved as dataset components.
78 ``StructuredDataNoComponents``
79 A monolithic write of a `MetricsExample`.
80 """
81 yamlDict = _addFullStorageClass(
82 butler,
83 "StructuredDataDictYaml",
84 "lsst.daf.butler.formatters.yaml.YamlFormatter",
85 pytype=dict,
86 )
88 yamlList = _addFullStorageClass(
89 butler,
90 "StructuredDataListYaml",
91 "lsst.daf.butler.formatters.yaml.YamlFormatter",
92 pytype=list,
93 parameters={"slice"},
94 delegate="lsst.daf.butler.tests.ListDelegate",
95 )
97 _addFullStorageClass(
98 butler,
99 "StructuredDataNoComponents",
100 "lsst.daf.butler.formatters.pickle.PickleFormatter",
101 pytype=MetricsExample,
102 parameters={"slice"},
103 delegate="lsst.daf.butler.tests.MetricsDelegate",
104 )
106 _addFullStorageClass(
107 butler,
108 "StructuredData",
109 "lsst.daf.butler.formatters.yaml.YamlFormatter",
110 pytype=MetricsExample,
111 components={
112 "summary": yamlDict,
113 "output": yamlDict,
114 "data": yamlList,
115 },
116 delegate="lsst.daf.butler.tests.MetricsDelegate",
117 )
120def _addFullStorageClass(
121 butler: Butler, name: str, formatter: str, *args: Any, **kwargs: Any
122) -> StorageClass:
123 """Create a storage class-formatter pair in a repository if it does not
124 already exist.
126 Parameters
127 ----------
128 butler : `lsst.daf.butler.Butler`
129 The repository that needs to contain the class.
130 name : `str`
131 The name to use for the class.
132 formatter : `str`
133 The formatter to use with the storage class. Ignored if ``butler``
134 does not use formatters.
135 *args
136 **kwargs
137 Arguments, other than ``name``, to the `~lsst.daf.butler.StorageClass`
138 constructor.
140 Returns
141 -------
142 class : `lsst.daf.butler.StorageClass`
143 The newly created storage class, or the class of the same name
144 previously found in the repository.
145 """
146 storageRegistry = butler._datastore.storageClassFactory
148 storage = StorageClass(name, *args, **kwargs)
149 try:
150 storageRegistry.registerStorageClass(storage)
151 except ValueError:
152 storage = storageRegistry.getStorageClass(name)
154 for registry in _getAllFormatterRegistries(butler._datastore):
155 registry.registerFormatter(storage, formatter)
157 return storage
160def _getAllFormatterRegistries(datastore: Datastore) -> list[FormatterFactory]:
161 """Return all formatter registries used by a datastore.
163 Parameters
164 ----------
165 datastore : `lsst.daf.butler.Datastore`
166 A datastore containing zero or more formatter registries.
168 Returns
169 -------
170 registries : `list` [`lsst.daf.butler.FormatterFactory`]
171 A possibly empty list of all formatter registries used
172 by ``datastore``.
173 """
174 try:
175 datastores = datastore.datastores # type: ignore[attr-defined]
176 except AttributeError:
177 datastores = [datastore]
179 registries = []
180 for datastore in datastores:
181 try:
182 # Not all datastores have a formatterFactory
183 formatterRegistry = datastore.formatterFactory # type: ignore[attr-defined]
184 except AttributeError:
185 pass # no formatter needed
186 else:
187 registries.append(formatterRegistry)
188 return registries
191class MetricsExample:
192 """Smorgasboard of information that might be the result of some
193 processing.
195 Parameters
196 ----------
197 summary : `dict`
198 Simple dictionary mapping key performance metrics to a scalar
199 result.
200 output : `dict`
201 Structured nested data.
202 data : `list`, optional
203 Arbitrary array data.
204 """
206 def __init__(
207 self,
208 summary: dict[str, Any] | None = None,
209 output: dict[str, Any] | None = None,
210 data: list[Any] | None = None,
211 ) -> None:
212 self.summary = summary
213 self.output = output
214 self.data = data
216 def __eq__(self, other: Any) -> bool:
217 try:
218 return self.summary == other.summary and self.output == other.output and self.data == other.data
219 except AttributeError:
220 pass
221 return NotImplemented
223 def __str__(self) -> str:
224 return str(self.exportAsDict())
226 def __repr__(self) -> str:
227 return f"MetricsExample({self.exportAsDict()})"
229 def exportAsDict(self) -> dict[str, list | dict | None]:
230 """Convert object contents to a single python dict."""
231 exportDict: dict[str, list | dict | None] = {"summary": self.summary, "output": self.output}
232 if self.data is not None:
233 exportDict["data"] = list(self.data)
234 else:
235 exportDict["data"] = None
236 return exportDict
238 def _asdict(self) -> dict[str, list | dict | None]:
239 """Convert object contents to a single Python dict.
241 This interface is used for JSON serialization.
243 Returns
244 -------
245 exportDict : `dict`
246 Object contents in the form of a dict with keys corresponding
247 to object attributes.
248 """
249 return self.exportAsDict()
251 @classmethod
252 def makeFromDict(cls, exportDict: dict[str, list | dict | None]) -> MetricsExample:
253 """Create a new object from a dict that is compatible with that
254 created by `exportAsDict`.
256 Parameters
257 ----------
258 exportDict : `dict`
259 `dict` with keys "summary", "output", and (optionally) "data".
261 Returns
262 -------
263 newobject : `MetricsExample`
264 New `MetricsExample` object.
265 """
266 data = exportDict["data"] if "data" in exportDict else None
267 assert isinstance(data, list | types.NoneType)
268 assert isinstance(exportDict["summary"], dict | types.NoneType)
269 assert isinstance(exportDict["output"], dict | types.NoneType)
270 return cls(exportDict["summary"], exportDict["output"], data)
273class MetricsExampleModel(BaseModel):
274 """A variant of `MetricsExample` based on model."""
276 summary: dict[str, Any] | None = None
277 output: dict[str, Any] | None = None
278 data: list[Any] | None = None
280 @classmethod
281 def from_metrics(cls, metrics: MetricsExample) -> MetricsExampleModel:
282 """Create a model based on an example."""
283 d = metrics.exportAsDict()
284 # Assume pydantic v2 but fallback to v1
285 try:
286 return cls.model_validate(d) # type: ignore
287 except AttributeError:
288 return cls.parse_obj(d)
291@dataclasses.dataclass
292class MetricsExampleDataclass:
293 """A variant of `MetricsExample` based on a dataclass."""
295 summary: dict[str, Any] | None
296 output: dict[str, Any] | None
297 data: list[Any] | None
300class ListDelegate(StorageClassDelegate):
301 """Parameter handler for list parameters."""
303 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any:
304 """Modify the in-memory dataset using the supplied parameters,
305 returning a possibly new object.
307 Parameters
308 ----------
309 inMemoryDataset : `object`
310 Object to modify based on the parameters.
311 parameters : `dict`
312 Parameters to apply. Values are specific to the parameter.
313 Supported parameters are defined in the associated
314 `StorageClass`. If no relevant parameters are specified the
315 inMemoryDataset will be return unchanged.
317 Returns
318 -------
319 inMemoryDataset : `object`
320 Updated form of supplied in-memory dataset, after parameters
321 have been used.
322 """
323 inMemoryDataset = copy.deepcopy(inMemoryDataset)
324 use = self.storageClass.filterParameters(parameters, subset={"slice"})
325 if use:
326 inMemoryDataset = inMemoryDataset[use["slice"]]
327 return inMemoryDataset
330class MetricsDelegate(StorageClassDelegate):
331 """Parameter handler for parameters using Metrics."""
333 def handleParameters(self, inMemoryDataset: Any, parameters: Mapping[str, Any] | None = None) -> Any:
334 """Modify the in-memory dataset using the supplied parameters,
335 returning a possibly new object.
337 Parameters
338 ----------
339 inMemoryDataset : `object`
340 Object to modify based on the parameters.
341 parameters : `dict`
342 Parameters to apply. Values are specific to the parameter.
343 Supported parameters are defined in the associated
344 `StorageClass`. If no relevant parameters are specified the
345 inMemoryDataset will be return unchanged.
347 Returns
348 -------
349 inMemoryDataset : `object`
350 Updated form of supplied in-memory dataset, after parameters
351 have been used.
352 """
353 inMemoryDataset = copy.deepcopy(inMemoryDataset)
354 use = self.storageClass.filterParameters(parameters, subset={"slice"})
355 if use:
356 inMemoryDataset.data = inMemoryDataset.data[use["slice"]]
357 return inMemoryDataset
359 def getComponent(self, composite: Any, componentName: str) -> Any:
360 if componentName == "counter":
361 return len(composite.data)
362 return super().getComponent(composite, componentName)
364 @classmethod
365 def selectResponsibleComponent(cls, readComponent: str, fromComponents: set[str | None]) -> str:
366 forwarderMap = {
367 "counter": "data",
368 }
369 forwarder = forwarderMap.get(readComponent)
370 if forwarder is not None and forwarder in fromComponents:
371 return forwarder
372 raise ValueError(f"Can not calculate read component {readComponent} from {fromComponents}")