Coverage for python/lsst/daf/butler/tests/testFormatters.py : 26%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("FormatterTest", "DoNothingFormatter", "LenientYamlFormatter", "MetricsExampleFormatter")
26from typing import (
27 TYPE_CHECKING,
28 Any,
29 Mapping,
30 Optional,
31)
33import yaml
35from ..core import Formatter
36from ..formatters.yaml import YamlFormatter
38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 from ..core import Location
42class DoNothingFormatter(Formatter):
43 """A test formatter that does not need to format anything and has
44 parameters."""
46 def read(self, component: Optional[str] = None) -> Any:
47 raise NotImplementedError("Type does not support reading")
49 def write(self, inMemoryDataset: Any) -> str:
50 raise NotImplementedError("Type does not support writing")
53class FormatterTest(Formatter):
54 """A test formatter that does not need to format anything."""
56 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"})
58 def read(self, component: Optional[str] = None) -> Any:
59 raise NotImplementedError("Type does not support reading")
61 def write(self, inMemoryDataset: Any) -> str:
62 raise NotImplementedError("Type does not support writing")
64 @staticmethod
65 def validateWriteRecipes(recipes: Optional[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]:
66 if not recipes:
67 return recipes
68 for recipeName in recipes:
69 if "mode" not in recipes[recipeName]:
70 raise RuntimeError("'mode' is a required write recipe parameter")
71 return recipes
74class LenientYamlFormatter(YamlFormatter):
75 """A test formatter that allows any file extension but always reads and
76 writes YAML."""
77 extension = ".yaml"
79 @classmethod
80 def validateExtension(cls, location: Location) -> None:
81 return
84class MetricsExampleFormatter(Formatter):
85 """A specialist test formatter for metrics that supports components
86 directly without assembler."""
88 extension = ".yaml"
89 """Always write YAML"""
91 def read(self, component=None):
92 """Read data from a file.
94 Parameters
95 ----------
96 component : `str`, optional
97 Component to read from the file. Only used if the `StorageClass`
98 for reading differed from the `StorageClass` used to write the
99 file.
101 Returns
102 -------
103 inMemoryDataset : `object`
104 The requested data as a Python object. The type of object
105 is controlled by the specific formatter.
107 Raises
108 ------
109 ValueError
110 Component requested but this file does not seem to be a concrete
111 composite.
112 KeyError
113 Raised when parameters passed with fileDescriptor are not
114 supported.
115 """
117 # This formatter can not read a subset from disk because it
118 # uses yaml.
119 path = self.fileDescriptor.location.path
120 with open(path, "r") as fd:
121 data = yaml.load(fd, Loader=yaml.SafeLoader)
123 # We can slice up front if required
124 parameters = self.fileDescriptor.parameters
125 if "data" in data and parameters and "slice" in parameters:
126 data["data"] = data["data"][parameters["slice"]]
128 pytype = self.fileDescriptor.storageClass.pytype
129 inMemoryDataset = pytype(**data)
131 if not component:
132 return inMemoryDataset
134 if component == "summary":
135 return inMemoryDataset.summary
136 elif component == "output":
137 return inMemoryDataset.output
138 elif component == "data":
139 return inMemoryDataset.data
140 elif component == "counter":
141 return len(inMemoryDataset.data)
142 raise ValueError(f"Unsupported component: {component}")
144 def write(self, inMemoryDataset: Any) -> str:
145 """Write a Dataset.
147 Parameters
148 ----------
149 inMemoryDataset : `object`
150 The Dataset to store.
152 Returns
153 -------
154 path : `str`
155 The path to where the Dataset was stored within the datastore.
156 """
157 fileDescriptor = self.fileDescriptor
159 # Update the location with the formatter-preferred file extension
160 fileDescriptor.location.updateExtension(self.extension)
162 with open(fileDescriptor.location.path, "w") as fd:
163 yaml.dump(inMemoryDataset._asdict(), fd)
164 return fileDescriptor.location.pathInStore
167class MetricsExampleDataFormatter(Formatter):
168 """A specialist test formatter for the data component of a MetricsExample.
170 This is needed if the MetricsExample is dissassembled and we want to
171 support the read-only component.
172 """
174 unsupportedParameters = None
175 """Let the assembler handle slice"""
177 extension = ".yaml"
178 """Always write YAML"""
180 def read(self, component=None):
181 """Read data from a file.
183 Parameters
184 ----------
185 component : `str`, optional
186 Component to read from the file. Only used if the `StorageClass`
187 for reading differed from the `StorageClass` used to write the
188 file.
190 Returns
191 -------
192 inMemoryDataset : `object`
193 The requested data as a Python object. The type of object
194 is controlled by the specific formatter.
196 Raises
197 ------
198 ValueError
199 Component requested but this file does not seem to be a concrete
200 composite.
201 KeyError
202 Raised when parameters passed with fileDescriptor are not
203 supported.
204 """
206 # This formatter can not read a subset from disk because it
207 # uses yaml.
208 path = self.fileDescriptor.location.path
209 with open(path, "r") as fd:
210 data = yaml.load(fd, Loader=yaml.SafeLoader)
212 # We can slice up front if required
213 parameters = self.fileDescriptor.parameters
214 if parameters and "slice" in parameters:
215 data = data[parameters["slice"]]
217 # This should be a native list
218 inMemoryDataset = data
220 if not component:
221 return inMemoryDataset
223 if component == "counter":
224 return len(inMemoryDataset)
225 raise ValueError(f"Unsupported component: {component}")
227 def write(self, inMemoryDataset: Any) -> str:
228 """Write a Dataset.
230 Parameters
231 ----------
232 inMemoryDataset : `object`
233 The Dataset to store.
235 Returns
236 -------
237 path : `str`
238 The path to where the Dataset was stored within the datastore.
239 """
240 fileDescriptor = self.fileDescriptor
242 # Update the location with the formatter-preferred file extension
243 fileDescriptor.location.updateExtension(self.extension)
245 with open(fileDescriptor.location.path, "w") as fd:
246 yaml.dump(inMemoryDataset, fd)
247 return fileDescriptor.location.pathInStore