Coverage for python/lsst/daf/butler/tests/testFormatters.py: 30%
89 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-12 10:56 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "FormatterTest",
26 "DoNothingFormatter",
27 "LenientYamlFormatter",
28 "MetricsExampleFormatter",
29 "MultipleExtensionsFormatter",
30 "SingleExtensionFormatter",
31)
33import json
34from collections.abc import Mapping
35from typing import TYPE_CHECKING, Any
37import yaml
39from ..core import Formatter
40from ..formatters.yaml import YamlFormatter
42if TYPE_CHECKING:
43 from ..core import Location
46class DoNothingFormatter(Formatter):
47 """A test formatter that does not need to format anything and has
48 parameters.
49 """
51 def read(self, component: str | None = None) -> Any:
52 raise NotImplementedError("Type does not support reading")
54 def write(self, inMemoryDataset: Any) -> None:
55 raise NotImplementedError("Type does not support writing")
58class FormatterTest(Formatter):
59 """A test formatter that does not need to format anything."""
61 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"})
63 def read(self, component: str | None = None) -> Any:
64 raise NotImplementedError("Type does not support reading")
66 def write(self, inMemoryDataset: Any) -> None:
67 raise NotImplementedError("Type does not support writing")
69 @staticmethod
70 def validateWriteRecipes(recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None:
71 if not recipes:
72 return recipes
73 for recipeName in recipes:
74 if "mode" not in recipes[recipeName]:
75 raise RuntimeError("'mode' is a required write recipe parameter")
76 return recipes
79class SingleExtensionFormatter(DoNothingFormatter):
80 """A do nothing formatter that has a single extension registered."""
82 extension = ".fits"
85class MultipleExtensionsFormatter(SingleExtensionFormatter):
86 """A formatter that has multiple extensions registered."""
88 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"})
91class LenientYamlFormatter(YamlFormatter):
92 """A test formatter that allows any file extension but always reads and
93 writes YAML.
94 """
96 extension = ".yaml"
98 @classmethod
99 def validateExtension(cls, location: Location) -> None:
100 return
103class MetricsExampleFormatter(Formatter):
104 """A specialist test formatter for metrics that supports components
105 directly without assembler delegate.
106 """
108 supportedExtensions = frozenset({".yaml", ".json"})
110 @property
111 def extension(self) -> str:
112 """Always write yaml by default."""
113 return ".yaml"
115 def read(self, component: str | None = None) -> Any:
116 """Read data from a file.
118 Parameters
119 ----------
120 component : `str`, optional
121 Component to read from the file. Only used if the `StorageClass`
122 for reading differed from the `StorageClass` used to write the
123 file.
125 Returns
126 -------
127 inMemoryDataset : `object`
128 The requested data as a Python object. The type of object
129 is controlled by the specific formatter.
131 Raises
132 ------
133 ValueError
134 Component requested but this file does not seem to be a concrete
135 composite.
136 KeyError
137 Raised when parameters passed with fileDescriptor are not
138 supported.
139 """
140 # This formatter can not read a subset from disk because it
141 # uses yaml or json.
142 path = self.fileDescriptor.location.path
144 with open(path) as fd:
145 if path.endswith(".yaml"):
146 data = yaml.load(fd, Loader=yaml.SafeLoader)
147 elif path.endswith(".json"):
148 data = json.load(fd)
149 else:
150 raise RuntimeError(f"Unsupported file extension found in path '{path}'")
152 # We can slice up front if required
153 parameters = self.fileDescriptor.parameters
154 if "data" in data and parameters and "slice" in parameters:
155 data["data"] = data["data"][parameters["slice"]]
157 pytype = self.fileDescriptor.storageClass.pytype
158 inMemoryDataset = pytype(**data)
160 if not component:
161 return inMemoryDataset
163 if component == "summary":
164 return inMemoryDataset.summary
165 elif component == "output":
166 return inMemoryDataset.output
167 elif component == "data":
168 return inMemoryDataset.data
169 elif component == "counter":
170 return len(inMemoryDataset.data)
171 raise ValueError(f"Unsupported component: {component}")
173 def write(self, inMemoryDataset: Any) -> None:
174 """Write a Dataset.
176 Parameters
177 ----------
178 inMemoryDataset : `object`
179 The Dataset to store.
181 Returns
182 -------
183 path : `str`
184 The path to where the Dataset was stored within the datastore.
185 """
186 fileDescriptor = self.fileDescriptor
188 # Update the location with the formatter-preferred file extension
189 fileDescriptor.location.updateExtension(self.extension)
191 with open(fileDescriptor.location.path, "w") as fd:
192 yaml.dump(inMemoryDataset._asdict(), fd)
195class MetricsExampleDataFormatter(Formatter):
196 """A specialist test formatter for the data component of a MetricsExample.
198 This is needed if the MetricsExample is disassembled and we want to
199 support the derived component.
200 """
202 unsupportedParameters = None
203 """Let the assembler delegate handle slice"""
205 extension = ".yaml"
206 """Always write YAML"""
208 def read(self, component: str | None = None) -> Any:
209 """Read data from a file.
211 Parameters
212 ----------
213 component : `str`, optional
214 Component to read from the file. Only used if the `StorageClass`
215 for reading differed from the `StorageClass` used to write the
216 file.
218 Returns
219 -------
220 inMemoryDataset : `object`
221 The requested data as a Python object. The type of object
222 is controlled by the specific formatter.
224 Raises
225 ------
226 ValueError
227 Component requested but this file does not seem to be a concrete
228 composite.
229 KeyError
230 Raised when parameters passed with fileDescriptor are not
231 supported.
232 """
233 # This formatter can not read a subset from disk because it
234 # uses yaml.
235 path = self.fileDescriptor.location.path
236 with open(path) as fd:
237 data = yaml.load(fd, Loader=yaml.SafeLoader)
239 # We can slice up front if required
240 parameters = self.fileDescriptor.parameters
241 if parameters and "slice" in parameters:
242 data = data[parameters["slice"]]
244 # This should be a native list
245 inMemoryDataset = data
247 if not component:
248 return inMemoryDataset
250 if component == "counter":
251 return len(inMemoryDataset)
252 raise ValueError(f"Unsupported component: {component}")
254 def write(self, inMemoryDataset: Any) -> None:
255 """Write a Dataset.
257 Parameters
258 ----------
259 inMemoryDataset : `object`
260 The Dataset to store.
262 Returns
263 -------
264 path : `str`
265 The path to where the Dataset was stored within the datastore.
266 """
267 fileDescriptor = self.fileDescriptor
269 # Update the location with the formatter-preferred file extension
270 fileDescriptor.location.updateExtension(self.extension)
272 with open(fileDescriptor.location.path, "w") as fd:
273 yaml.dump(inMemoryDataset, fd)