Coverage for python/lsst/daf/butler/tests/testFormatters.py: 34%
89 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-15 02:03 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-15 02:03 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = (
31 "FormatterTest",
32 "DoNothingFormatter",
33 "LenientYamlFormatter",
34 "MetricsExampleFormatter",
35 "MultipleExtensionsFormatter",
36 "SingleExtensionFormatter",
37)
39import json
40from collections.abc import Mapping
41from typing import TYPE_CHECKING, Any
43import yaml
45from .._formatter import Formatter
46from ..formatters.yaml import YamlFormatter
48if TYPE_CHECKING:
49 from .._location import Location
52class DoNothingFormatter(Formatter):
53 """A test formatter that does not need to format anything and has
54 parameters.
55 """
57 def read(self, component: str | None = None) -> Any:
58 raise NotImplementedError("Type does not support reading")
60 def write(self, inMemoryDataset: Any) -> None:
61 raise NotImplementedError("Type does not support writing")
64class FormatterTest(Formatter):
65 """A test formatter that does not need to format anything."""
67 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"})
69 def read(self, component: str | None = None) -> Any:
70 raise NotImplementedError("Type does not support reading")
72 def write(self, inMemoryDataset: Any) -> None:
73 raise NotImplementedError("Type does not support writing")
75 @staticmethod
76 def validateWriteRecipes(recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None:
77 if not recipes:
78 return recipes
79 for recipeName in recipes:
80 if "mode" not in recipes[recipeName]:
81 raise RuntimeError("'mode' is a required write recipe parameter")
82 return recipes
85class SingleExtensionFormatter(DoNothingFormatter):
86 """A do nothing formatter that has a single extension registered."""
88 extension = ".fits"
91class MultipleExtensionsFormatter(SingleExtensionFormatter):
92 """A formatter that has multiple extensions registered."""
94 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"})
97class LenientYamlFormatter(YamlFormatter):
98 """A test formatter that allows any file extension but always reads and
99 writes YAML.
100 """
102 extension = ".yaml"
104 @classmethod
105 def validateExtension(cls, location: Location) -> None:
106 return
109class MetricsExampleFormatter(Formatter):
110 """A specialist test formatter for metrics that supports components
111 directly without assembler delegate.
112 """
114 supportedExtensions = frozenset({".yaml", ".json"})
116 @property
117 def extension(self) -> str:
118 """Always write yaml by default."""
119 return ".yaml"
121 def read(self, component: str | None = None) -> Any:
122 """Read data from a file.
124 Parameters
125 ----------
126 component : `str`, optional
127 Component to read from the file. Only used if the `StorageClass`
128 for reading differed from the `StorageClass` used to write the
129 file.
131 Returns
132 -------
133 inMemoryDataset : `object`
134 The requested data as a Python object. The type of object
135 is controlled by the specific formatter.
137 Raises
138 ------
139 ValueError
140 Component requested but this file does not seem to be a concrete
141 composite.
142 KeyError
143 Raised when parameters passed with fileDescriptor are not
144 supported.
145 """
146 # This formatter can not read a subset from disk because it
147 # uses yaml or json.
148 path = self.fileDescriptor.location.path
150 with open(path) as fd:
151 if path.endswith(".yaml"):
152 data = yaml.load(fd, Loader=yaml.SafeLoader)
153 elif path.endswith(".json"):
154 data = json.load(fd)
155 else:
156 raise RuntimeError(f"Unsupported file extension found in path '{path}'")
158 # We can slice up front if required
159 parameters = self.fileDescriptor.parameters
160 if "data" in data and parameters and "slice" in parameters:
161 data["data"] = data["data"][parameters["slice"]]
163 pytype = self.fileDescriptor.storageClass.pytype
164 inMemoryDataset = pytype(**data)
166 if not component:
167 return inMemoryDataset
169 if component == "summary":
170 return inMemoryDataset.summary
171 elif component == "output":
172 return inMemoryDataset.output
173 elif component == "data":
174 return inMemoryDataset.data
175 elif component == "counter":
176 return len(inMemoryDataset.data)
177 raise ValueError(f"Unsupported component: {component}")
179 def write(self, inMemoryDataset: Any) -> None:
180 """Write a Dataset.
182 Parameters
183 ----------
184 inMemoryDataset : `object`
185 The Dataset to store.
187 Returns
188 -------
189 path : `str`
190 The path to where the Dataset was stored within the datastore.
191 """
192 fileDescriptor = self.fileDescriptor
194 # Update the location with the formatter-preferred file extension
195 fileDescriptor.location.updateExtension(self.extension)
197 with open(fileDescriptor.location.path, "w") as fd:
198 yaml.dump(inMemoryDataset._asdict(), fd)
201class MetricsExampleDataFormatter(Formatter):
202 """A specialist test formatter for the data component of a MetricsExample.
204 This is needed if the MetricsExample is disassembled and we want to
205 support the derived component.
206 """
208 unsupportedParameters = None
209 """Let the assembler delegate handle slice"""
211 extension = ".yaml"
212 """Always write YAML"""
214 def read(self, component: str | None = None) -> Any:
215 """Read data from a file.
217 Parameters
218 ----------
219 component : `str`, optional
220 Component to read from the file. Only used if the `StorageClass`
221 for reading differed from the `StorageClass` used to write the
222 file.
224 Returns
225 -------
226 inMemoryDataset : `object`
227 The requested data as a Python object. The type of object
228 is controlled by the specific formatter.
230 Raises
231 ------
232 ValueError
233 Component requested but this file does not seem to be a concrete
234 composite.
235 KeyError
236 Raised when parameters passed with fileDescriptor are not
237 supported.
238 """
239 # This formatter can not read a subset from disk because it
240 # uses yaml.
241 path = self.fileDescriptor.location.path
242 with open(path) as fd:
243 data = yaml.load(fd, Loader=yaml.SafeLoader)
245 # We can slice up front if required
246 parameters = self.fileDescriptor.parameters
247 if parameters and "slice" in parameters:
248 data = data[parameters["slice"]]
250 # This should be a native list
251 inMemoryDataset = data
253 if not component:
254 return inMemoryDataset
256 if component == "counter":
257 return len(inMemoryDataset)
258 raise ValueError(f"Unsupported component: {component}")
260 def write(self, inMemoryDataset: Any) -> None:
261 """Write a Dataset.
263 Parameters
264 ----------
265 inMemoryDataset : `object`
266 The Dataset to store.
268 Returns
269 -------
270 path : `str`
271 The path to where the Dataset was stored within the datastore.
272 """
273 fileDescriptor = self.fileDescriptor
275 # Update the location with the formatter-preferred file extension
276 fileDescriptor.location.updateExtension(self.extension)
278 with open(fileDescriptor.location.path, "w") as fd:
279 yaml.dump(inMemoryDataset, fd)