Coverage for python/lsst/daf/butler/tests/testFormatters.py : 30%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("FormatterTest", "DoNothingFormatter", "LenientYamlFormatter", "MetricsExampleFormatter",
25 "MultipleExtensionsFormatter", "SingleExtensionFormatter")
27from typing import (
28 TYPE_CHECKING,
29 Any,
30 Mapping,
31 Optional,
32)
34import yaml
36from ..core import Formatter
37from ..formatters.yaml import YamlFormatter
39if TYPE_CHECKING: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true
40 from ..core import Location
43class DoNothingFormatter(Formatter):
44 """A test formatter that does not need to format anything and has
45 parameters."""
47 def read(self, component: Optional[str] = None) -> Any:
48 raise NotImplementedError("Type does not support reading")
50 def write(self, inMemoryDataset: Any) -> str:
51 raise NotImplementedError("Type does not support writing")
54class FormatterTest(Formatter):
55 """A test formatter that does not need to format anything."""
57 supportedWriteParameters = frozenset({"min", "max", "median", "comment", "extra", "recipe"})
59 def read(self, component: Optional[str] = None) -> Any:
60 raise NotImplementedError("Type does not support reading")
62 def write(self, inMemoryDataset: Any) -> str:
63 raise NotImplementedError("Type does not support writing")
65 @staticmethod
66 def validateWriteRecipes(recipes: Optional[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]:
67 if not recipes:
68 return recipes
69 for recipeName in recipes:
70 if "mode" not in recipes[recipeName]:
71 raise RuntimeError("'mode' is a required write recipe parameter")
72 return recipes
75class SingleExtensionFormatter(DoNothingFormatter):
76 """A do nothing formatter that has a single extension registered."""
77 extension = ".fits"
80class MultipleExtensionsFormatter(SingleExtensionFormatter):
81 """A formatter that has multiple extensions registered."""
82 supportedExtensions = frozenset({".fits.gz", ".fits.fz", ".fit"})
85class LenientYamlFormatter(YamlFormatter):
86 """A test formatter that allows any file extension but always reads and
87 writes YAML."""
88 extension = ".yaml"
90 @classmethod
91 def validateExtension(cls, location: Location) -> None:
92 return
95class MetricsExampleFormatter(Formatter):
96 """A specialist test formatter for metrics that supports components
97 directly without assembler delegate."""
99 extension = ".yaml"
100 """Always write YAML"""
102 def read(self, component=None):
103 """Read data from a file.
105 Parameters
106 ----------
107 component : `str`, optional
108 Component to read from the file. Only used if the `StorageClass`
109 for reading differed from the `StorageClass` used to write the
110 file.
112 Returns
113 -------
114 inMemoryDataset : `object`
115 The requested data as a Python object. The type of object
116 is controlled by the specific formatter.
118 Raises
119 ------
120 ValueError
121 Component requested but this file does not seem to be a concrete
122 composite.
123 KeyError
124 Raised when parameters passed with fileDescriptor are not
125 supported.
126 """
128 # This formatter can not read a subset from disk because it
129 # uses yaml.
130 path = self.fileDescriptor.location.path
131 with open(path, "r") as fd:
132 data = yaml.load(fd, Loader=yaml.SafeLoader)
134 # We can slice up front if required
135 parameters = self.fileDescriptor.parameters
136 if "data" in data and parameters and "slice" in parameters:
137 data["data"] = data["data"][parameters["slice"]]
139 pytype = self.fileDescriptor.storageClass.pytype
140 inMemoryDataset = pytype(**data)
142 if not component:
143 return inMemoryDataset
145 if component == "summary":
146 return inMemoryDataset.summary
147 elif component == "output":
148 return inMemoryDataset.output
149 elif component == "data":
150 return inMemoryDataset.data
151 elif component == "counter":
152 return len(inMemoryDataset.data)
153 raise ValueError(f"Unsupported component: {component}")
155 def write(self, inMemoryDataset: Any) -> str:
156 """Write a Dataset.
158 Parameters
159 ----------
160 inMemoryDataset : `object`
161 The Dataset to store.
163 Returns
164 -------
165 path : `str`
166 The path to where the Dataset was stored within the datastore.
167 """
168 fileDescriptor = self.fileDescriptor
170 # Update the location with the formatter-preferred file extension
171 fileDescriptor.location.updateExtension(self.extension)
173 with open(fileDescriptor.location.path, "w") as fd:
174 yaml.dump(inMemoryDataset._asdict(), fd)
175 return fileDescriptor.location.pathInStore
178class MetricsExampleDataFormatter(Formatter):
179 """A specialist test formatter for the data component of a MetricsExample.
181 This is needed if the MetricsExample is dissassembled and we want to
182 support the derived component.
183 """
185 unsupportedParameters = None
186 """Let the assembler delegate handle slice"""
188 extension = ".yaml"
189 """Always write YAML"""
191 def read(self, component=None):
192 """Read data from a file.
194 Parameters
195 ----------
196 component : `str`, optional
197 Component to read from the file. Only used if the `StorageClass`
198 for reading differed from the `StorageClass` used to write the
199 file.
201 Returns
202 -------
203 inMemoryDataset : `object`
204 The requested data as a Python object. The type of object
205 is controlled by the specific formatter.
207 Raises
208 ------
209 ValueError
210 Component requested but this file does not seem to be a concrete
211 composite.
212 KeyError
213 Raised when parameters passed with fileDescriptor are not
214 supported.
215 """
217 # This formatter can not read a subset from disk because it
218 # uses yaml.
219 path = self.fileDescriptor.location.path
220 with open(path, "r") as fd:
221 data = yaml.load(fd, Loader=yaml.SafeLoader)
223 # We can slice up front if required
224 parameters = self.fileDescriptor.parameters
225 if parameters and "slice" in parameters:
226 data = data[parameters["slice"]]
228 # This should be a native list
229 inMemoryDataset = data
231 if not component:
232 return inMemoryDataset
234 if component == "counter":
235 return len(inMemoryDataset)
236 raise ValueError(f"Unsupported component: {component}")
238 def write(self, inMemoryDataset: Any) -> str:
239 """Write a Dataset.
241 Parameters
242 ----------
243 inMemoryDataset : `object`
244 The Dataset to store.
246 Returns
247 -------
248 path : `str`
249 The path to where the Dataset was stored within the datastore.
250 """
251 fileDescriptor = self.fileDescriptor
253 # Update the location with the formatter-preferred file extension
254 fileDescriptor.location.updateExtension(self.extension)
256 with open(fileDescriptor.location.path, "w") as fd:
257 yaml.dump(inMemoryDataset, fd)
258 return fileDescriptor.location.pathInStore