Coverage for tests/test_formatter.py: 10%
121 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-12 10:56 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-12 10:56 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Tests related to the formatter infrastructure.
23"""
25import inspect
26import os.path
27import unittest
29from lsst.daf.butler import (
30 Config,
31 DataCoordinate,
32 DatasetType,
33 DimensionGraph,
34 DimensionUniverse,
35 FileDescriptor,
36 Formatter,
37 FormatterFactory,
38 Location,
39 StorageClass,
40)
41from lsst.daf.butler.tests import DatasetTestHelper
42from lsst.daf.butler.tests.testFormatters import (
43 DoNothingFormatter,
44 MultipleExtensionsFormatter,
45 SingleExtensionFormatter,
46)
48TESTDIR = os.path.abspath(os.path.dirname(__file__))
51class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper):
52 """Tests of the formatter factory infrastructure."""
54 def setUp(self):
55 self.id = 0
56 self.factory = FormatterFactory()
57 self.universe = DimensionUniverse()
58 self.dataId = DataCoordinate.makeEmpty(self.universe)
60 # Dummy FileDescriptor for testing getFormatter
61 self.fileDescriptor = FileDescriptor(
62 Location("/a/b/c", "d"), StorageClass("DummyStorageClass", dict, None)
63 )
65 def assertIsFormatter(self, formatter):
66 """Check that the supplied parameter is either a Formatter instance
67 or Formatter class.
68 """
69 if inspect.isclass(formatter):
70 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter")
71 else:
72 self.assertIsInstance(formatter, Formatter)
74 def testFormatter(self):
75 """Check basic parameter exceptions"""
76 f = DoNothingFormatter(self.fileDescriptor, self.dataId)
77 self.assertEqual(f.writeRecipes, {})
78 self.assertEqual(f.writeParameters, {})
79 self.assertIn("DoNothingFormatter", repr(f))
81 with self.assertRaises(TypeError):
82 DoNothingFormatter()
84 with self.assertRaises(ValueError):
85 DoNothingFormatter(self.fileDescriptor, self.dataId, writeParameters={"param1": 0})
87 with self.assertRaises(RuntimeError):
88 DoNothingFormatter(self.fileDescriptor, self.dataId, writeRecipes={"label": "value"})
90 with self.assertRaises(NotImplementedError):
91 f.makeUpdatedLocation(Location("a", "b"))
93 with self.assertRaises(NotImplementedError):
94 f.write("str")
96 def testExtensionValidation(self):
97 """Test extension validation"""
98 for file, single_ok, multi_ok in (
99 ("e.fits", True, True),
100 ("e.fit", False, True),
101 ("e.fits.fz", False, True),
102 ("e.txt", False, False),
103 ("e.1.4.fits", True, True),
104 ("e.3.fit", False, True),
105 ("e.1.4.fits.gz", False, True),
106 ):
107 loc = Location("/a/b/c", file)
109 for formatter, passes in (
110 (SingleExtensionFormatter, single_ok),
111 (MultipleExtensionsFormatter, multi_ok),
112 ):
113 if passes:
114 formatter.validateExtension(loc)
115 else:
116 with self.assertRaises(ValueError):
117 formatter.validateExtension(loc)
119 def testRegistry(self):
120 """Check that formatters can be stored in the registry."""
121 formatterTypeName = "lsst.daf.butler.tests.deferredFormatter.DeferredFormatter"
122 storageClassName = "Image"
123 self.factory.registerFormatter(storageClassName, formatterTypeName)
124 f = self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId)
125 self.assertIsFormatter(f)
126 self.assertEqual(f.name(), formatterTypeName)
127 self.assertIn(formatterTypeName, str(f))
128 self.assertIn(self.fileDescriptor.location.path, str(f))
130 fcls = self.factory.getFormatterClass(storageClassName)
131 self.assertIsFormatter(fcls)
132 # Defer the import so that we ensure that the infrastructure loaded
133 # it on demand previously
134 from lsst.daf.butler.tests.deferredFormatter import DeferredFormatter
136 self.assertEqual(type(f), DeferredFormatter)
138 with self.assertRaises(TypeError):
139 # Requires a constructor parameter
140 self.factory.getFormatter(storageClassName)
142 with self.assertRaises(KeyError):
143 self.factory.getFormatter("Missing", self.fileDescriptor)
145 # Check that a bad formatter path fails
146 storageClassName = "BadImage"
147 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.tests.deferredFormatter.Unknown")
148 with self.assertRaises(ImportError):
149 self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId)
151 def testRegistryWithStorageClass(self):
152 """Test that the registry can be given a StorageClass object."""
153 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter"
154 storageClassName = "TestClass"
155 sc = StorageClass(storageClassName, dict, None)
157 datasetType = DatasetType("calexp", self.universe.empty, sc)
159 # Store using an instance
160 self.factory.registerFormatter(sc, formatterTypeName)
162 # Retrieve using the class
163 f = self.factory.getFormatter(sc, self.fileDescriptor, self.dataId)
164 self.assertIsFormatter(f)
165 self.assertEqual(f.fileDescriptor, self.fileDescriptor)
167 # Retrieve using the DatasetType
168 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor, self.dataId)
169 self.assertIsFormatter(f2)
170 self.assertEqual(f.name(), f2.name())
172 # Class directly
173 f2cls = self.factory.getFormatterClass(datasetType)
174 self.assertIsFormatter(f2cls)
176 # This might defer the import, pytest may have already loaded it
177 from lsst.daf.butler.formatters.yaml import YamlFormatter
179 self.assertEqual(type(f), YamlFormatter)
181 with self.assertRaises(KeyError):
182 # Attempt to overwrite using a different value
183 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.formatters.json.JsonFormatter")
185 def testRegistryConfig(self):
186 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml")
187 config = Config(configFile)
188 self.factory.registerFormatters(config["datastore", "formatters"], universe=self.universe)
190 # Create a DatasetRef with and without instrument matching the
191 # one in the config file.
192 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
193 constant_dataId = {"physical_filter": "v", "visit": 1}
194 sc = StorageClass("DummySC", dict, None)
195 refPviHsc = self.makeDatasetRef(
196 "pvi",
197 dimensions,
198 sc,
199 {"instrument": "DummyHSC", **constant_dataId},
200 )
201 refPviHscFmt = self.factory.getFormatterClass(refPviHsc)
202 self.assertIsFormatter(refPviHscFmt)
203 self.assertIn("JsonFormatter", refPviHscFmt.name())
205 refPviNotHsc = self.makeDatasetRef(
206 "pvi",
207 dimensions,
208 sc,
209 {"instrument": "DummyNotHSC", **constant_dataId},
210 )
211 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc)
212 self.assertIsFormatter(refPviNotHscFmt)
213 self.assertIn("PickleFormatter", refPviNotHscFmt.name())
215 # Create a DatasetRef that should fall back to using Dimensions
216 refPvixHsc = self.makeDatasetRef(
217 "pvix",
218 dimensions,
219 sc,
220 {"instrument": "DummyHSC", **constant_dataId},
221 )
222 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc)
223 self.assertIsFormatter(refPvixNotHscFmt)
224 self.assertIn("PickleFormatter", refPvixNotHscFmt.name())
226 # Create a DatasetRef that should fall back to using StorageClass
227 dimensionsNoV = DimensionGraph(self.universe, names=("physical_filter", "instrument"))
228 refPvixNotHscDims = self.makeDatasetRef(
229 "pvix",
230 dimensionsNoV,
231 sc,
232 {"instrument": "DummyHSC", "physical_filter": "v"},
233 )
234 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims)
235 self.assertIsFormatter(refPvixNotHscDims_fmt)
236 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
238 # Check that parameters are stored
239 refParam = self.makeDatasetRef(
240 "paramtest",
241 dimensions,
242 sc,
243 {"instrument": "DummyNotHSC", **constant_dataId},
244 )
245 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam)
246 self.assertIn("writeParameters", kwargs)
247 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"}
248 self.assertEqual(kwargs["writeParameters"], expected)
249 self.assertIn("FormatterTest", refParam_fmt.name())
251 f = self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId)
252 self.assertEqual(f.writeParameters, expected)
254 f = self.factory.getFormatter(
255 refParam, self.fileDescriptor, self.dataId, writeParameters={"min": 22, "extra": 50}
256 )
257 self.assertEqual(
258 f.writeParameters,
259 {"max": 5, "min": 22, "comment": "Additional commentary", "extra": 50, "recipe": "recipe1"},
260 )
262 self.assertIn("recipe1", f.writeRecipes)
263 self.assertEqual(f.writeParameters["recipe"], "recipe1")
265 with self.assertRaises(ValueError):
266 # "new" is not allowed as a write parameter
267 self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId, writeParameters={"new": 1})
269 with self.assertRaises(RuntimeError):
270 # "mode" is a required recipe parameter
271 self.factory.getFormatter(
272 refParam, self.fileDescriptor, self.dataId, writeRecipes={"recipe3": {"notmode": 1}}
273 )
276if __name__ == "__main__":
277 unittest.main()