Coverage for tests/test_formatter.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Tests related to the formatter infrastructure.
23"""
25import inspect
26import os.path
27import unittest
29from lsst.daf.butler.tests import DatasetTestHelper
30from lsst.daf.butler import (Formatter, FormatterFactory, StorageClass, DatasetType, Config,
31 FileDescriptor, Location, DimensionUniverse, DimensionGraph)
32from lsst.daf.butler.tests.testFormatters import (DoNothingFormatter, MultipleExtensionsFormatter,
33 SingleExtensionFormatter)
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper):
39 """Tests of the formatter factory infrastructure.
40 """
42 def setUp(self):
43 self.id = 0
44 self.factory = FormatterFactory()
46 # Dummy FileDescriptor for testing getFormatter
47 self.fileDescriptor = FileDescriptor(Location("/a/b/c", "d"),
48 StorageClass("DummyStorageClass", dict, None))
50 def assertIsFormatter(self, formatter):
51 """Check that the supplied parameter is either a Formatter instance
52 or Formatter class."""
54 if inspect.isclass(formatter):
55 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter")
56 else:
57 self.assertIsInstance(formatter, Formatter)
59 def testFormatter(self):
60 """Check basic parameter exceptions"""
61 f = DoNothingFormatter(self.fileDescriptor)
62 self.assertEqual(f.writeRecipes, {})
63 self.assertEqual(f.writeParameters, {})
64 self.assertIn("DoNothingFormatter", repr(f))
66 with self.assertRaises(TypeError):
67 DoNothingFormatter()
69 with self.assertRaises(ValueError):
70 DoNothingFormatter(self.fileDescriptor, writeParameters={"param1": 0})
72 with self.assertRaises(RuntimeError):
73 DoNothingFormatter(self.fileDescriptor, writeRecipes={"label": "value"})
75 with self.assertRaises(NotImplementedError):
76 f.makeUpdatedLocation(Location("a", "b"))
78 with self.assertRaises(NotImplementedError):
79 f.write("str")
81 def testExtensionValidation(self):
82 """Test extension validation"""
84 for file, single_ok, multi_ok in (("e.fits", True, True),
85 ("e.fit", False, True),
86 ("e.fits.fz", False, True),
87 ("e.txt", False, False),
88 ("e.1.4.fits", True, True),
89 ("e.3.fit", False, True),
90 ("e.1.4.fits.gz", False, True),
91 ):
92 loc = Location("/a/b/c", file)
94 for formatter, passes in ((SingleExtensionFormatter, single_ok),
95 (MultipleExtensionsFormatter, multi_ok)):
96 if passes:
97 formatter.validateExtension(loc)
98 else:
99 with self.assertRaises(ValueError):
100 formatter.validateExtension(loc)
102 def testRegistry(self):
103 """Check that formatters can be stored in the registry.
104 """
105 formatterTypeName = "lsst.daf.butler.tests.deferredFormatter.DeferredFormatter"
106 storageClassName = "Image"
107 self.factory.registerFormatter(storageClassName, formatterTypeName)
108 f = self.factory.getFormatter(storageClassName, self.fileDescriptor)
109 self.assertIsFormatter(f)
110 self.assertEqual(f.name(), formatterTypeName)
111 self.assertIn(formatterTypeName, str(f))
112 self.assertIn(self.fileDescriptor.location.path, str(f))
114 fcls = self.factory.getFormatterClass(storageClassName)
115 self.assertIsFormatter(fcls)
116 # Defer the import so that we ensure that the infrastructure loaded
117 # it on demand previously
118 from lsst.daf.butler.tests.deferredFormatter import DeferredFormatter
119 self.assertEqual(type(f), DeferredFormatter)
121 with self.assertRaises(TypeError):
122 # Requires a constructor parameter
123 self.factory.getFormatter(storageClassName)
125 with self.assertRaises(KeyError):
126 self.factory.getFormatter("Missing", self.fileDescriptor)
128 # Check that a bad formatter path fails
129 storageClassName = "BadImage"
130 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.tests.deferredFormatter.Unknown")
131 with self.assertRaises(ImportError):
132 self.factory.getFormatter(storageClassName, self.fileDescriptor)
134 def testRegistryWithStorageClass(self):
135 """Test that the registry can be given a StorageClass object.
136 """
137 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter"
138 storageClassName = "TestClass"
139 sc = StorageClass(storageClassName, dict, None)
141 universe = DimensionUniverse()
142 datasetType = DatasetType("calexp", universe.empty, sc)
144 # Store using an instance
145 self.factory.registerFormatter(sc, formatterTypeName)
147 # Retrieve using the class
148 f = self.factory.getFormatter(sc, self.fileDescriptor)
149 self.assertIsFormatter(f)
150 self.assertEqual(f.fileDescriptor, self.fileDescriptor)
152 # Retrieve using the DatasetType
153 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor)
154 self.assertIsFormatter(f2)
155 self.assertEqual(f.name(), f2.name())
157 # Class directly
158 f2cls = self.factory.getFormatterClass(datasetType)
159 self.assertIsFormatter(f2cls)
161 # This might defer the import, pytest may have already loaded it
162 from lsst.daf.butler.formatters.yaml import YamlFormatter
163 self.assertEqual(type(f), YamlFormatter)
165 with self.assertRaises(KeyError):
166 # Attempt to overwrite using a different value
167 self.factory.registerFormatter(storageClassName,
168 "lsst.daf.butler.formatters.json.JsonFormatter")
170 def testRegistryConfig(self):
171 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml")
172 config = Config(configFile)
173 universe = DimensionUniverse()
174 self.factory.registerFormatters(config["datastore", "formatters"], universe=universe)
176 # Create a DatasetRef with and without instrument matching the
177 # one in the config file.
178 dimensions = universe.extract(("visit", "physical_filter", "instrument"))
179 sc = StorageClass("DummySC", dict, None)
180 refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC",
181 "physical_filter": "v"},
182 conform=False)
183 refPviHscFmt = self.factory.getFormatterClass(refPviHsc)
184 self.assertIsFormatter(refPviHscFmt)
185 self.assertIn("JsonFormatter", refPviHscFmt.name())
187 refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC",
188 "physical_filter": "v"},
189 conform=False)
190 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc)
191 self.assertIsFormatter(refPviNotHscFmt)
192 self.assertIn("PickleFormatter", refPviNotHscFmt.name())
194 # Create a DatasetRef that should fall back to using Dimensions
195 refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC",
196 "physical_filter": "v"},
197 conform=False)
198 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc)
199 self.assertIsFormatter(refPvixNotHscFmt)
200 self.assertIn("PickleFormatter", refPvixNotHscFmt.name())
202 # Create a DatasetRef that should fall back to using StorageClass
203 dimensionsNoV = DimensionGraph(universe, names=("physical_filter", "instrument"))
204 refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC",
205 "physical_filter": "v"},
206 conform=False)
207 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims)
208 self.assertIsFormatter(refPvixNotHscDims_fmt)
209 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
211 # Check that parameters are stored
212 refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC",
213 "physical_filter": "v"},
214 conform=False)
215 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam)
216 self.assertIn("writeParameters", kwargs)
217 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"}
218 self.assertEqual(kwargs["writeParameters"], expected)
219 self.assertIn("FormatterTest", refParam_fmt.name())
221 f = self.factory.getFormatter(refParam, self.fileDescriptor)
222 self.assertEqual(f.writeParameters, expected)
224 f = self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"min": 22,
225 "extra": 50})
226 self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary",
227 "extra": 50, "recipe": "recipe1"})
229 self.assertIn("recipe1", f.writeRecipes)
230 self.assertEqual(f.writeParameters["recipe"], "recipe1")
232 with self.assertRaises(ValueError):
233 # "new" is not allowed as a write parameter
234 self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"new": 1})
236 with self.assertRaises(RuntimeError):
237 # "mode" is a required recipe parameter
238 self.factory.getFormatter(refParam, self.fileDescriptor, writeRecipes={"recipe3": {"notmode": 1}})
241if __name__ == "__main__": 241 ↛ 242line 241 didn't jump to line 242, because the condition on line 241 was never true
242 unittest.main()