Coverage for tests/test_config.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
23import os
24import contextlib
25import collections
26import itertools
28from lsst.daf.butler import ConfigSubset, Config
31@contextlib.contextmanager
32def modified_environment(**environ):
33 """
34 Temporarily set environment variables.
36 >>> with modified_environment(DAF_BUTLER_DIR="/somewhere"):
37 ... os.environ["DAF_BUTLER_DIR"] == "/somewhere"
38 True
40 >>> "DAF_BUTLER_DIR" != "/somewhere"
41 True
43 Parameters
44 ----------
45 environ : `dict`
46 Key value pairs of environment variables to temporarily set.
47 """
48 old_environ = dict(os.environ)
49 os.environ.update(environ)
50 try:
51 yield
52 finally:
53 os.environ.clear()
54 os.environ.update(old_environ)
57class ExampleWithConfigFileReference:
58 defaultConfigFile = "viacls.yaml"
61class ExampleWithConfigFileReference2:
62 defaultConfigFile = "viacls2.yaml"
65class ConfigTest(ConfigSubset):
66 component = "comp"
67 requiredKeys = ("item1", "item2")
68 defaultConfigFile = "testconfig.yaml"
71class ConfigTestEmpty(ConfigTest):
72 defaultConfigFile = "testconfig_empty.yaml"
73 requiredKeys = ()
76class ConfigTestButlerDir(ConfigTest):
77 defaultConfigFile = "testConfigs/testconfig.yaml"
80class ConfigTestNoDefaults(ConfigTest):
81 defaultConfigFile = None
82 requiredKeys = ()
85class ConfigTestAbsPath(ConfigTest):
86 defaultConfigFile = None
87 requiredKeys = ()
90class ConfigTestCls(ConfigTest):
91 defaultConfigFile = "withcls.yaml"
94class ConfigTestCase(unittest.TestCase):
95 """Tests of simple Config"""
97 def testBadConfig(self):
98 for badArg in ([], "file.fits"):
99 with self.assertRaises(RuntimeError):
100 Config(badArg)
102 def testBasics(self):
103 c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}})
104 pretty = c.ppprint()
105 self.assertIn("key3", pretty)
106 r = repr(c)
107 self.assertIn("key3", r)
108 regex = r"^Config\(\{.*\}\)$"
109 self.assertRegex(r, regex)
110 c2 = eval(r)
111 self.assertIn("1", c)
112 for n in c.names():
113 self.assertEqual(c2[n], c[n])
114 self.assertEqual(c, c2)
115 s = str(c)
116 self.assertIn("\n", s)
117 self.assertNotRegex(s, regex)
119 self.assertCountEqual(c.keys(), ["1", "3", "key3", "dict"])
120 self.assertEqual(list(c), list(c.keys()))
121 self.assertEqual(list(c.values()), [c[k] for k in c.keys()])
122 self.assertEqual(list(c.items()), [(k, c[k]) for k in c.keys()])
124 newKeys = ("key4", ".dict.q", ("dict", "r"), "5")
125 oldKeys = ("key3", ".dict.a", ("dict", "b"), "3")
126 remainingKey = "1"
128 # Check get with existing key
129 for k in oldKeys:
130 self.assertEqual(c.get(k, "missing"), c[k])
132 # Check get, pop with nonexistent key
133 for k in newKeys:
134 self.assertEqual(c.get(k, "missing"), "missing")
135 self.assertEqual(c.pop(k, "missing"), "missing")
137 # Check setdefault with existing key
138 for k in oldKeys:
139 c.setdefault(k, 8)
140 self.assertNotEqual(c[k], 8)
142 # Check setdefault with nonexistent key (mutates c, adding newKeys)
143 for k in newKeys:
144 c.setdefault(k, 8)
145 self.assertEqual(c[k], 8)
147 # Check pop with existing key (mutates c, removing newKeys)
148 for k in newKeys:
149 v = c[k]
150 self.assertEqual(c.pop(k, "missing"), v)
152 # Check deletion (mutates c, removing oldKeys)
153 for k in ("key3", ".dict.a", ("dict", "b"), "3"):
154 self.assertIn(k, c)
155 del c[k]
156 self.assertNotIn(k, c)
158 # Check that `dict` still exists, but is now empty (then remove
159 # it, mutatic c)
160 self.assertIn("dict", c)
161 del c["dict"]
163 # Check popitem (mutates c, removing remainingKey)
164 v = c[remainingKey]
165 self.assertEqual(c.popitem(), (remainingKey, v))
167 # Check that c is now empty
168 self.assertFalse(c)
170 def assertSplit(self, answer, *args):
171 """Helper function to compare string splitting"""
172 for s in (answer, *args):
173 split = Config._splitIntoKeys(s)
174 self.assertEqual(split, answer)
176 def testSplitting(self):
177 """Test of the internal splitting API."""
178 # Try lots of keys that will return the same answer
179 answer = ["a", "b", "c", "d"]
180 self.assertSplit(answer, ".a.b.c.d", ":a:b:c:d", "\ta\tb\tc\td", "\ra\rb\rc\rd")
182 answer = ["a", "calexp.wcs", "b"]
183 self.assertSplit(answer, r".a.calexp\.wcs.b", ":a:calexp.wcs:b")
185 self.assertSplit(["a.b.c"])
186 self.assertSplit(["a", r"b\.c"], r"_a_b\.c")
188 # Escaping a backslash before a delimiter currently fails
189 with self.assertRaises(ValueError):
190 Config._splitIntoKeys(r".a.calexp\\.wcs.b")
192 # The next two fail because internally \r is magic when escaping
193 # a delimiter.
194 with self.assertRaises(ValueError):
195 Config._splitIntoKeys("\ra\rcalexp\\\rwcs\rb")
197 with self.assertRaises(ValueError):
198 Config._splitIntoKeys(".a.cal\rexp\\.wcs.b")
200 def testEscape(self):
201 c = Config({"a": {"foo.bar": 1}, "b😂c": {"bar_baz": 2}})
202 self.assertEqual(c[r".a.foo\.bar"], 1)
203 self.assertEqual(c[":a:foo.bar"], 1)
204 self.assertEqual(c[".b😂c.bar_baz"], 2)
205 self.assertEqual(c[r"😂b\😂c😂bar_baz"], 2)
206 self.assertEqual(c[r"\a\foo.bar"], 1)
207 self.assertEqual(c["\ra\rfoo.bar"], 1)
208 with self.assertRaises(ValueError):
209 c[".a.foo\\.bar\r"]
211 def testOperators(self):
212 c1 = Config({"a": {"b": 1}, "c": 2})
213 c2 = c1.copy()
214 self.assertEqual(c1, c2)
215 self.assertIsInstance(c2, Config)
216 c2[".a.b"] = 5
217 self.assertNotEqual(c1, c2)
219 def testUpdate(self):
220 c = Config({"a": {"b": 1}})
221 c.update({"a": {"c": 2}})
222 self.assertEqual(c[".a.b"], 1)
223 self.assertEqual(c[".a.c"], 2)
224 c.update({"a": {"d": [3, 4]}})
225 self.assertEqual(c[".a.d.0"], 3)
226 c.update({"z": [5, 6, {"g": 2, "h": 3}]})
227 self.assertEqual(c[".z.1"], 6)
229 # This is detached from parent
230 c2 = c[".z.2"]
231 self.assertEqual(c2["g"], 2)
232 c2.update({"h": 4, "j": 5})
233 self.assertEqual(c2["h"], 4)
234 self.assertNotIn(".z.2.j", c)
235 self.assertNotEqual(c[".z.2.h"], 4)
237 with self.assertRaises(RuntimeError):
238 c.update([1, 2, 3])
240 def testHierarchy(self):
241 c = Config()
243 # Simple dict
244 c["a"] = {"z": 52, "x": "string"}
245 self.assertIn(".a.z", c)
246 self.assertEqual(c[".a.x"], "string")
248 # Try different delimiters
249 self.assertEqual(c["⇛a⇛z"], 52)
250 self.assertEqual(c[("a", "z")], 52)
251 self.assertEqual(c["a", "z"], 52)
253 c[".b.new.thing1"] = "thing1"
254 c[".b.new.thing2"] = "thing2"
255 c[".b.new.thing3.supp"] = "supplemental"
256 self.assertEqual(c[".b.new.thing1"], "thing1")
257 tmp = c[".b.new"]
258 self.assertEqual(tmp["thing2"], "thing2")
259 self.assertEqual(c[".b.new.thing3.supp"], "supplemental")
261 # Test that we can index into lists
262 c[".a.b.c"] = [1, "7", 3, {"1": 4, "5": "Five"}, "hello"]
263 self.assertIn(".a.b.c.3.5", c)
264 self.assertNotIn(".a.b.c.10", c)
265 self.assertNotIn(".a.b.c.10.d", c)
266 self.assertEqual(c[".a.b.c.3.5"], "Five")
267 # Is the value in the list?
268 self.assertIn(".a.b.c.hello", c)
269 self.assertNotIn(".a.b.c.hello.not", c)
271 # And assign to an element in the list
272 self.assertEqual(c[".a.b.c.1"], "7")
273 c[".a.b.c.1"] = 8
274 self.assertEqual(c[".a.b.c.1"], 8)
275 self.assertIsInstance(c[".a.b.c"], collections.abc.Sequence)
277 # Test we do get lists back from asArray
278 a = c.asArray(".a.b.c")
279 self.assertIsInstance(a, list)
281 # Is it the *same* list as in the config
282 a.append("Sentinel")
283 self.assertIn("Sentinel", c[".a.b.c"])
284 self.assertIn(".a.b.c.Sentinel", c)
286 # Test we always get a list
287 for k in c.names():
288 a = c.asArray(k)
289 self.assertIsInstance(a, list)
291 # Check we get the same top level keys
292 self.assertEqual(set(c.names(topLevelOnly=True)), set(c._data.keys()))
294 # Check that we can iterate through items
295 for k, v in c.items():
296 self.assertEqual(c[k], v)
298 # Check that lists still work even if assigned a dict
299 c = Config({"cls": "lsst.daf.butler",
300 "formatters": {"calexp.wcs": "{component}",
301 "calexp": "{datasetType}"},
302 "datastores": [{"datastore": {"cls": "datastore1"}},
303 {"datastore": {"cls": "datastore2"}}]})
304 c[".datastores.1.datastore"] = {"cls": "datastore2modified"}
305 self.assertEqual(c[".datastores.0.datastore.cls"], "datastore1")
306 self.assertEqual(c[".datastores.1.datastore.cls"], "datastore2modified")
307 self.assertIsInstance(c["datastores"], collections.abc.Sequence)
309 # Test that we can get all the listed names.
310 # and also that they are marked as "in" the Config
311 # Try delimited names and tuples
312 for n in itertools.chain(c.names(), c.nameTuples()):
313 val = c[n]
314 self.assertIsNotNone(val)
315 self.assertIn(n, c)
317 names = c.names()
318 nameTuples = c.nameTuples()
319 self.assertEqual(len(names), len(nameTuples))
320 self.assertEqual(len(names), 11)
321 self.assertEqual(len(nameTuples), 11)
323 # Test that delimiter escaping works
324 names = c.names(delimiter=".")
325 for n in names:
326 self.assertIn(n, c)
327 self.assertIn(".formatters.calexp\\.wcs", names)
329 # Use a name that includes the internal default delimiter
330 # to test automatic adjustment of delimiter
331 strangeKey = f"calexp{c._D}wcs"
332 c["formatters", strangeKey] = "dynamic"
333 names = c.names()
334 self.assertIn(strangeKey, "-".join(names))
335 self.assertFalse(names[0].startswith(c._D))
336 for n in names:
337 self.assertIn(n, c)
339 top = c.nameTuples(topLevelOnly=True)
340 self.assertIsInstance(top[0], tuple)
342 # Investigate a possible delimeter in a key
343 c = Config({"formatters": {"calexp.wcs": 2, "calexp": 3}})
344 self.assertEqual(c[":formatters:calexp.wcs"], 2)
345 self.assertEqual(c[":formatters:calexp"], 3)
346 for k, v in c["formatters"].items():
347 self.assertEqual(c["formatters", k], v)
349 # Check internal delimiter inheritance
350 c._D = "."
351 c2 = c["formatters"]
352 self.assertEqual(c._D, c2._D) # Check that the child inherits
353 self.assertNotEqual(c2._D, Config._D)
356class ConfigSubsetTestCase(unittest.TestCase):
357 """Tests for ConfigSubset
358 """
360 def setUp(self):
361 self.testDir = os.path.abspath(os.path.dirname(__file__))
362 self.configDir = os.path.join(self.testDir, "config", "testConfigs")
363 self.configDir2 = os.path.join(self.testDir, "config", "testConfigs", "test2")
364 self.configDir3 = os.path.join(self.testDir, "config", "testConfigs", "test3")
366 def testEmpty(self):
367 """Ensure that we can read an empty file."""
368 c = ConfigTestEmpty(searchPaths=(self.configDir,))
369 self.assertIsInstance(c, ConfigSubset)
371 def testDefaults(self):
372 """Read of defaults"""
374 # Supply the search path explicitly
375 c = ConfigTest(searchPaths=(self.configDir,))
376 self.assertIsInstance(c, ConfigSubset)
377 self.assertIn("item3", c)
378 self.assertEqual(c["item3"], 3)
380 # Use environment
381 with modified_environment(DAF_BUTLER_CONFIG_PATH=self.configDir):
382 c = ConfigTest()
383 self.assertIsInstance(c, ConfigSubset)
384 self.assertEqual(c["item3"], 3)
386 # No default so this should fail
387 with self.assertRaises(KeyError):
388 c = ConfigTest()
390 def testButlerDir(self):
391 """Test that DAF_BUTLER_DIR is used to locate files."""
392 # with modified_environment(DAF_BUTLER_DIR=self.testDir):
393 # c = ConfigTestButlerDir()
394 # self.assertIn("item3", c)
396 # Again with a search path
397 with modified_environment(DAF_BUTLER_DIR=self.testDir,
398 DAF_BUTLER_CONFIG_PATH=self.configDir2):
399 c = ConfigTestButlerDir()
400 self.assertIn("item3", c)
401 self.assertEqual(c["item3"], "override")
402 self.assertEqual(c["item4"], "new")
404 def testExternalOverride(self):
405 """Ensure that external values win"""
406 c = ConfigTest({"item3": "newval"}, searchPaths=(self.configDir,))
407 self.assertIn("item3", c)
408 self.assertEqual(c["item3"], "newval")
410 def testSearchPaths(self):
411 """Two search paths"""
412 c = ConfigTest(searchPaths=(self.configDir2, self.configDir))
413 self.assertIsInstance(c, ConfigSubset)
414 self.assertIn("item3", c)
415 self.assertEqual(c["item3"], "override")
416 self.assertEqual(c["item4"], "new")
418 c = ConfigTest(searchPaths=(self.configDir, self.configDir2))
419 self.assertIsInstance(c, ConfigSubset)
420 self.assertIn("item3", c)
421 self.assertEqual(c["item3"], 3)
422 self.assertEqual(c["item4"], "new")
424 def testExternalHierarchy(self):
425 """Test that we can provide external config parameters in hierarchy"""
426 c = ConfigTest({"comp": {"item1": 6, "item2": "a", "a": "b",
427 "item3": 7}, "item4": 8})
428 self.assertIn("a", c)
429 self.assertEqual(c["a"], "b")
430 self.assertNotIn("item4", c)
432 def testNoDefaults(self):
433 """Ensure that defaults can be turned off."""
435 # Mandatory keys but no defaults
436 c = ConfigTest({"item1": "a", "item2": "b", "item6": 6})
437 self.assertEqual(len(c.filesRead), 0)
438 self.assertIn("item1", c)
439 self.assertEqual(c["item6"], 6)
441 c = ConfigTestNoDefaults()
442 self.assertEqual(len(c.filesRead), 0)
444 def testAbsPath(self):
445 """Read default config from an absolute path"""
446 # Force the path to be absolute in the class
447 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, "abspath.yaml")
448 c = ConfigTestAbsPath()
449 self.assertEqual(c["item11"], "eleventh")
450 self.assertEqual(len(c.filesRead), 1)
452 # Now specify the normal config file with an absolute path
453 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, ConfigTest.defaultConfigFile)
454 c = ConfigTestAbsPath()
455 self.assertEqual(c["item11"], 11)
456 self.assertEqual(len(c.filesRead), 1)
458 # and a search path that will also include the file
459 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,))
460 self.assertEqual(c["item11"], 11)
461 self.assertEqual(len(c.filesRead), 1)
463 # Same as above but this time with relative path and two search paths
464 # to ensure the count changes
465 ConfigTestAbsPath.defaultConfigFile = ConfigTest.defaultConfigFile
466 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,))
467 self.assertEqual(len(c.filesRead), 2)
469 # Reset the class
470 ConfigTestAbsPath.defaultConfigFile = None
472 def testClassDerived(self):
473 """Read config specified in class determined from config"""
474 c = ConfigTestCls(searchPaths=(self.configDir,))
475 self.assertEqual(c["item50"], 50)
476 self.assertEqual(c["help"], "derived")
478 # Same thing but additional search path
479 c = ConfigTestCls(searchPaths=(self.configDir, self.configDir2))
480 self.assertEqual(c["item50"], 50)
481 self.assertEqual(c["help"], "derived")
482 self.assertEqual(c["help2"], "second")
484 # Same thing but reverse the two paths
485 c = ConfigTestCls(searchPaths=(self.configDir2, self.configDir))
486 self.assertEqual(c["item50"], 500)
487 self.assertEqual(c["help"], "class")
488 self.assertEqual(c["help2"], "second")
489 self.assertEqual(c["help3"], "third")
491 def testInclude(self):
492 """Read a config that has an include directive"""
493 c = Config(os.path.join(self.configDir, "testinclude.yaml"))
494 self.assertEqual(c[".comp1.item1"], 58)
495 self.assertEqual(c[".comp2.comp.item1"], 1)
496 self.assertEqual(c[".comp3.1.comp.item1"], "posix")
497 self.assertEqual(c[".comp4.0.comp.item1"], "posix")
498 self.assertEqual(c[".comp4.1.comp.item1"], 1)
499 self.assertEqual(c[".comp5.comp6.comp.item1"], "posix")
501 # Test a specific name and then test that all
502 # returned names are "in" the config.
503 names = c.names()
504 self.assertIn(c._D.join(("", "comp3", "1", "comp", "item1")), names)
505 for n in names:
506 self.assertIn(n, c)
508 # Test that override delimiter works
509 delimiter = "-"
510 names = c.names(delimiter=delimiter)
511 self.assertIn(delimiter.join(("", "comp3", "1", "comp", "item1")), names)
513 def testIncludeConfigs(self):
514 """Test the special includeConfigs key for pulling in additional
515 files."""
516 c = Config(os.path.join(self.configDir, "configIncludes.yaml"))
517 self.assertEqual(c["comp", "item2"], "hello")
518 self.assertEqual(c["comp", "item50"], 5000)
519 self.assertEqual(c["comp", "item1"], "first")
520 self.assertEqual(c["comp", "item10"], "tenth")
521 self.assertEqual(c["comp", "item11"], "eleventh")
522 self.assertEqual(c["unrelated"], 1)
523 self.assertEqual(c["addon", "comp", "item1"], "posix")
524 self.assertEqual(c["addon", "comp", "item11"], -1)
525 self.assertEqual(c["addon", "comp", "item50"], 500)
527 # Now test with an environment variable in includeConfigs
528 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3):
529 c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
530 self.assertEqual(c["comp", "item2"], "hello")
531 self.assertEqual(c["comp", "item50"], 5000)
532 self.assertEqual(c["comp", "item1"], "first")
533 self.assertEqual(c["comp", "item10"], "tenth")
534 self.assertEqual(c["comp", "item11"], "eleventh")
535 self.assertEqual(c["unrelated"], 1)
536 self.assertEqual(c["addon", "comp", "item1"], "envvar")
537 self.assertEqual(c["addon", "comp", "item11"], -1)
538 self.assertEqual(c["addon", "comp", "item50"], 501)
540 # This will fail
541 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2):
542 with self.assertRaises(FileNotFoundError):
543 Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
546if __name__ == "__main__": 546 ↛ 547line 546 didn't jump to line 547, because the condition on line 546 was never true
547 unittest.main()