Coverage for tests/test_config.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
23import os
24import contextlib
25import collections
26import itertools
27import shutil
28import tempfile
30from lsst.daf.butler import ConfigSubset, Config
33@contextlib.contextmanager
34def modified_environment(**environ):
35 """
36 Temporarily set environment variables.
38 >>> with modified_environment(DAF_BUTLER_DIR="/somewhere"):
39 ... os.environ["DAF_BUTLER_DIR"] == "/somewhere"
40 True
42 >>> "DAF_BUTLER_DIR" != "/somewhere"
43 True
45 Parameters
46 ----------
47 environ : `dict`
48 Key value pairs of environment variables to temporarily set.
49 """
50 old_environ = dict(os.environ)
51 os.environ.update(environ)
52 try:
53 yield
54 finally:
55 os.environ.clear()
56 os.environ.update(old_environ)
59class ExampleWithConfigFileReference:
60 defaultConfigFile = "viacls.yaml"
63class ExampleWithConfigFileReference2:
64 defaultConfigFile = "viacls2.yaml"
67class ConfigTest(ConfigSubset):
68 component = "comp"
69 requiredKeys = ("item1", "item2")
70 defaultConfigFile = "testconfig.yaml"
73class ConfigTestEmpty(ConfigTest):
74 defaultConfigFile = "testconfig_empty.yaml"
75 requiredKeys = ()
78class ConfigTestButlerDir(ConfigTest):
79 defaultConfigFile = "testConfigs/testconfig.yaml"
82class ConfigTestNoDefaults(ConfigTest):
83 defaultConfigFile = None
84 requiredKeys = ()
87class ConfigTestAbsPath(ConfigTest):
88 defaultConfigFile = None
89 requiredKeys = ()
92class ConfigTestCls(ConfigTest):
93 defaultConfigFile = "withcls.yaml"
96class ConfigTestCase(unittest.TestCase):
97 """Tests of simple Config"""
99 def testBadConfig(self):
100 for badArg in ([], "file.fits"):
101 with self.assertRaises(RuntimeError):
102 Config(badArg)
104 def testBasics(self):
105 c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}})
106 pretty = c.ppprint()
107 self.assertIn("key3", pretty)
108 r = repr(c)
109 self.assertIn("key3", r)
110 regex = r"^Config\(\{.*\}\)$"
111 self.assertRegex(r, regex)
112 c2 = eval(r)
113 self.assertIn("1", c)
114 for n in c.names():
115 self.assertEqual(c2[n], c[n])
116 self.assertEqual(c, c2)
117 s = str(c)
118 self.assertIn("\n", s)
119 self.assertNotRegex(s, regex)
121 self.assertCountEqual(c.keys(), ["1", "3", "key3", "dict"])
122 self.assertEqual(list(c), list(c.keys()))
123 self.assertEqual(list(c.values()), [c[k] for k in c.keys()])
124 self.assertEqual(list(c.items()), [(k, c[k]) for k in c.keys()])
126 newKeys = ("key4", ".dict.q", ("dict", "r"), "5")
127 oldKeys = ("key3", ".dict.a", ("dict", "b"), "3")
128 remainingKey = "1"
130 # Check get with existing key
131 for k in oldKeys:
132 self.assertEqual(c.get(k, "missing"), c[k])
134 # Check get, pop with nonexistent key
135 for k in newKeys:
136 self.assertEqual(c.get(k, "missing"), "missing")
137 self.assertEqual(c.pop(k, "missing"), "missing")
139 # Check setdefault with existing key
140 for k in oldKeys:
141 c.setdefault(k, 8)
142 self.assertNotEqual(c[k], 8)
144 # Check setdefault with nonexistent key (mutates c, adding newKeys)
145 for k in newKeys:
146 c.setdefault(k, 8)
147 self.assertEqual(c[k], 8)
149 # Check pop with existing key (mutates c, removing newKeys)
150 for k in newKeys:
151 v = c[k]
152 self.assertEqual(c.pop(k, "missing"), v)
154 # Check deletion (mutates c, removing oldKeys)
155 for k in ("key3", ".dict.a", ("dict", "b"), "3"):
156 self.assertIn(k, c)
157 del c[k]
158 self.assertNotIn(k, c)
160 # Check that `dict` still exists, but is now empty (then remove
161 # it, mutatic c)
162 self.assertIn("dict", c)
163 del c["dict"]
165 # Check popitem (mutates c, removing remainingKey)
166 v = c[remainingKey]
167 self.assertEqual(c.popitem(), (remainingKey, v))
169 # Check that c is now empty
170 self.assertFalse(c)
172 def assertSplit(self, answer, *args):
173 """Helper function to compare string splitting"""
174 for s in (answer, *args):
175 split = Config._splitIntoKeys(s)
176 self.assertEqual(split, answer)
178 def testSplitting(self):
179 """Test of the internal splitting API."""
180 # Try lots of keys that will return the same answer
181 answer = ["a", "b", "c", "d"]
182 self.assertSplit(answer, ".a.b.c.d", ":a:b:c:d", "\ta\tb\tc\td", "\ra\rb\rc\rd")
184 answer = ["a", "calexp.wcs", "b"]
185 self.assertSplit(answer, r".a.calexp\.wcs.b", ":a:calexp.wcs:b")
187 self.assertSplit(["a.b.c"])
188 self.assertSplit(["a", r"b\.c"], r"_a_b\.c")
190 # Escaping a backslash before a delimiter currently fails
191 with self.assertRaises(ValueError):
192 Config._splitIntoKeys(r".a.calexp\\.wcs.b")
194 # The next two fail because internally \r is magic when escaping
195 # a delimiter.
196 with self.assertRaises(ValueError):
197 Config._splitIntoKeys("\ra\rcalexp\\\rwcs\rb")
199 with self.assertRaises(ValueError):
200 Config._splitIntoKeys(".a.cal\rexp\\.wcs.b")
202 def testEscape(self):
203 c = Config({"a": {"foo.bar": 1}, "b😂c": {"bar_baz": 2}})
204 self.assertEqual(c[r".a.foo\.bar"], 1)
205 self.assertEqual(c[":a:foo.bar"], 1)
206 self.assertEqual(c[".b😂c.bar_baz"], 2)
207 self.assertEqual(c[r"😂b\😂c😂bar_baz"], 2)
208 self.assertEqual(c[r"\a\foo.bar"], 1)
209 self.assertEqual(c["\ra\rfoo.bar"], 1)
210 with self.assertRaises(ValueError):
211 c[".a.foo\\.bar\r"]
213 def testOperators(self):
214 c1 = Config({"a": {"b": 1}, "c": 2})
215 c2 = c1.copy()
216 self.assertEqual(c1, c2)
217 self.assertIsInstance(c2, Config)
218 c2[".a.b"] = 5
219 self.assertNotEqual(c1, c2)
221 def testUpdate(self):
222 c = Config({"a": {"b": 1}})
223 c.update({"a": {"c": 2}})
224 self.assertEqual(c[".a.b"], 1)
225 self.assertEqual(c[".a.c"], 2)
226 c.update({"a": {"d": [3, 4]}})
227 self.assertEqual(c[".a.d.0"], 3)
228 c.update({"z": [5, 6, {"g": 2, "h": 3}]})
229 self.assertEqual(c[".z.1"], 6)
231 # This is detached from parent
232 c2 = c[".z.2"]
233 self.assertEqual(c2["g"], 2)
234 c2.update({"h": 4, "j": 5})
235 self.assertEqual(c2["h"], 4)
236 self.assertNotIn(".z.2.j", c)
237 self.assertNotEqual(c[".z.2.h"], 4)
239 with self.assertRaises(RuntimeError):
240 c.update([1, 2, 3])
242 def testHierarchy(self):
243 c = Config()
245 # Simple dict
246 c["a"] = {"z": 52, "x": "string"}
247 self.assertIn(".a.z", c)
248 self.assertEqual(c[".a.x"], "string")
250 # Try different delimiters
251 self.assertEqual(c["⇛a⇛z"], 52)
252 self.assertEqual(c[("a", "z")], 52)
253 self.assertEqual(c["a", "z"], 52)
255 c[".b.new.thing1"] = "thing1"
256 c[".b.new.thing2"] = "thing2"
257 c[".b.new.thing3.supp"] = "supplemental"
258 self.assertEqual(c[".b.new.thing1"], "thing1")
259 tmp = c[".b.new"]
260 self.assertEqual(tmp["thing2"], "thing2")
261 self.assertEqual(c[".b.new.thing3.supp"], "supplemental")
263 # Test that we can index into lists
264 c[".a.b.c"] = [1, "7", 3, {"1": 4, "5": "Five"}, "hello"]
265 self.assertIn(".a.b.c.3.5", c)
266 self.assertNotIn(".a.b.c.10", c)
267 self.assertNotIn(".a.b.c.10.d", c)
268 self.assertEqual(c[".a.b.c.3.5"], "Five")
269 # Is the value in the list?
270 self.assertIn(".a.b.c.hello", c)
271 self.assertNotIn(".a.b.c.hello.not", c)
273 # And assign to an element in the list
274 self.assertEqual(c[".a.b.c.1"], "7")
275 c[".a.b.c.1"] = 8
276 self.assertEqual(c[".a.b.c.1"], 8)
277 self.assertIsInstance(c[".a.b.c"], collections.abc.Sequence)
279 # Test we do get lists back from asArray
280 a = c.asArray(".a.b.c")
281 self.assertIsInstance(a, list)
283 # Is it the *same* list as in the config
284 a.append("Sentinel")
285 self.assertIn("Sentinel", c[".a.b.c"])
286 self.assertIn(".a.b.c.Sentinel", c)
288 # Test we always get a list
289 for k in c.names():
290 a = c.asArray(k)
291 self.assertIsInstance(a, list)
293 # Check we get the same top level keys
294 self.assertEqual(set(c.names(topLevelOnly=True)), set(c._data.keys()))
296 # Check that we can iterate through items
297 for k, v in c.items():
298 self.assertEqual(c[k], v)
300 # Check that lists still work even if assigned a dict
301 c = Config({"cls": "lsst.daf.butler",
302 "formatters": {"calexp.wcs": "{component}",
303 "calexp": "{datasetType}"},
304 "datastores": [{"datastore": {"cls": "datastore1"}},
305 {"datastore": {"cls": "datastore2"}}]})
306 c[".datastores.1.datastore"] = {"cls": "datastore2modified"}
307 self.assertEqual(c[".datastores.0.datastore.cls"], "datastore1")
308 self.assertEqual(c[".datastores.1.datastore.cls"], "datastore2modified")
309 self.assertIsInstance(c["datastores"], collections.abc.Sequence)
311 # Test that we can get all the listed names.
312 # and also that they are marked as "in" the Config
313 # Try delimited names and tuples
314 for n in itertools.chain(c.names(), c.nameTuples()):
315 val = c[n]
316 self.assertIsNotNone(val)
317 self.assertIn(n, c)
319 names = c.names()
320 nameTuples = c.nameTuples()
321 self.assertEqual(len(names), len(nameTuples))
322 self.assertEqual(len(names), 11)
323 self.assertEqual(len(nameTuples), 11)
325 # Test that delimiter escaping works
326 names = c.names(delimiter=".")
327 for n in names:
328 self.assertIn(n, c)
329 self.assertIn(".formatters.calexp\\.wcs", names)
331 # Use a name that includes the internal default delimiter
332 # to test automatic adjustment of delimiter
333 strangeKey = f"calexp{c._D}wcs"
334 c["formatters", strangeKey] = "dynamic"
335 names = c.names()
336 self.assertIn(strangeKey, "-".join(names))
337 self.assertFalse(names[0].startswith(c._D))
338 for n in names:
339 self.assertIn(n, c)
341 top = c.nameTuples(topLevelOnly=True)
342 self.assertIsInstance(top[0], tuple)
344 # Investigate a possible delimeter in a key
345 c = Config({"formatters": {"calexp.wcs": 2, "calexp": 3}})
346 self.assertEqual(c[":formatters:calexp.wcs"], 2)
347 self.assertEqual(c[":formatters:calexp"], 3)
348 for k, v in c["formatters"].items():
349 self.assertEqual(c["formatters", k], v)
351 # Check internal delimiter inheritance
352 c._D = "."
353 c2 = c["formatters"]
354 self.assertEqual(c._D, c2._D) # Check that the child inherits
355 self.assertNotEqual(c2._D, Config._D)
357 def testStringYaml(self):
358 """Test that we can create configs from strings"""
360 c = Config.fromYaml("""
361testing: hello
362formatters:
363 calexp: 3""")
364 self.assertEqual(c["formatters", "calexp"], 3)
365 self.assertEqual(c["testing"], "hello")
368class ConfigSubsetTestCase(unittest.TestCase):
369 """Tests for ConfigSubset
370 """
372 def setUp(self):
373 self.testDir = os.path.abspath(os.path.dirname(__file__))
374 self.configDir = os.path.join(self.testDir, "config", "testConfigs")
375 self.configDir2 = os.path.join(self.testDir, "config", "testConfigs", "test2")
376 self.configDir3 = os.path.join(self.testDir, "config", "testConfigs", "test3")
378 def testEmpty(self):
379 """Ensure that we can read an empty file."""
380 c = ConfigTestEmpty(searchPaths=(self.configDir,))
381 self.assertIsInstance(c, ConfigSubset)
383 def testDefaults(self):
384 """Read of defaults"""
386 # Supply the search path explicitly
387 c = ConfigTest(searchPaths=(self.configDir,))
388 self.assertIsInstance(c, ConfigSubset)
389 self.assertIn("item3", c)
390 self.assertEqual(c["item3"], 3)
392 # Use environment
393 with modified_environment(DAF_BUTLER_CONFIG_PATH=self.configDir):
394 c = ConfigTest()
395 self.assertIsInstance(c, ConfigSubset)
396 self.assertEqual(c["item3"], 3)
398 # No default so this should fail
399 with self.assertRaises(KeyError):
400 c = ConfigTest()
402 def testButlerDir(self):
403 """Test that DAF_BUTLER_DIR is used to locate files."""
404 # with modified_environment(DAF_BUTLER_DIR=self.testDir):
405 # c = ConfigTestButlerDir()
406 # self.assertIn("item3", c)
408 # Again with a search path
409 with modified_environment(DAF_BUTLER_DIR=self.testDir,
410 DAF_BUTLER_CONFIG_PATH=self.configDir2):
411 c = ConfigTestButlerDir()
412 self.assertIn("item3", c)
413 self.assertEqual(c["item3"], "override")
414 self.assertEqual(c["item4"], "new")
416 def testExternalOverride(self):
417 """Ensure that external values win"""
418 c = ConfigTest({"item3": "newval"}, searchPaths=(self.configDir,))
419 self.assertIn("item3", c)
420 self.assertEqual(c["item3"], "newval")
422 def testSearchPaths(self):
423 """Two search paths"""
424 c = ConfigTest(searchPaths=(self.configDir2, self.configDir))
425 self.assertIsInstance(c, ConfigSubset)
426 self.assertIn("item3", c)
427 self.assertEqual(c["item3"], "override")
428 self.assertEqual(c["item4"], "new")
430 c = ConfigTest(searchPaths=(self.configDir, self.configDir2))
431 self.assertIsInstance(c, ConfigSubset)
432 self.assertIn("item3", c)
433 self.assertEqual(c["item3"], 3)
434 self.assertEqual(c["item4"], "new")
436 def testExternalHierarchy(self):
437 """Test that we can provide external config parameters in hierarchy"""
438 c = ConfigTest({"comp": {"item1": 6, "item2": "a", "a": "b",
439 "item3": 7}, "item4": 8})
440 self.assertIn("a", c)
441 self.assertEqual(c["a"], "b")
442 self.assertNotIn("item4", c)
444 def testNoDefaults(self):
445 """Ensure that defaults can be turned off."""
447 # Mandatory keys but no defaults
448 c = ConfigTest({"item1": "a", "item2": "b", "item6": 6})
449 self.assertEqual(len(c.filesRead), 0)
450 self.assertIn("item1", c)
451 self.assertEqual(c["item6"], 6)
453 c = ConfigTestNoDefaults()
454 self.assertEqual(len(c.filesRead), 0)
456 def testAbsPath(self):
457 """Read default config from an absolute path"""
458 # Force the path to be absolute in the class
459 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, "abspath.yaml")
460 c = ConfigTestAbsPath()
461 self.assertEqual(c["item11"], "eleventh")
462 self.assertEqual(len(c.filesRead), 1)
464 # Now specify the normal config file with an absolute path
465 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, ConfigTest.defaultConfigFile)
466 c = ConfigTestAbsPath()
467 self.assertEqual(c["item11"], 11)
468 self.assertEqual(len(c.filesRead), 1)
470 # and a search path that will also include the file
471 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,))
472 self.assertEqual(c["item11"], 11)
473 self.assertEqual(len(c.filesRead), 1)
475 # Same as above but this time with relative path and two search paths
476 # to ensure the count changes
477 ConfigTestAbsPath.defaultConfigFile = ConfigTest.defaultConfigFile
478 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,))
479 self.assertEqual(len(c.filesRead), 2)
481 # Reset the class
482 ConfigTestAbsPath.defaultConfigFile = None
484 def testClassDerived(self):
485 """Read config specified in class determined from config"""
486 c = ConfigTestCls(searchPaths=(self.configDir,))
487 self.assertEqual(c["item50"], 50)
488 self.assertEqual(c["help"], "derived")
490 # Same thing but additional search path
491 c = ConfigTestCls(searchPaths=(self.configDir, self.configDir2))
492 self.assertEqual(c["item50"], 50)
493 self.assertEqual(c["help"], "derived")
494 self.assertEqual(c["help2"], "second")
496 # Same thing but reverse the two paths
497 c = ConfigTestCls(searchPaths=(self.configDir2, self.configDir))
498 self.assertEqual(c["item50"], 500)
499 self.assertEqual(c["help"], "class")
500 self.assertEqual(c["help2"], "second")
501 self.assertEqual(c["help3"], "third")
503 def testInclude(self):
504 """Read a config that has an include directive"""
505 c = Config(os.path.join(self.configDir, "testinclude.yaml"))
506 self.assertEqual(c[".comp1.item1"], 58)
507 self.assertEqual(c[".comp2.comp.item1"], 1)
508 self.assertEqual(c[".comp3.1.comp.item1"], "posix")
509 self.assertEqual(c[".comp4.0.comp.item1"], "posix")
510 self.assertEqual(c[".comp4.1.comp.item1"], 1)
511 self.assertEqual(c[".comp5.comp6.comp.item1"], "posix")
513 # Test a specific name and then test that all
514 # returned names are "in" the config.
515 names = c.names()
516 self.assertIn(c._D.join(("", "comp3", "1", "comp", "item1")), names)
517 for n in names:
518 self.assertIn(n, c)
520 # Test that override delimiter works
521 delimiter = "-"
522 names = c.names(delimiter=delimiter)
523 self.assertIn(delimiter.join(("", "comp3", "1", "comp", "item1")), names)
525 def testStringInclude(self):
526 """Using include directives in strings"""
528 # See if include works for absolute path
529 c = Config.fromYaml(f"something: !include {os.path.join(self.configDir, 'testconfig.yaml')}")
530 self.assertEqual(c["something", "comp", "item3"], 3)
532 with self.assertRaises(FileNotFoundError) as cm:
533 Config.fromYaml("something: !include /not/here.yaml")
534 # Test that it really was trying to open the absolute path
535 self.assertIn("'/not/here.yaml'", str(cm.exception))
537 def testIncludeConfigs(self):
538 """Test the special includeConfigs key for pulling in additional
539 files."""
540 c = Config(os.path.join(self.configDir, "configIncludes.yaml"))
541 self.assertEqual(c["comp", "item2"], "hello")
542 self.assertEqual(c["comp", "item50"], 5000)
543 self.assertEqual(c["comp", "item1"], "first")
544 self.assertEqual(c["comp", "item10"], "tenth")
545 self.assertEqual(c["comp", "item11"], "eleventh")
546 self.assertEqual(c["unrelated"], 1)
547 self.assertEqual(c["addon", "comp", "item1"], "posix")
548 self.assertEqual(c["addon", "comp", "item11"], -1)
549 self.assertEqual(c["addon", "comp", "item50"], 500)
551 # Now test with an environment variable in includeConfigs
552 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3):
553 c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
554 self.assertEqual(c["comp", "item2"], "hello")
555 self.assertEqual(c["comp", "item50"], 5000)
556 self.assertEqual(c["comp", "item1"], "first")
557 self.assertEqual(c["comp", "item10"], "tenth")
558 self.assertEqual(c["comp", "item11"], "eleventh")
559 self.assertEqual(c["unrelated"], 1)
560 self.assertEqual(c["addon", "comp", "item1"], "envvar")
561 self.assertEqual(c["addon", "comp", "item11"], -1)
562 self.assertEqual(c["addon", "comp", "item50"], 501)
564 # This will fail
565 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2):
566 with self.assertRaises(FileNotFoundError):
567 Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
570class FileWriteConfigTestCase(unittest.TestCase):
572 def setUp(self):
573 self.tmpdir = tempfile.mkdtemp()
575 def tearDown(self):
576 if os.path.exists(self.tmpdir):
577 shutil.rmtree(self.tmpdir, ignore_errors=True)
579 def testDump(self):
580 """Test that we can write and read a configuration."""
582 c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}})
584 outpath = os.path.join(self.tmpdir, "test.yaml")
585 c.dumpToUri(outpath)
587 c2 = Config(outpath)
588 self.assertEqual(c2, c)
590 c.dumpToUri(outpath, overwrite=True)
591 with self.assertRaises(FileExistsError):
592 c.dumpToUri(outpath, overwrite=False)
595if __name__ == "__main__": 595 ↛ 596line 595 didn't jump to line 596, because the condition on line 595 was never true
596 unittest.main()