Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import os 

24import contextlib 

25import collections 

26import itertools 

27 

28from lsst.daf.butler import ConfigSubset, Config 

29 

30 

31@contextlib.contextmanager 

32def modified_environment(**environ): 

33 """ 

34 Temporarily set environment variables. 

35 

36 >>> with modified_environment(DAF_BUTLER_DIR="/somewhere"): 

37 ... os.environ["DAF_BUTLER_DIR"] == "/somewhere" 

38 True 

39 

40 >>> "DAF_BUTLER_DIR" != "/somewhere" 

41 True 

42 

43 Parameters 

44 ---------- 

45 environ : `dict` 

46 Key value pairs of environment variables to temporarily set. 

47 """ 

48 old_environ = dict(os.environ) 

49 os.environ.update(environ) 

50 try: 

51 yield 

52 finally: 

53 os.environ.clear() 

54 os.environ.update(old_environ) 

55 

56 

57class ExampleWithConfigFileReference: 

58 defaultConfigFile = "viacls.yaml" 

59 

60 

61class ExampleWithConfigFileReference2: 

62 defaultConfigFile = "viacls2.yaml" 

63 

64 

65class ConfigTest(ConfigSubset): 

66 component = "comp" 

67 requiredKeys = ("item1", "item2") 

68 defaultConfigFile = "testconfig.yaml" 

69 

70 

71class ConfigTestEmpty(ConfigTest): 

72 defaultConfigFile = "testconfig_empty.yaml" 

73 requiredKeys = () 

74 

75 

76class ConfigTestButlerDir(ConfigTest): 

77 defaultConfigFile = "testConfigs/testconfig.yaml" 

78 

79 

80class ConfigTestNoDefaults(ConfigTest): 

81 defaultConfigFile = None 

82 requiredKeys = () 

83 

84 

85class ConfigTestAbsPath(ConfigTest): 

86 defaultConfigFile = None 

87 requiredKeys = () 

88 

89 

90class ConfigTestCls(ConfigTest): 

91 defaultConfigFile = "withcls.yaml" 

92 

93 

94class ConfigTestCase(unittest.TestCase): 

95 """Tests of simple Config""" 

96 

97 def testBadConfig(self): 

98 for badArg in ([], "file.fits"): 

99 with self.assertRaises(RuntimeError): 

100 Config(badArg) 

101 

102 def testBasics(self): 

103 c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}}) 

104 pretty = c.ppprint() 

105 self.assertIn("key3", pretty) 

106 r = repr(c) 

107 self.assertIn("key3", r) 

108 regex = r"^Config\(\{.*\}\)$" 

109 self.assertRegex(r, regex) 

110 c2 = eval(r) 

111 self.assertIn("1", c) 

112 for n in c.names(): 

113 self.assertEqual(c2[n], c[n]) 

114 self.assertEqual(c, c2) 

115 s = str(c) 

116 self.assertIn("\n", s) 

117 self.assertNotRegex(s, regex) 

118 

119 self.assertCountEqual(c.keys(), ["1", "3", "key3", "dict"]) 

120 self.assertEqual(list(c), list(c.keys())) 

121 self.assertEqual(list(c.values()), [c[k] for k in c.keys()]) 

122 self.assertEqual(list(c.items()), [(k, c[k]) for k in c.keys()]) 

123 

124 newKeys = ("key4", ".dict.q", ("dict", "r"), "5") 

125 oldKeys = ("key3", ".dict.a", ("dict", "b"), "3") 

126 remainingKey = "1" 

127 

128 # Check get with existing key 

129 for k in oldKeys: 

130 self.assertEqual(c.get(k, "missing"), c[k]) 

131 

132 # Check get, pop with nonexistent key 

133 for k in newKeys: 

134 self.assertEqual(c.get(k, "missing"), "missing") 

135 self.assertEqual(c.pop(k, "missing"), "missing") 

136 

137 # Check setdefault with existing key 

138 for k in oldKeys: 

139 c.setdefault(k, 8) 

140 self.assertNotEqual(c[k], 8) 

141 

142 # Check setdefault with nonexistent key (mutates c, adding newKeys) 

143 for k in newKeys: 

144 c.setdefault(k, 8) 

145 self.assertEqual(c[k], 8) 

146 

147 # Check pop with existing key (mutates c, removing newKeys) 

148 for k in newKeys: 

149 v = c[k] 

150 self.assertEqual(c.pop(k, "missing"), v) 

151 

152 # Check deletion (mutates c, removing oldKeys) 

153 for k in ("key3", ".dict.a", ("dict", "b"), "3"): 

154 self.assertIn(k, c) 

155 del c[k] 

156 self.assertNotIn(k, c) 

157 

158 # Check that `dict` still exists, but is now empty (then remove 

159 # it, mutatic c) 

160 self.assertIn("dict", c) 

161 del c["dict"] 

162 

163 # Check popitem (mutates c, removing remainingKey) 

164 v = c[remainingKey] 

165 self.assertEqual(c.popitem(), (remainingKey, v)) 

166 

167 # Check that c is now empty 

168 self.assertFalse(c) 

169 

170 def assertSplit(self, answer, *args): 

171 """Helper function to compare string splitting""" 

172 for s in (answer, *args): 

173 split = Config._splitIntoKeys(s) 

174 self.assertEqual(split, answer) 

175 

176 def testSplitting(self): 

177 """Test of the internal splitting API.""" 

178 # Try lots of keys that will return the same answer 

179 answer = ["a", "b", "c", "d"] 

180 self.assertSplit(answer, ".a.b.c.d", ":a:b:c:d", "\ta\tb\tc\td", "\ra\rb\rc\rd") 

181 

182 answer = ["a", "calexp.wcs", "b"] 

183 self.assertSplit(answer, r".a.calexp\.wcs.b", ":a:calexp.wcs:b") 

184 

185 self.assertSplit(["a.b.c"]) 

186 self.assertSplit(["a", r"b\.c"], r"_a_b\.c") 

187 

188 # Escaping a backslash before a delimiter currently fails 

189 with self.assertRaises(ValueError): 

190 Config._splitIntoKeys(r".a.calexp\\.wcs.b") 

191 

192 # The next two fail because internally \r is magic when escaping 

193 # a delimiter. 

194 with self.assertRaises(ValueError): 

195 Config._splitIntoKeys("\ra\rcalexp\\\rwcs\rb") 

196 

197 with self.assertRaises(ValueError): 

198 Config._splitIntoKeys(".a.cal\rexp\\.wcs.b") 

199 

200 def testEscape(self): 

201 c = Config({"a": {"foo.bar": 1}, "b😂c": {"bar_baz": 2}}) 

202 self.assertEqual(c[r".a.foo\.bar"], 1) 

203 self.assertEqual(c[":a:foo.bar"], 1) 

204 self.assertEqual(c[".b😂c.bar_baz"], 2) 

205 self.assertEqual(c[r"😂b\😂c😂bar_baz"], 2) 

206 self.assertEqual(c[r"\a\foo.bar"], 1) 

207 self.assertEqual(c["\ra\rfoo.bar"], 1) 

208 with self.assertRaises(ValueError): 

209 c[".a.foo\\.bar\r"] 

210 

211 def testOperators(self): 

212 c1 = Config({"a": {"b": 1}, "c": 2}) 

213 c2 = c1.copy() 

214 self.assertEqual(c1, c2) 

215 self.assertIsInstance(c2, Config) 

216 c2[".a.b"] = 5 

217 self.assertNotEqual(c1, c2) 

218 

219 def testUpdate(self): 

220 c = Config({"a": {"b": 1}}) 

221 c.update({"a": {"c": 2}}) 

222 self.assertEqual(c[".a.b"], 1) 

223 self.assertEqual(c[".a.c"], 2) 

224 c.update({"a": {"d": [3, 4]}}) 

225 self.assertEqual(c[".a.d.0"], 3) 

226 c.update({"z": [5, 6, {"g": 2, "h": 3}]}) 

227 self.assertEqual(c[".z.1"], 6) 

228 

229 # This is detached from parent 

230 c2 = c[".z.2"] 

231 self.assertEqual(c2["g"], 2) 

232 c2.update({"h": 4, "j": 5}) 

233 self.assertEqual(c2["h"], 4) 

234 self.assertNotIn(".z.2.j", c) 

235 self.assertNotEqual(c[".z.2.h"], 4) 

236 

237 with self.assertRaises(RuntimeError): 

238 c.update([1, 2, 3]) 

239 

240 def testHierarchy(self): 

241 c = Config() 

242 

243 # Simple dict 

244 c["a"] = {"z": 52, "x": "string"} 

245 self.assertIn(".a.z", c) 

246 self.assertEqual(c[".a.x"], "string") 

247 

248 # Try different delimiters 

249 self.assertEqual(c["⇛a⇛z"], 52) 

250 self.assertEqual(c[("a", "z")], 52) 

251 self.assertEqual(c["a", "z"], 52) 

252 

253 c[".b.new.thing1"] = "thing1" 

254 c[".b.new.thing2"] = "thing2" 

255 c[".b.new.thing3.supp"] = "supplemental" 

256 self.assertEqual(c[".b.new.thing1"], "thing1") 

257 tmp = c[".b.new"] 

258 self.assertEqual(tmp["thing2"], "thing2") 

259 self.assertEqual(c[".b.new.thing3.supp"], "supplemental") 

260 

261 # Test that we can index into lists 

262 c[".a.b.c"] = [1, "7", 3, {"1": 4, "5": "Five"}, "hello"] 

263 self.assertIn(".a.b.c.3.5", c) 

264 self.assertNotIn(".a.b.c.10", c) 

265 self.assertNotIn(".a.b.c.10.d", c) 

266 self.assertEqual(c[".a.b.c.3.5"], "Five") 

267 # Is the value in the list? 

268 self.assertIn(".a.b.c.hello", c) 

269 self.assertNotIn(".a.b.c.hello.not", c) 

270 

271 # And assign to an element in the list 

272 self.assertEqual(c[".a.b.c.1"], "7") 

273 c[".a.b.c.1"] = 8 

274 self.assertEqual(c[".a.b.c.1"], 8) 

275 self.assertIsInstance(c[".a.b.c"], collections.abc.Sequence) 

276 

277 # Test we do get lists back from asArray 

278 a = c.asArray(".a.b.c") 

279 self.assertIsInstance(a, list) 

280 

281 # Is it the *same* list as in the config 

282 a.append("Sentinel") 

283 self.assertIn("Sentinel", c[".a.b.c"]) 

284 self.assertIn(".a.b.c.Sentinel", c) 

285 

286 # Test we always get a list 

287 for k in c.names(): 

288 a = c.asArray(k) 

289 self.assertIsInstance(a, list) 

290 

291 # Check we get the same top level keys 

292 self.assertEqual(set(c.names(topLevelOnly=True)), set(c._data.keys())) 

293 

294 # Check that we can iterate through items 

295 for k, v in c.items(): 

296 self.assertEqual(c[k], v) 

297 

298 # Check that lists still work even if assigned a dict 

299 c = Config({"cls": "lsst.daf.butler", 

300 "formatters": {"calexp.wcs": "{component}", 

301 "calexp": "{datasetType}"}, 

302 "datastores": [{"datastore": {"cls": "datastore1"}}, 

303 {"datastore": {"cls": "datastore2"}}]}) 

304 c[".datastores.1.datastore"] = {"cls": "datastore2modified"} 

305 self.assertEqual(c[".datastores.0.datastore.cls"], "datastore1") 

306 self.assertEqual(c[".datastores.1.datastore.cls"], "datastore2modified") 

307 self.assertIsInstance(c["datastores"], collections.abc.Sequence) 

308 

309 # Test that we can get all the listed names. 

310 # and also that they are marked as "in" the Config 

311 # Try delimited names and tuples 

312 for n in itertools.chain(c.names(), c.nameTuples()): 

313 val = c[n] 

314 self.assertIsNotNone(val) 

315 self.assertIn(n, c) 

316 

317 names = c.names() 

318 nameTuples = c.nameTuples() 

319 self.assertEqual(len(names), len(nameTuples)) 

320 self.assertEqual(len(names), 11) 

321 self.assertEqual(len(nameTuples), 11) 

322 

323 # Test that delimiter escaping works 

324 names = c.names(delimiter=".") 

325 for n in names: 

326 self.assertIn(n, c) 

327 self.assertIn(".formatters.calexp\\.wcs", names) 

328 

329 # Use a name that includes the internal default delimiter 

330 # to test automatic adjustment of delimiter 

331 strangeKey = f"calexp{c._D}wcs" 

332 c["formatters", strangeKey] = "dynamic" 

333 names = c.names() 

334 self.assertIn(strangeKey, "-".join(names)) 

335 self.assertFalse(names[0].startswith(c._D)) 

336 for n in names: 

337 self.assertIn(n, c) 

338 

339 top = c.nameTuples(topLevelOnly=True) 

340 self.assertIsInstance(top[0], tuple) 

341 

342 # Investigate a possible delimeter in a key 

343 c = Config({"formatters": {"calexp.wcs": 2, "calexp": 3}}) 

344 self.assertEqual(c[":formatters:calexp.wcs"], 2) 

345 self.assertEqual(c[":formatters:calexp"], 3) 

346 for k, v in c["formatters"].items(): 

347 self.assertEqual(c["formatters", k], v) 

348 

349 # Check internal delimiter inheritance 

350 c._D = "." 

351 c2 = c["formatters"] 

352 self.assertEqual(c._D, c2._D) # Check that the child inherits 

353 self.assertNotEqual(c2._D, Config._D) 

354 

355 

356class ConfigSubsetTestCase(unittest.TestCase): 

357 """Tests for ConfigSubset 

358 """ 

359 

360 def setUp(self): 

361 self.testDir = os.path.abspath(os.path.dirname(__file__)) 

362 self.configDir = os.path.join(self.testDir, "config", "testConfigs") 

363 self.configDir2 = os.path.join(self.testDir, "config", "testConfigs", "test2") 

364 self.configDir3 = os.path.join(self.testDir, "config", "testConfigs", "test3") 

365 

366 def testEmpty(self): 

367 """Ensure that we can read an empty file.""" 

368 c = ConfigTestEmpty(searchPaths=(self.configDir,)) 

369 self.assertIsInstance(c, ConfigSubset) 

370 

371 def testDefaults(self): 

372 """Read of defaults""" 

373 

374 # Supply the search path explicitly 

375 c = ConfigTest(searchPaths=(self.configDir,)) 

376 self.assertIsInstance(c, ConfigSubset) 

377 self.assertIn("item3", c) 

378 self.assertEqual(c["item3"], 3) 

379 

380 # Use environment 

381 with modified_environment(DAF_BUTLER_CONFIG_PATH=self.configDir): 

382 c = ConfigTest() 

383 self.assertIsInstance(c, ConfigSubset) 

384 self.assertEqual(c["item3"], 3) 

385 

386 # No default so this should fail 

387 with self.assertRaises(KeyError): 

388 c = ConfigTest() 

389 

390 def testButlerDir(self): 

391 """Test that DAF_BUTLER_DIR is used to locate files.""" 

392 # with modified_environment(DAF_BUTLER_DIR=self.testDir): 

393 # c = ConfigTestButlerDir() 

394 # self.assertIn("item3", c) 

395 

396 # Again with a search path 

397 with modified_environment(DAF_BUTLER_DIR=self.testDir, 

398 DAF_BUTLER_CONFIG_PATH=self.configDir2): 

399 c = ConfigTestButlerDir() 

400 self.assertIn("item3", c) 

401 self.assertEqual(c["item3"], "override") 

402 self.assertEqual(c["item4"], "new") 

403 

404 def testExternalOverride(self): 

405 """Ensure that external values win""" 

406 c = ConfigTest({"item3": "newval"}, searchPaths=(self.configDir,)) 

407 self.assertIn("item3", c) 

408 self.assertEqual(c["item3"], "newval") 

409 

410 def testSearchPaths(self): 

411 """Two search paths""" 

412 c = ConfigTest(searchPaths=(self.configDir2, self.configDir)) 

413 self.assertIsInstance(c, ConfigSubset) 

414 self.assertIn("item3", c) 

415 self.assertEqual(c["item3"], "override") 

416 self.assertEqual(c["item4"], "new") 

417 

418 c = ConfigTest(searchPaths=(self.configDir, self.configDir2)) 

419 self.assertIsInstance(c, ConfigSubset) 

420 self.assertIn("item3", c) 

421 self.assertEqual(c["item3"], 3) 

422 self.assertEqual(c["item4"], "new") 

423 

424 def testExternalHierarchy(self): 

425 """Test that we can provide external config parameters in hierarchy""" 

426 c = ConfigTest({"comp": {"item1": 6, "item2": "a", "a": "b", 

427 "item3": 7}, "item4": 8}) 

428 self.assertIn("a", c) 

429 self.assertEqual(c["a"], "b") 

430 self.assertNotIn("item4", c) 

431 

432 def testNoDefaults(self): 

433 """Ensure that defaults can be turned off.""" 

434 

435 # Mandatory keys but no defaults 

436 c = ConfigTest({"item1": "a", "item2": "b", "item6": 6}) 

437 self.assertEqual(len(c.filesRead), 0) 

438 self.assertIn("item1", c) 

439 self.assertEqual(c["item6"], 6) 

440 

441 c = ConfigTestNoDefaults() 

442 self.assertEqual(len(c.filesRead), 0) 

443 

444 def testAbsPath(self): 

445 """Read default config from an absolute path""" 

446 # Force the path to be absolute in the class 

447 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, "abspath.yaml") 

448 c = ConfigTestAbsPath() 

449 self.assertEqual(c["item11"], "eleventh") 

450 self.assertEqual(len(c.filesRead), 1) 

451 

452 # Now specify the normal config file with an absolute path 

453 ConfigTestAbsPath.defaultConfigFile = os.path.join(self.configDir, ConfigTest.defaultConfigFile) 

454 c = ConfigTestAbsPath() 

455 self.assertEqual(c["item11"], 11) 

456 self.assertEqual(len(c.filesRead), 1) 

457 

458 # and a search path that will also include the file 

459 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,)) 

460 self.assertEqual(c["item11"], 11) 

461 self.assertEqual(len(c.filesRead), 1) 

462 

463 # Same as above but this time with relative path and two search paths 

464 # to ensure the count changes 

465 ConfigTestAbsPath.defaultConfigFile = ConfigTest.defaultConfigFile 

466 c = ConfigTestAbsPath(searchPaths=(self.configDir, self.configDir2,)) 

467 self.assertEqual(len(c.filesRead), 2) 

468 

469 # Reset the class 

470 ConfigTestAbsPath.defaultConfigFile = None 

471 

472 def testClassDerived(self): 

473 """Read config specified in class determined from config""" 

474 c = ConfigTestCls(searchPaths=(self.configDir,)) 

475 self.assertEqual(c["item50"], 50) 

476 self.assertEqual(c["help"], "derived") 

477 

478 # Same thing but additional search path 

479 c = ConfigTestCls(searchPaths=(self.configDir, self.configDir2)) 

480 self.assertEqual(c["item50"], 50) 

481 self.assertEqual(c["help"], "derived") 

482 self.assertEqual(c["help2"], "second") 

483 

484 # Same thing but reverse the two paths 

485 c = ConfigTestCls(searchPaths=(self.configDir2, self.configDir)) 

486 self.assertEqual(c["item50"], 500) 

487 self.assertEqual(c["help"], "class") 

488 self.assertEqual(c["help2"], "second") 

489 self.assertEqual(c["help3"], "third") 

490 

491 def testInclude(self): 

492 """Read a config that has an include directive""" 

493 c = Config(os.path.join(self.configDir, "testinclude.yaml")) 

494 self.assertEqual(c[".comp1.item1"], 58) 

495 self.assertEqual(c[".comp2.comp.item1"], 1) 

496 self.assertEqual(c[".comp3.1.comp.item1"], "posix") 

497 self.assertEqual(c[".comp4.0.comp.item1"], "posix") 

498 self.assertEqual(c[".comp4.1.comp.item1"], 1) 

499 self.assertEqual(c[".comp5.comp6.comp.item1"], "posix") 

500 

501 # Test a specific name and then test that all 

502 # returned names are "in" the config. 

503 names = c.names() 

504 self.assertIn(c._D.join(("", "comp3", "1", "comp", "item1")), names) 

505 for n in names: 

506 self.assertIn(n, c) 

507 

508 # Test that override delimiter works 

509 delimiter = "-" 

510 names = c.names(delimiter=delimiter) 

511 self.assertIn(delimiter.join(("", "comp3", "1", "comp", "item1")), names) 

512 

513 def testIncludeConfigs(self): 

514 """Test the special includeConfigs key for pulling in additional 

515 files.""" 

516 c = Config(os.path.join(self.configDir, "configIncludes.yaml")) 

517 self.assertEqual(c["comp", "item2"], "hello") 

518 self.assertEqual(c["comp", "item50"], 5000) 

519 self.assertEqual(c["comp", "item1"], "first") 

520 self.assertEqual(c["comp", "item10"], "tenth") 

521 self.assertEqual(c["comp", "item11"], "eleventh") 

522 self.assertEqual(c["unrelated"], 1) 

523 self.assertEqual(c["addon", "comp", "item1"], "posix") 

524 self.assertEqual(c["addon", "comp", "item11"], -1) 

525 self.assertEqual(c["addon", "comp", "item50"], 500) 

526 

527 # Now test with an environment variable in includeConfigs 

528 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3): 

529 c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml")) 

530 self.assertEqual(c["comp", "item2"], "hello") 

531 self.assertEqual(c["comp", "item50"], 5000) 

532 self.assertEqual(c["comp", "item1"], "first") 

533 self.assertEqual(c["comp", "item10"], "tenth") 

534 self.assertEqual(c["comp", "item11"], "eleventh") 

535 self.assertEqual(c["unrelated"], 1) 

536 self.assertEqual(c["addon", "comp", "item1"], "envvar") 

537 self.assertEqual(c["addon", "comp", "item11"], -1) 

538 self.assertEqual(c["addon", "comp", "item50"], 501) 

539 

540 # This will fail 

541 with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2): 

542 with self.assertRaises(FileNotFoundError): 

543 Config(os.path.join(self.configDir, "configIncludesEnv.yaml")) 

544 

545 

546if __name__ == "__main__": 546 ↛ 547line 546 didn't jump to line 547, because the condition on line 546 was never true

547 unittest.main()