Coverage for tests/test_Config.py: 17%

365 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-01 12:22 +0000

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import io 

29import itertools 

30import os 

31import pickle 

32import re 

33import tempfile 

34import unittest 

35from types import SimpleNamespace 

36 

37try: 

38 import yaml 

39except ImportError: 

40 yaml = None 

41 

42import lsst.pex.config as pexConfig 

43 

44# Some tests depend on daf_base. 

45# Skip them if it is not found. 

46try: 

47 import lsst.daf.base as dafBase 

48except ImportError: 

49 dafBase = None 

50 

51GLOBAL_REGISTRY = {} 

52 

53 

54class Simple(pexConfig.Config): 

55 """A simple config used for testing.""" 

56 

57 i = pexConfig.Field("integer test", int, optional=True) 

58 f = pexConfig.Field("float test", float, default=3.0) 

59 b = pexConfig.Field("boolean test", bool, default=False, optional=False) 

60 c = pexConfig.ChoiceField( 

61 "choice test", str, default="Hello", allowed={"Hello": "First choice", "World": "second choice"} 

62 ) 

63 r = pexConfig.RangeField("Range test", float, default=3.0, optional=False, min=3.0, inclusiveMin=True) 

64 ll = pexConfig.ListField( 64 ↛ exitline 64 didn't jump to the function exit

65 "list test", int, default=[1, 2, 3], maxLength=5, itemCheck=lambda x: x is not None and x > 0 

66 ) 

67 d = pexConfig.DictField( 67 ↛ exitline 67 didn't jump to the function exit

68 "dict test", str, str, default={"key": "value"}, itemCheck=lambda x: x.startswith("v") 

69 ) 

70 n = pexConfig.Field("nan test", float, default=float("NAN")) 

71 

72 

73GLOBAL_REGISTRY["AAA"] = Simple 

74 

75 

76class InnerConfig(pexConfig.Config): 

77 """Inner config used for testing.""" 

78 

79 f = pexConfig.Field("Inner.f", float, default=0.0, check=lambda x: x >= 0, optional=False) 79 ↛ exitline 79 didn't run the lambda on line 79

80 

81 

82GLOBAL_REGISTRY["BBB"] = InnerConfig 

83 

84 

85class OuterConfig(InnerConfig, pexConfig.Config): 

86 """Outer config used for testing.""" 

87 

88 i = pexConfig.ConfigField("Outer.i", InnerConfig) 

89 

90 def __init__(self): 

91 pexConfig.Config.__init__(self) 

92 self.i.f = 5.0 

93 

94 def validate(self): 

95 pexConfig.Config.validate(self) 

96 if self.i.f < 5: 

97 raise ValueError("validation failed, outer.i.f must be greater than 5") 

98 

99 

100class Complex(pexConfig.Config): 

101 """A complex config for testing.""" 

102 

103 c = pexConfig.ConfigField("an inner config", InnerConfig) 

104 r = pexConfig.ConfigChoiceField( 

105 "a registry field", typemap=GLOBAL_REGISTRY, default="AAA", optional=False 

106 ) 

107 p = pexConfig.ConfigChoiceField("another registry", typemap=GLOBAL_REGISTRY, default="BBB", optional=True) 

108 

109 

110class Deprecation(pexConfig.Config): 

111 """A test config with a deprecated field.""" 

112 

113 old = pexConfig.Field("Something.", int, default=10, deprecated="not used!") 

114 

115 

116class ConfigTest(unittest.TestCase): 

117 """Tests of basic Config functionality.""" 

118 

119 def setUp(self): 

120 self.simple = Simple() 

121 self.inner = InnerConfig() 

122 self.outer = OuterConfig() 

123 self.comp = Complex() 

124 self.deprecation = Deprecation() 

125 

126 def tearDown(self): 

127 del self.simple 

128 del self.inner 

129 del self.outer 

130 del self.comp 

131 

132 def testFieldTypeAnnotationRuntime(self): 

133 # test parsing type annotation for runtime dtype 

134 testField = pexConfig.Field[str](doc="test") 

135 self.assertEqual(testField.dtype, str) 

136 

137 # verify that forward references work correctly 

138 testField = pexConfig.Field["float"](doc="test") 

139 self.assertEqual(testField.dtype, float) 

140 

141 # verify that Field rejects multiple types 

142 with self.assertRaises(ValueError): 

143 pexConfig.Field[str, int](doc="test") # type: ignore 

144 

145 # verify that Field raises in conflict with dtype: 

146 with self.assertRaises(ValueError): 

147 pexConfig.Field[str](doc="test", dtype=int) 

148 

149 # verify that Field does not raise if dtype agrees 

150 testField = pexConfig.Field[int](doc="test", dtype=int) 

151 self.assertEqual(testField.dtype, int) 

152 

153 def testInit(self): 

154 self.assertIsNone(self.simple.i) 

155 self.assertEqual(self.simple.f, 3.0) 

156 self.assertFalse(self.simple.b) 

157 self.assertEqual(self.simple.c, "Hello") 

158 self.assertEqual(list(self.simple.ll), [1, 2, 3]) 

159 self.assertEqual(self.simple.d["key"], "value") 

160 self.assertEqual(self.inner.f, 0.0) 

161 self.assertEqual(self.deprecation.old, 10) 

162 

163 self.assertEqual(self.deprecation._fields["old"].doc, "Something. Deprecated: not used!") 

164 

165 self.assertEqual(self.outer.i.f, 5.0) 

166 self.assertEqual(self.outer.f, 0.0) 

167 

168 self.assertEqual(self.comp.c.f, 0.0) 

169 self.assertEqual(self.comp.r.name, "AAA") 

170 self.assertEqual(self.comp.r.active.f, 3.0) 

171 self.assertEqual(self.comp.r["BBB"].f, 0.0) 

172 

173 def testDeprecationWarning(self): 

174 """Test that a deprecated field emits a warning when it is set.""" 

175 with self.assertWarns(FutureWarning) as w: 

176 self.deprecation.old = 5 

177 self.assertEqual(self.deprecation.old, 5) 

178 

179 self.assertIn(self.deprecation._fields["old"].deprecated, str(w.warnings[-1].message)) 

180 

181 def testDeprecationOutput(self): 

182 """Test that a deprecated field is not written out unless it is set.""" 

183 stream = io.StringIO() 

184 self.deprecation.saveToStream(stream) 

185 self.assertNotIn("config.old", stream.getvalue()) 

186 with self.assertWarns(FutureWarning): 

187 self.deprecation.old = 5 

188 stream = io.StringIO() 

189 self.deprecation.saveToStream(stream) 

190 self.assertIn("config.old=5\n", stream.getvalue()) 

191 

192 def testDocstring(self): 

193 """Test that the docstring is not allowed to be empty.""" 

194 with self.assertRaises(ValueError): 

195 pexConfig.Field("", int, default=1) 

196 

197 with self.assertRaises(ValueError): 

198 pexConfig.RangeField("", int, default=3, min=3, max=4) 

199 

200 with self.assertRaises(ValueError): 

201 pexConfig.DictField("", str, str, default={"key": "value"}) 

202 

203 with self.assertRaises(ValueError): 

204 pexConfig.ListField("", int, default=[1, 2, 3]) 

205 

206 with self.assertRaises(ValueError): 

207 pexConfig.ConfigField("", InnerConfig) 

208 

209 with self.assertRaises(ValueError): 

210 pexConfig.ConfigChoiceField("", typemap=GLOBAL_REGISTRY, default="AAA") 

211 

212 def testValidate(self): 

213 self.simple.validate() 

214 

215 self.inner.validate() 

216 self.assertRaises(ValueError, setattr, self.outer.i, "f", -5) 

217 self.outer.i.f = 10.0 

218 self.outer.validate() 

219 

220 try: 

221 self.simple.d["failKey"] = "failValue" 

222 except pexConfig.FieldValidationError: 

223 pass 

224 except Exception: 

225 raise "Validation error Expected" 

226 self.simple.validate() 

227 

228 self.outer.i = InnerConfig 

229 self.assertRaises(ValueError, self.outer.validate) 

230 self.outer.i = InnerConfig() 

231 self.assertRaises(ValueError, self.outer.validate) 

232 

233 self.comp.validate() 

234 self.comp.r = None 

235 self.assertRaises(ValueError, self.comp.validate) 

236 self.comp.r = "BBB" 

237 self.comp.validate() 

238 

239 def testRangeFieldConstructor(self): 

240 """Test RangeField constructor's checking of min, max.""" 

241 val = 3 

242 self.assertRaises(ValueError, pexConfig.RangeField, "test", int, default=val, min=val, max=val - 1) 

243 self.assertRaises( 

244 ValueError, pexConfig.RangeField, "test", float, default=val, min=val, max=val - 1e-15 

245 ) 

246 for inclusiveMin, inclusiveMax in itertools.product((False, True), (False, True)): 

247 if inclusiveMin and inclusiveMax: 

248 # should not raise 

249 class Cfg1(pexConfig.Config): 

250 r1 = pexConfig.RangeField( 

251 doc="test", 

252 dtype=int, 

253 default=val, 

254 min=val, 

255 max=val, 

256 inclusiveMin=inclusiveMin, 

257 inclusiveMax=inclusiveMax, 

258 ) 

259 r2 = pexConfig.RangeField( 

260 doc="test", 

261 dtype=float, 

262 default=val, 

263 min=val, 

264 max=val, 

265 inclusiveMin=inclusiveMin, 

266 inclusiveMax=inclusiveMax, 

267 ) 

268 

269 Cfg1() 

270 else: 

271 # raise while constructing the RangeField (hence cannot make 

272 # it part of a Config) 

273 self.assertRaises( 

274 ValueError, 

275 pexConfig.RangeField, 

276 doc="test", 

277 dtype=int, 

278 default=val, 

279 min=val, 

280 max=val, 

281 inclusiveMin=inclusiveMin, 

282 inclusiveMax=inclusiveMax, 

283 ) 

284 self.assertRaises( 

285 ValueError, 

286 pexConfig.RangeField, 

287 doc="test", 

288 dtype=float, 

289 default=val, 

290 min=val, 

291 max=val, 

292 inclusiveMin=inclusiveMin, 

293 inclusiveMax=inclusiveMax, 

294 ) 

295 

296 def testRangeFieldDefault(self): 

297 """Test RangeField's checking of the default value.""" 

298 minVal = 3 

299 maxVal = 4 

300 for val, inclusiveMin, inclusiveMax, shouldRaise in ( 

301 (minVal, False, True, True), 

302 (minVal, True, True, False), 

303 (maxVal, True, False, True), 

304 (maxVal, True, True, False), 

305 ): 

306 

307 class Cfg1(pexConfig.Config): 

308 r = pexConfig.RangeField( 

309 doc="test", 

310 dtype=int, 

311 default=val, 

312 min=minVal, 

313 max=maxVal, 

314 inclusiveMin=inclusiveMin, 

315 inclusiveMax=inclusiveMax, 

316 ) 

317 

318 class Cfg2(pexConfig.Config): 

319 r2 = pexConfig.RangeField( 

320 doc="test", 

321 dtype=float, 

322 default=val, 

323 min=minVal, 

324 max=maxVal, 

325 inclusiveMin=inclusiveMin, 

326 inclusiveMax=inclusiveMax, 

327 ) 

328 

329 if shouldRaise: 

330 self.assertRaises(pexConfig.FieldValidationError, Cfg1) 

331 self.assertRaises(pexConfig.FieldValidationError, Cfg2) 

332 else: 

333 Cfg1() 

334 Cfg2() 

335 

336 def testSave(self): 

337 self.comp.r = "BBB" 

338 self.comp.p = "AAA" 

339 self.comp.c.f = 5.0 

340 with tempfile.TemporaryDirectory(prefix="config-save-test", ignore_cleanup_errors=True) as tmpdir: 

341 roundtrip_path = os.path.join(tmpdir, "roundtrip.test") 

342 self.comp.save(roundtrip_path) 

343 

344 roundTrip = Complex() 

345 roundTrip.load(roundtrip_path) 

346 self.assertEqual(self.comp.c.f, roundTrip.c.f) 

347 self.assertEqual(self.comp.r.name, roundTrip.r.name) 

348 del roundTrip 

349 

350 # test saving to an open file 

351 roundtrip_path = os.path.join(tmpdir, "roundtrip_open.test") 

352 with open(roundtrip_path, "w") as outfile: 

353 self.comp.saveToStream(outfile) 

354 roundTrip = Complex() 

355 with open(roundtrip_path) as infile: 

356 roundTrip.loadFromStream(infile) 

357 self.assertEqual(self.comp.c.f, roundTrip.c.f) 

358 self.assertEqual(self.comp.r.name, roundTrip.r.name) 

359 del roundTrip 

360 

361 # Test an override of the default variable name. 

362 roundtrip_path = os.path.join(tmpdir, "roundtrip_def.test") 

363 with open(roundtrip_path, "w") as outfile: 

364 self.comp.saveToStream(outfile, root="root") 

365 roundTrip = Complex() 

366 with self.assertRaises(NameError): 

367 roundTrip.load(roundtrip_path) 

368 roundTrip.load(roundtrip_path, root="root") 

369 self.assertEqual(self.comp.c.f, roundTrip.c.f) 

370 self.assertEqual(self.comp.r.name, roundTrip.r.name) 

371 

372 # test saving to a string. 

373 saved_string = self.comp.saveToString() 

374 saved_string += "config.c.f = parameters.value" 

375 namespace = SimpleNamespace(value=7) 

376 extraLocals = {"parameters": namespace} 

377 roundTrip = Complex() 

378 roundTrip.loadFromString(saved_string, extraLocals=extraLocals) 

379 self.assertEqual(namespace.value, roundTrip.c.f) 

380 self.assertEqual(self.comp.r.name, roundTrip.r.name) 

381 with self.assertRaises(ValueError): 

382 roundTrip.loadFromString(saved_string, root="config", extraLocals={"config": 6}) 

383 del roundTrip 

384 

385 def testDuplicateRegistryNames(self): 

386 self.comp.r["AAA"].f = 5.0 

387 self.assertEqual(self.comp.p["AAA"].f, 3.0) 

388 

389 def testInheritance(self): 

390 class AAA(pexConfig.Config): 

391 a = pexConfig.Field("AAA.a", int, default=4) 

392 

393 class BBB(AAA): 

394 b = pexConfig.Field("BBB.b", int, default=3) 

395 

396 class CCC(BBB): 

397 c = pexConfig.Field("CCC.c", int, default=2) 

398 

399 # test multi-level inheritance 

400 c = CCC() 

401 self.assertIn("a", c.toDict()) 

402 self.assertEqual(c._fields["a"].dtype, int) 

403 self.assertEqual(c.a, 4) 

404 

405 # test conflicting multiple inheritance 

406 class DDD(pexConfig.Config): 

407 a = pexConfig.Field("DDD.a", float, default=0.0) 

408 

409 class EEE(DDD, AAA): 

410 pass 

411 

412 e = EEE() 

413 self.assertEqual(e._fields["a"].dtype, float) 

414 self.assertIn("a", e.toDict()) 

415 self.assertEqual(e.a, 0.0) 

416 

417 class FFF(AAA, DDD): 

418 pass 

419 

420 f = FFF() 

421 self.assertEqual(f._fields["a"].dtype, int) 

422 self.assertIn("a", f.toDict()) 

423 self.assertEqual(f.a, 4) 

424 

425 # test inheritance from non Config objects 

426 class GGG: 

427 a = pexConfig.Field("AAA.a", float, default=10.0) 

428 

429 class HHH(GGG, AAA): 

430 pass 

431 

432 h = HHH() 

433 self.assertEqual(h._fields["a"].dtype, float) 

434 self.assertIn("a", h.toDict()) 

435 self.assertEqual(h.a, 10.0) 

436 

437 # test partial Field redefinition 

438 

439 class III(AAA): 

440 pass 

441 

442 III.a.default = 5 

443 

444 self.assertEqual(III.a.default, 5) 

445 self.assertEqual(AAA.a.default, 4) 

446 

447 @unittest.skipIf(dafBase is None, "lsst.daf.base is required") 

448 def testConvertPropertySet(self): 

449 ps = pexConfig.makePropertySet(self.simple) 

450 self.assertFalse(ps.exists("i")) 

451 self.assertEqual(ps.getScalar("f"), self.simple.f) 

452 self.assertEqual(ps.getScalar("b"), self.simple.b) 

453 self.assertEqual(ps.getScalar("c"), self.simple.c) 

454 self.assertEqual(list(ps.getArray("ll")), list(self.simple.ll)) 

455 

456 ps = pexConfig.makePropertySet(self.comp) 

457 self.assertEqual(ps.getScalar("c.f"), self.comp.c.f) 

458 

459 def testFreeze(self): 

460 self.comp.freeze() 

461 

462 self.assertRaises(pexConfig.FieldValidationError, setattr, self.comp.c, "f", 10.0) 

463 self.assertRaises(pexConfig.FieldValidationError, setattr, self.comp, "r", "AAA") 

464 self.assertRaises(pexConfig.FieldValidationError, setattr, self.comp, "p", "AAA") 

465 self.assertRaises(pexConfig.FieldValidationError, setattr, self.comp.p["AAA"], "f", 5.0) 

466 

467 def checkImportRoundTrip(self, importStatement, searchString, shouldBeThere): 

468 self.comp.c.f = 5.0 

469 

470 # Generate a Config through loading 

471 stream = io.StringIO() 

472 stream.write(str(importStatement)) 

473 self.comp.saveToStream(stream) 

474 roundtrip = Complex() 

475 roundtrip.loadFromStream(stream.getvalue()) 

476 self.assertEqual(self.comp.c.f, roundtrip.c.f) 

477 

478 # Check the save stream 

479 stream = io.StringIO() 

480 roundtrip.saveToStream(stream) 

481 self.assertEqual(self.comp.c.f, roundtrip.c.f) 

482 streamStr = stream.getvalue() 

483 if shouldBeThere: 

484 self.assertTrue(re.search(searchString, streamStr)) 

485 else: 

486 self.assertFalse(re.search(searchString, streamStr)) 

487 

488 def testImports(self): 

489 # A module not used by anything else, but which exists 

490 importing = "import lsst.pex.config._doNotImportMe\n" 

491 self.checkImportRoundTrip(importing, importing, True) 

492 

493 def testBadImports(self): 

494 dummy = "somethingThatDoesntExist" 

495 importing = ( 

496 """ 

497try: 

498 import %s 

499except ImportError: 

500 pass 

501""" 

502 % dummy 

503 ) 

504 self.checkImportRoundTrip(importing, dummy, False) 

505 

506 def testPickle(self): 

507 self.simple.f = 5 

508 simple = pickle.loads(pickle.dumps(self.simple)) 

509 self.assertIsInstance(simple, Simple) 

510 self.assertEqual(self.simple.f, simple.f) 

511 

512 self.comp.c.f = 5 

513 comp = pickle.loads(pickle.dumps(self.comp)) 

514 self.assertIsInstance(comp, Complex) 

515 self.assertEqual(self.comp.c.f, comp.c.f) 

516 

517 @unittest.skipIf(yaml is None, "Test requires pyyaml") 

518 def testYaml(self): 

519 self.simple.f = 5 

520 simple = yaml.safe_load(yaml.dump(self.simple)) 

521 self.assertIsInstance(simple, Simple) 

522 self.assertEqual(self.simple.f, simple.f) 

523 

524 self.comp.c.f = 5 

525 # Use a different loader to check that it also works 

526 comp = yaml.load(yaml.dump(self.comp), Loader=yaml.FullLoader) 

527 self.assertIsInstance(comp, Complex) 

528 self.assertEqual(self.comp.c.f, comp.c.f) 

529 

530 def testCompare(self): 

531 comp2 = Complex() 

532 inner2 = InnerConfig() 

533 simple2 = Simple() 

534 self.assertTrue(self.comp.compare(comp2)) 

535 self.assertTrue(comp2.compare(self.comp)) 

536 self.assertTrue(self.comp.c.compare(inner2)) 

537 self.assertTrue(self.simple.compare(simple2)) 

538 self.assertTrue(simple2.compare(self.simple)) 

539 self.assertEqual(self.simple, simple2) 

540 self.assertEqual(simple2, self.simple) 

541 outList = [] 

542 

543 def outFunc(msg): 

544 outList.append(msg) 

545 

546 simple2.b = True 

547 simple2.ll.append(4) 

548 simple2.d["foo"] = "var" 

549 self.assertFalse(self.simple.compare(simple2, shortcut=True, output=outFunc)) 

550 self.assertEqual(len(outList), 1) 

551 del outList[:] 

552 self.assertFalse(self.simple.compare(simple2, shortcut=False, output=outFunc)) 

553 output = "\n".join(outList) 

554 self.assertIn("Inequality in b", output) 

555 self.assertIn("Inequality in size for ll", output) 

556 self.assertIn("Inequality in keys for d", output) 

557 del outList[:] 

558 self.simple.d["foo"] = "vast" 

559 self.simple.ll.append(5) 

560 self.simple.b = True 

561 self.simple.f += 1e8 

562 self.assertFalse(self.simple.compare(simple2, shortcut=False, output=outFunc)) 

563 output = "\n".join(outList) 

564 self.assertIn("Inequality in f", output) 

565 self.assertIn("Inequality in ll[3]", output) 

566 self.assertIn("Inequality in d['foo']", output) 

567 del outList[:] 

568 comp2.r["BBB"].f = 1.0 # changing the non-selected item shouldn't break equality 

569 self.assertTrue(self.comp.compare(comp2)) 

570 comp2.r["AAA"].i = 56 # changing the selected item should break equality 

571 comp2.c.f = 1.0 

572 self.assertFalse(self.comp.compare(comp2, shortcut=False, output=outFunc)) 

573 output = "\n".join(outList) 

574 self.assertIn("Inequality in c.f", output) 

575 self.assertIn("Inequality in r['AAA']", output) 

576 self.assertNotIn("Inequality in r['BBB']", output) 

577 

578 # Before DM-16561, this incorrectly returned `True`. 

579 self.assertFalse(self.inner.compare(self.outer)) 

580 # Before DM-16561, this raised. 

581 self.assertFalse(self.outer.compare(self.inner)) 

582 

583 def testLoadError(self): 

584 """Check that loading allows errors in the file being loaded to 

585 propagate. 

586 """ 

587 self.assertRaises(SyntaxError, self.simple.loadFromStream, "bork bork bork") 

588 self.assertRaises(NameError, self.simple.loadFromStream, "config.f = bork") 

589 

590 def testNames(self): 

591 """Check that the names() method returns valid keys. 

592 

593 Also check that we have the right number of keys, and as they are 

594 all known to be valid we know that we got them all. 

595 """ 

596 names = self.simple.names() 

597 self.assertEqual(len(names), 8) 

598 for name in names: 

599 self.assertTrue(hasattr(self.simple, name)) 

600 

601 def testIteration(self): 

602 self.assertIn("ll", self.simple) 

603 self.assertIn("ll", self.simple.keys()) 

604 self.assertIn("Hello", self.simple.values()) 

605 self.assertEqual(len(self.simple.values()), 8) 

606 

607 for k, v, (k1, v1) in zip(self.simple.keys(), self.simple.values(), self.simple.items()): 

608 self.assertEqual(k, k1) 

609 if k == "n": 

610 self.assertNotEqual(v, v1) 

611 else: 

612 self.assertEqual(v, v1) 

613 

614 

615if __name__ == "__main__": 615 ↛ 616line 615 didn't jump to line 616, because the condition on line 615 was never true

616 unittest.main()