Coverage for tests/test_bpsconfig.py: 100%

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <https://www.gnu.org/licenses/>. 

27import os 

28import tempfile 

29import unittest 

30 

31import yaml 

32 

33from lsst.ctrl.bps import BPS_SEARCH_ORDER, BpsConfig 

34from lsst.daf.butler import Config 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class TestBpsConfigConstructor(unittest.TestCase): 

40 """Test BpsConfig construction.""" 

41 

42 def setUp(self): 

43 self.filename = os.path.join(TESTDIR, "data/config.yaml") 

44 with open(self.filename) as f: 

45 self.dictionary = yaml.safe_load(f) 

46 

47 def tearDown(self): 

48 pass 

49 

50 def testFromFilename(self): 

51 """Test initialization from a file, but without defaults.""" 

52 config = BpsConfig(self.filename) 

53 self.assertEqual(set(config), {"foo", "bar", "baz"} | set(BPS_SEARCH_ORDER)) 

54 self.assertEqual(set(config["foo"]), {"qux"}) 

55 self.assertEqual(set(config["bar"]), {"qux"}) 

56 self.assertEqual(set(config["baz"]), {"grault", "garply"}) 

57 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

58 

59 def testFromDict(self): 

60 """Test initialization from a dictionary.""" 

61 config = BpsConfig(self.dictionary) 

62 self.assertEqual(set(config), {"foo", "bar", "baz"} | set(BPS_SEARCH_ORDER)) 

63 self.assertEqual(set(config["foo"]), {"qux"}) 

64 self.assertEqual(set(config["bar"]), {"qux"}) 

65 self.assertEqual(set(config["baz"]), {"grault", "garply"}) 

66 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

67 

68 def testFromConfig(self): 

69 """Test initialization from other Config object.""" 

70 config_daf = Config(self.dictionary) 

71 config = BpsConfig(config_daf) 

72 self.assertEqual(set(config), {"foo", "bar", "baz"} | set(BPS_SEARCH_ORDER)) 

73 self.assertEqual(set(config["foo"]), {"qux"}) 

74 self.assertEqual(set(config["bar"]), {"qux"}) 

75 self.assertEqual(set(config["baz"]), {"grault", "garply"}) 

76 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

77 

78 def testFromBpsConfig(self): 

79 """Test initialization from other BpsConfig object.""" 

80 config_ref = BpsConfig(self.dictionary) 

81 config = BpsConfig(config_ref) 

82 self.assertEqual(set(config), set(config_ref)) 

83 self.assertEqual(config.search_order, config_ref.search_order) 

84 

85 def testDefaultsInclusion(self): 

86 """Test including defaults.""" 

87 config = BpsConfig(self.filename, defaults={"quux": 1}) 

88 self.assertEqual(set(config), {"foo", "bar", "baz", "quux"} | set(BPS_SEARCH_ORDER)) 

89 self.assertEqual(config["quux"], 1) 

90 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

91 

92 def testDefaultsOverride(self): 

93 """Test overriding defaults.""" 

94 config = BpsConfig(self.filename, defaults={"foo": {"qux": 2}, "quux": 1}) 

95 self.assertEqual(set(config), {"foo", "bar", "baz", "quux"} | set(BPS_SEARCH_ORDER)) 

96 self.assertEqual(config["quux"], 1) 

97 self.assertEqual(config["foo"], BpsConfig({"qux": 1}, search_order=[])) 

98 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

99 

100 def testWmsFromCmdline(self): 

101 """Test initialization when WMS class specified at the cmdline.""" 

102 with unittest.mock.patch.dict( 

103 os.environ, {"BPS_WMS_SERVICE_CLASS": "wms_test_utils.WmsServiceFromEnv"} 

104 ): 

105 filename = os.path.join(TESTDIR, "data/config_with_wms.yaml") 

106 config = BpsConfig( 

107 filename, 

108 defaults={"wmsServiceClass": "wms_test_utils.WmsServiceFromDefaults"}, 

109 wms_service_class_fqn="wms_test_utils.WmsServiceFromCmdline", 

110 ) 

111 self.assertEqual(set(config), {"corge", "wmsServiceClass"} | set(BPS_SEARCH_ORDER)) 

112 self.assertEqual(config["corge"], "cmdline") 

113 

114 def testWmsFromConfig(self): 

115 """Test initialization when WMS class specified in the config.""" 

116 with unittest.mock.patch.dict( 

117 os.environ, {"BPS_WMS_SERVICE_CLASS": "wms_test_utils.WmsServiceFromEnv"} 

118 ): 

119 filename = os.path.join(TESTDIR, "data/config_with_wms.yaml") 

120 config = BpsConfig( 

121 filename, 

122 defaults={"wmsServiceClass": "wms_test_utils.WmsServiceFromDefaults"}, 

123 wms_service_class_fqn=None, 

124 ) 

125 self.assertEqual(set(config), {"corge", "wmsServiceClass"} | set(BPS_SEARCH_ORDER)) 

126 self.assertEqual(config["corge"], "config") 

127 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

128 

129 def testWmsFromEnv(self): 

130 """Test initialization with WMS class in the env.""" 

131 with unittest.mock.patch.dict( 

132 os.environ, {"BPS_WMS_SERVICE_CLASS": "wms_test_utils.WmsServiceFromEnv"} 

133 ): 

134 config = BpsConfig( 

135 {}, 

136 defaults={"wmsServiceClass": "wms_test_utils.WmsServiceFromDefaults"}, 

137 wms_service_class_fqn=None, 

138 ) 

139 self.assertEqual(set(config), {"corge", "wmsServiceClass"} | set(BPS_SEARCH_ORDER)) 

140 self.assertEqual(config["corge"], "env") 

141 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

142 

143 def testWmsFromDefaults(self): 

144 """Test initialization with a default WMS class.""" 

145 config = BpsConfig({}, defaults={"wmsServiceClass": "wms_test_utils.WmsServiceFromDefaults"}) 

146 self.assertEqual(set(config), {"corge", "wmsServiceClass"} | set(BPS_SEARCH_ORDER)) 

147 self.assertEqual(config["corge"], "defaults") 

148 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

149 

150 def testDefaultsExclusion(self): 

151 config = BpsConfig( 

152 self.filename, defaults=None, wms_service_class_fqn="wms_test_utils.WmsServiceFromCmdline" 

153 ) 

154 self.assertEqual(set(config), {"foo", "bar", "baz"} | set(BPS_SEARCH_ORDER)) 

155 self.assertEqual(set(config["foo"]), {"qux"}) 

156 self.assertEqual(set(config["bar"]), {"qux"}) 

157 self.assertEqual(set(config["baz"]), {"grault", "garply"}) 

158 self.assertEqual(config.search_order, BPS_SEARCH_ORDER) 

159 

160 def testInvalidArg(self): 

161 """Test if exception is raised for an argument of unsupported type.""" 

162 sequence = ["wibble", "wobble", "wubble", "flob"] 

163 with self.assertRaises(ValueError): 

164 BpsConfig(sequence) 

165 

166 

167class TestBpsConfigGet(unittest.TestCase): 

168 """Test retrieval of items from BpsConfig.""" 

169 

170 def setUp(self): 

171 self.config = BpsConfig({"foo": "bar"}) 

172 

173 def tearDown(self): 

174 pass 

175 

176 def testKeyExistsNoDefault(self): 

177 """Test if the value is returned when the key is in the dictionary.""" 

178 self.assertEqual("bar", self.config.get("foo")) 

179 

180 def testKeyExistsDefaultProvided(self): 

181 """Test if the value is returned when the key is in the dictionary.""" 

182 self.assertEqual("bar", self.config.get("foo", "qux")) 

183 

184 def testKeyMissingNoDefault(self): 

185 """Test if the provided default is returned if the key is missing.""" 

186 self.assertEqual("", self.config.get("baz")) 

187 

188 def testKeyMissingDefaultProvided(self): 

189 """Test if the custom default is returned if the key is missing.""" 

190 self.assertEqual("qux", self.config.get("baz", "qux")) 

191 

192 

193class TestBpsConfigSearch(unittest.TestCase): 

194 """Test searching of BpsConfig.""" 

195 

196 def setUp(self): 

197 filename = os.path.join(TESTDIR, "data/config.yaml") 

198 self.config = BpsConfig(filename, search_order=["baz", "bar", "foo"], defaults={}) 

199 os.environ["GARPLY"] = "garply" 

200 

201 def tearDown(self): 

202 del os.environ["GARPLY"] 

203 

204 def testSectionSearchOrder(self): 

205 """Test if sections are searched in the prescribed order.""" 

206 key = "qux" 

207 found, value = self.config.search(key) 

208 self.assertEqual(found, True) 

209 self.assertEqual(value, 2) 

210 

211 def testCurrentValues(self): 

212 """Test if a current value overrides of the one in configuration.""" 

213 found, value = self.config.search("qux", opt={"curvals": {"qux": -3}}) 

214 self.assertEqual(found, True) 

215 self.assertEqual(value, -3) 

216 

217 def testSearchobjValues(self): 

218 """Test if a serachobj value overrides of the one in configuration.""" 

219 options = {"searchobj": {"qux": 4}} 

220 found, value = self.config.search("qux", opt=options) 

221 self.assertEqual(found, True) 

222 self.assertEqual(value, 4) 

223 

224 def testSubsectionSearch(self): 

225 options = {"curvals": {"curr_baz": "garply"}} 

226 found, value = self.config.search("qux", opt=options) 

227 self.assertEqual(found, True) 

228 self.assertEqual(value, 3) 

229 

230 def testDefault(self): 

231 """Test if a default value is properly set.""" 

232 found, value = self.config.search("plugh", opt={"default": 4}) 

233 self.assertEqual(found, True) 

234 self.assertEqual(value, 4) 

235 

236 def testReplaceVars(self): 

237 """Test replaceVars method.""" 

238 test_opt = {"default": 555} 

239 orig_str = "<ENV:GARPLY>/waldo/{qux:03}/{notthere}" 

240 value = self.config.replace_vars(orig_str, opt=test_opt) 

241 self.assertEqual(value, "<ENV:GARPLY>/waldo/002/") 

242 self.assertEqual(test_opt["default"], 555) 

243 

244 def testReplaceVarsSkipNames(self): 

245 test_opt = {"default": 555, "skipNames": ["qgraphFile", "butlerConfig"]} 

246 orig_str = "<ENV:GARPLY>/waldo/{qux:03} {qgraphFile} {butlerConfig}" 

247 value = self.config.replace_vars(orig_str, opt=test_opt) 

248 self.assertEqual(value, "<ENV:GARPLY>/waldo/002 {qgraphFile} {butlerConfig}") 

249 self.assertEqual(test_opt["default"], 555) 

250 

251 def testVariables(self): 

252 """Test combinations of expandEnvVars, replaceEnvVars, 

253 and replaceVars. 

254 """ 

255 test_opt = {"expandEnvVars": False, "replaceEnvVars": False, "replaceVars": False} 

256 found, value = self.config.search("grault", opt=test_opt) 

257 self.assertEqual(found, True) 

258 self.assertEqual(value, "${GARPLY}/waldo/{qux:03}") 

259 

260 test_opt = {"expandEnvVars": False, "replaceEnvVars": False, "replaceVars": True} 

261 found, value = self.config.search("grault", opt=test_opt) 

262 self.assertEqual(found, True) 

263 self.assertEqual(value, "${GARPLY}/waldo/002") 

264 

265 test_opt = {"expandEnvVars": False, "replaceEnvVars": True, "replaceVars": False} 

266 found, value = self.config.search("grault", opt=test_opt) 

267 self.assertEqual(found, True) 

268 self.assertEqual(value, "<ENV:GARPLY>/waldo/{qux:03}") 

269 

270 test_opt = {"expandEnvVars": False, "replaceEnvVars": True, "replaceVars": True} 

271 found, value = self.config.search("grault", opt=test_opt) 

272 self.assertEqual(found, True) 

273 self.assertEqual(value, "<ENV:GARPLY>/waldo/002") 

274 

275 test_opt = {"expandEnvVars": True, "replaceEnvVars": False, "replaceVars": False} 

276 found, value = self.config.search("grault", opt=test_opt) 

277 self.assertEqual(found, True) 

278 self.assertEqual(value, "garply/waldo/{qux:03}") 

279 

280 test_opt = {"expandEnvVars": True, "replaceEnvVars": False, "replaceVars": True} 

281 found, value = self.config.search("grault", opt=test_opt) 

282 self.assertEqual(found, True) 

283 self.assertEqual(value, "garply/waldo/002") 

284 

285 test_opt = {"expandEnvVars": True, "replaceEnvVars": True, "replaceVars": False} 

286 found, value = self.config.search("grault", opt=test_opt) 

287 self.assertEqual(found, True) 

288 self.assertEqual(value, "garply/waldo/{qux:03}") 

289 

290 test_opt = {"expandEnvVars": True, "replaceEnvVars": True, "replaceVars": True} 

291 found, value = self.config.search("grault", opt=test_opt) 

292 self.assertEqual(found, True) 

293 self.assertEqual(value, "garply/waldo/002") 

294 

295 def testRequired(self): 

296 """Test if exception is raised if a required setting is missing.""" 

297 with self.assertRaises(KeyError): 

298 self.config.search("fred", opt={"required": True}) 

299 

300 def testS3Path(self): 

301 """Test that double slashes aren't replaced with single slash.""" 

302 config = BpsConfig(self.config) 

303 s3 = "s3://user1@rubin-place-users/butler-pipeline1-processing.yaml" 

304 config["butlerConfig"] = s3 

305 test_opt = {"expandEnvVars": True, "replaceEnvVars": True, "replaceVars": True} 

306 found, value = config.search("butlerConfig", opt=test_opt) 

307 self.assertEqual(found, True) 

308 self.assertEqual(value, s3) 

309 

310 def testSubDirTemplate(self): 

311 """Test that double slashes are replaced with single slash 

312 in subDirTemplate. 

313 """ 

314 config = BpsConfig(self.config) 

315 

316 # template doesn't end in slash, post-slashes value didn't end in slash 

317 config["subDirTemplate"] = "{label}/{val1}/{val2}/{val3}/{val4}/{val5}" 

318 test_opt = { 

319 "expandEnvVars": True, 

320 "replaceEnvVars": True, 

321 "replaceVars": True, 

322 "curvals": {"label": "label1", "val5": 12345}, 

323 } 

324 found, value = config.search("subDirTemplate", opt=test_opt) 

325 self.assertEqual(found, True) 

326 self.assertEqual(value, "label1/12345") 

327 

328 # template doesn't end in slash, post-slashes value ended in slash 

329 config["subDirTemplate"] = "{label}/{val1}/{val2}/{val3}/{val4}/{val5}" 

330 test_opt = { 

331 "expandEnvVars": True, 

332 "replaceEnvVars": True, 

333 "replaceVars": True, 

334 "curvals": {"label": "label2", "val4": 12345}, 

335 } 

336 found, value = config.search("subDirTemplate", opt=test_opt) 

337 self.assertEqual(found, True) 

338 self.assertEqual(value, "label2/12345") 

339 

340 # template ends in slash, post-slashes value didn't end in slash 

341 config["subDirTemplate"] = "{label}/{val1}/{val2}/{val3}/{val4}/{val5}/" 

342 test_opt = { 

343 "expandEnvVars": True, 

344 "replaceEnvVars": True, 

345 "replaceVars": True, 

346 "curvals": {"label": "label3", "val4": 12345}, 

347 } 

348 found, value = config.search("subDirTemplate", opt=test_opt) 

349 self.assertEqual(found, True) 

350 self.assertEqual(value, "label3/12345/") 

351 

352 # template ends in slash, post-slashes value ended in slash 

353 config["subDirTemplate"] = "{label}/{val1}/{val2}/{val3}/{val4}/{val5}/" 

354 test_opt = { 

355 "expandEnvVars": True, 

356 "replaceEnvVars": True, 

357 "replaceVars": True, 

358 "curvals": {"label": "label4", "val2": 12345}, 

359 } 

360 found, value = config.search("subDirTemplate", opt=test_opt) 

361 self.assertEqual(found, True) 

362 self.assertEqual(value, "label4/12345/") 

363 

364 def testOtherTemplate(self): 

365 """Test that other subDirTemplate-like value doesn't have 

366 double slashes replaced. 

367 """ 

368 config = BpsConfig(self.config) 

369 

370 config["otherTemplate"] = "{label}/{val1}/{val2}/{val3}/{val4}/{val5}" 

371 test_opt = { 

372 "expandEnvVars": True, 

373 "replaceEnvVars": True, 

374 "replaceVars": True, 

375 "curvals": {"label": "label1", "val4": 12345, "val5": ""}, 

376 } 

377 found, value = config.search("otherTemplate", opt=test_opt) 

378 self.assertEqual(found, True) 

379 self.assertEqual(value, "label1////12345/") 

380 

381 

382class TestBpsConfigGenerateConfig(unittest.TestCase): 

383 """Test BpsConfig.generate_config and bpsEval methods.""" 

384 

385 def setUp(self): 

386 # Just to shorten string length in tests 

387 self.test_prefix = "lsst.ctrl.bps.tests.config_test_utils" 

388 

389 filename = os.path.join(TESTDIR, "data/initialize_config.yaml") 

390 self.config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

391 

392 def testUnparsableValue(self): 

393 config = BpsConfig( 

394 { 

395 "p1": 3, 

396 "p3": 16, 

397 }, 

398 BPS_SEARCH_ORDER, 

399 ) 

400 # invalid function name 

401 config["bpsGenerateConfig"] = "not a valid name({p1}, param3={p3})" 

402 with self.assertRaisesRegex(ValueError, "Unparsable bpsGenerateConfig value='not a valid"): 

403 config.generate_config() 

404 

405 def testMissingParen(self): 

406 config = BpsConfig( 

407 { 

408 "p1": 3, 

409 "p3": 16, 

410 }, 

411 BPS_SEARCH_ORDER, 

412 ) 

413 # invalid function name 

414 config["bpsGenerateConfig"] = self.test_prefix + ".generate_config_1(1, param3=2" 

415 with self.assertRaisesRegex(ValueError, "Unparsable bpsGenerateConfig value='"): 

416 config.generate_config() 

417 

418 def testBadFunctionName(self): 

419 config = BpsConfig( 

420 { 

421 "p1": 3, 

422 "p3": 16, 

423 }, 

424 BPS_SEARCH_ORDER, 

425 ) 

426 # invalid function name 

427 config["bpsGenerateConfig"] = self.test_prefix + ".notthere({p1}, param3={p3})" 

428 with self.assertRaisesRegex(ImportError, "notthere"): 

429 config.generate_config() 

430 

431 def testBadModuleName(self): 

432 config = BpsConfig( 

433 { 

434 "p1": 3, 

435 "p3": 16, 

436 }, 

437 BPS_SEARCH_ORDER, 

438 ) 

439 # invalid module name 

440 config["bpsGenerateConfig"] = "lsst.ctrl.bps.notthere.generate_config(1, 2)" 

441 with self.assertRaisesRegex(ImportError, "notthere"): 

442 config.generate_config() 

443 

444 def testBadParamName(self): 

445 config = BpsConfig( 

446 {"p1": 3, "p3": 16, "bpsGenerateConfig": self.test_prefix + ".generate_config_1(1, param5=2)"}, 

447 BPS_SEARCH_ORDER, 

448 ) 

449 with self.assertRaisesRegex(TypeError, "unexpected keyword argument 'param5'"): 

450 config.generate_config() 

451 

452 def testExtraParam(self): 

453 config = BpsConfig( 

454 { 

455 "p1": 3, 

456 "p3": 16, 

457 "bpsGenerateConfig": self.test_prefix + ".generate_config_1({p1}, 2, {p3}, 4)", 

458 }, 

459 BPS_SEARCH_ORDER, 

460 ) 

461 with self.assertRaisesRegex(TypeError, "positional arguments"): 

462 config.generate_config() 

463 

464 def testWithSearchOrder(self): 

465 # Check that bpsGenerateConfig is replaced in search sections 

466 # (e.g., pipetask) And when replacing vars in subsections 

467 # config ordering is used. 

468 # Ditto for finalJob (which isn't a search section). 

469 # Checking all in single function to ensure doesn't quit early. 

470 self.maxDiff = None 

471 

472 self.config.generate_config() 

473 

474 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

475 truth = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

476 self.assertEqual(self.config, truth) 

477 

478 def testBpsEval(self): 

479 """Test replacing bpsEval when need to import module.""" 

480 test_opt = { 

481 "expandEnvVars": True, 

482 "replaceEnvVars": True, 

483 "replaceVars": True, 

484 "curvals": {"curr_pipetask": "ptask1"}, 

485 } 

486 found, value = self.config.search("genval1", opt=test_opt) 

487 self.assertEqual(found, True) 

488 self.assertEqual(value, "-c val1:0.1 -c val2:3") 

489 

490 def testBpsEvalBuiltin(self): 

491 """Test replacing bpsEval with builtin function.""" 

492 test_opt = { 

493 "expandEnvVars": True, 

494 "replaceEnvVars": True, 

495 "replaceVars": True, 

496 "curvals": {"curr_pipetask": "ptask1"}, 

497 } 

498 found, value = self.config.search("genval2", opt=test_opt) 

499 self.assertEqual(found, True) 

500 self.assertEqual(value, "-c val1:32") 

501 

502 def testBpsEvalInvalid(self): 

503 """Test reporting not replacing bpsEval.""" 

504 test_opt = { 

505 "expandEnvVars": True, 

506 "replaceEnvVars": True, 

507 "replaceVars": True, 

508 "curvals": {"curr_pipetask": "ptask1"}, 

509 } 

510 

511 self.config["badkey1"] = "badval1 bpsEval('sum([1,2]') blah" 

512 with self.assertRaisesRegex(ValueError, "Unparsable bpsEval in 'badval1 bpsEval"): 

513 _ = self.config.search("badkey1", opt=test_opt) 

514 

515 

516class TestBpsConfigModifyValue(unittest.TestCase): 

517 """Tests for BpsConfig.modify_value.""" 

518 

519 def testReplaceVarsRoot(self): 

520 config = BpsConfig( 

521 { 

522 "var2": "two", 

523 } 

524 ) 

525 

526 value = "one {var2} three" 

527 opt = {"replaceVars": True} 

528 truth = "one two three" 

529 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

530 results = config.modify_value("KEY1", value, opt) 

531 self.assertEqual(truth, results) 

532 

533 def testReplaceVarsSubDirTemplateNoSlash(self): 

534 config = BpsConfig({}) 

535 

536 value = "{label}/{t1}/{t2}/{t3}/{t4}/{t5}/{t6}/{t7}" 

537 search_obj = {"label": "label1", "t6": 12345} 

538 opt = {"replaceVars": True, "searchobj": search_obj} 

539 truth = "label1/12345" 

540 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

541 results = config.modify_value("subDirTemplate", value, opt) 

542 self.assertEqual(truth, results) 

543 

544 def testExpandEnvVars(self): 

545 config = BpsConfig( 

546 { 

547 "var2": "two", 

548 } 

549 ) 

550 

551 value = "one <ENV:var2> three <ENV:not_there>" 

552 opt = {"expandEnvVars": True} 

553 truth = "one env_two three ${not_there}" 

554 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

555 results = config.modify_value("KEY1", value, opt) 

556 self.assertEqual(truth, results) 

557 

558 def testReplaceEnvVars(self): 

559 # Backward compatibility 

560 config = BpsConfig( 

561 { 

562 "var2": "two", 

563 } 

564 ) 

565 

566 value = "one ${var2} three" 

567 opt = {"expandEnvVars": False, "replaceEnvVars": True} 

568 truth = "one <ENV:var2> three" 

569 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

570 results = config.modify_value("KEY1", value, opt) 

571 self.assertEqual(truth, results) 

572 

573 def testReplaceEnvShell2Bps(self): 

574 config = BpsConfig( 

575 { 

576 "var2": "two", 

577 } 

578 ) 

579 value = "one ${var2} three" 

580 opt = {"expandEnvVars": False, "replaceEnvVars": True} 

581 truth = "one <ENV:var2> three" 

582 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

583 results = config.modify_value("KEY1", value, opt) 

584 self.assertEqual(truth, results) 

585 

586 def testReplaceEnvBps2Shell(self): 

587 config = BpsConfig( 

588 { 

589 "var2": "two", 

590 } 

591 ) 

592 value = "one ${var2} three" 

593 opt = {"expandEnvVars": False, "replaceEnvVars": True} 

594 truth = "one <ENV:var2> three" 

595 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

596 results = config.modify_value("KEY1", value, opt) 

597 self.assertEqual(truth, results) 

598 

599 def testSearchObjEnv(self): 

600 config = BpsConfig({}) 

601 

602 value = "{label}/{t1}/{t2}/{t3}/{t4}/{t5}/{t6}/{t7}" 

603 search_obj = {"label": "label1", "t6": 12345} 

604 opt = {"replaceVars": True, "searchobj": search_obj} 

605 truth = "label1/12345" 

606 with unittest.mock.patch.dict(os.environ, {"var2": "env_two"}): 

607 results = config.modify_value("subDirTemplate", value, opt) 

608 self.assertEqual(truth, results) 

609 

610 

611class TestBpsConfigGetSearchOpts(unittest.TestCase): 

612 """Tests for BpsConfig.get_search_opts.""" 

613 

614 def testPipetaskLabel(self): 

615 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

616 config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

617 config["computeCloud"] = "cloud1" 

618 config["computeSite"] = "site1" 

619 results = config.get_search_opts("ptask1") 

620 truth = { 

621 "curvals": { 

622 "label": "ptask1", 

623 "curr_pipetask": "ptask1", 

624 "curr_site": "site1", 

625 "curr_cloud": "cloud1", 

626 } 

627 } 

628 self.assertEqual(truth, results) 

629 

630 def testClusterLabel(self): 

631 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

632 config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

633 config["computeCloud"] = "cloud1" 

634 config["computeSite"] = "site1" 

635 results = config.get_search_opts("cl1") 

636 truth = { 

637 "curvals": {"label": "cl1", "curr_cluster": "cl1", "curr_site": "site1", "curr_cloud": "cloud1"} 

638 } 

639 self.assertEqual(truth, results) 

640 

641 def testNoMatchingLabel(self): 

642 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

643 config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

644 config["computeCloud"] = "cloud1" 

645 config["computeSite"] = "site1" 

646 results = config.get_search_opts("notthere") 

647 truth = {"curvals": {"label": "notthere", "curr_site": "site1", "curr_cloud": "cloud1"}} 

648 self.assertEqual(truth, results) 

649 

650 def testNoMatchingAnything(self): 

651 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

652 config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

653 config["computeCloud"] = "cloud_notthere" 

654 config["computeSite"] = "site_notthere" 

655 results = config.get_search_opts("notthere") 

656 truth = { 

657 "curvals": {"label": "notthere", "curr_site": "site_notthere", "curr_cloud": "cloud_notthere"} 

658 } 

659 self.assertEqual(truth, results) 

660 

661 def testFinal(self): 

662 filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml") 

663 config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={}) 

664 config["computeCloud"] = "cloud1" 

665 config["computeSite"] = "site1" 

666 results = config.get_search_opts("finalJob") 

667 truth = { 

668 "curvals": { 

669 "label": "finalJob", 

670 "curr_site": "site1", 

671 "curr_cloud": "cloud1", 

672 }, 

673 "searchobj": config["finalJob"], 

674 } 

675 self.assertEqual(truth, results) 

676 

677 

678class TestBpsConfigEnvironmentChaining(unittest.TestCase): 

679 """Test that ``${VAR}`` references in environment sections chain onto 

680 values from earlier config files instead of replacing them. 

681 """ 

682 

683 def setUp(self): 

684 self.tmpdir = tempfile.TemporaryDirectory() 

685 self.addCleanup(self.tmpdir.cleanup) 

686 self._write( 

687 "cache.yaml", 

688 { 

689 "environment": { 

690 "BPS_TEST_PATH": "${CACHE}/config:${BPS_TEST_PATH}", 

691 "BPS_TEST_DIRS": "${CACHE}/dirs:${BPS_TEST_DIRS}", 

692 }, 

693 "BPS_TEST_PATH": "cache:${BPS_TEST_PATH}", 

694 }, 

695 ) 

696 self._write( 

697 "compression.yaml", 

698 { 

699 "environment": { 

700 "BPS_TEST_PATH": "${COMPRESS}/config:${BPS_TEST_PATH}", 

701 "BPS_TEST_DIRS": "${COMPRESS}/dirs:${BPS_TEST_DIRS}", 

702 "OTHER": "from compression", 

703 }, 

704 "BPS_TEST_PATH": "compression:${BPS_TEST_PATH}", 

705 }, 

706 ) 

707 self.submit = self._write( 

708 "submit.yaml", 

709 { 

710 "includeConfigs": ["cache.yaml", "compression.yaml"], 

711 "environment": { 

712 "BPS_TEST_PATH": "${SUBMIT}/config:${BPS_TEST_PATH}", 

713 "OTHER": "from submit", 

714 }, 

715 "site": { 

716 "site1": { 

717 "includeConfigs": "cache.yaml", 

718 "environment": {"BPS_TEST_DIRS": "site:${BPS_TEST_DIRS}"}, 

719 } 

720 }, 

721 }, 

722 ) 

723 

724 def _write(self, name, data): 

725 filename = os.path.join(self.tmpdir.name, name) 

726 with open(filename, "w") as f: 

727 yaml.safe_dump(data, f) 

728 return filename 

729 

730 @staticmethod 

731 def _raw(config, key): 

732 """Return the stored value without BPS variable expansion.""" 

733 return Config.__getitem__(config, key) 

734 

735 def testIncludedFilesChain(self): 

736 """Later includes chain onto earlier ones and the including file 

737 chains onto all of them, whatever the variable is called. 

738 """ 

739 config = BpsConfig(self.submit) 

740 self.assertEqual( 

741 self._raw(config, ".environment.BPS_TEST_PATH"), 

742 "${SUBMIT}/config:${COMPRESS}/config:${CACHE}/config:${BPS_TEST_PATH}", 

743 ) 

744 self.assertEqual( 

745 self._raw(config, ".environment.BPS_TEST_DIRS"),