Coverage for tests/test_quantum_clustering_funcs.py: 26%

111 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-07 17:21 +0000

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <https://www.gnu.org/licenses/>. 

27"""Unit tests for the clustering methods. 

28""" 

29 

30# Turn off "doesn't conform to snake_case naming style" because matching 

31# the unittest casing. 

32# pylint: disable=invalid-name 

33 

34import os 

35import unittest 

36 

37from cqg_test_utils import check_cqg 

38from lsst.ctrl.bps import BpsConfig 

39from lsst.ctrl.bps.quantum_clustering_funcs import dimension_clustering, single_quantum_clustering 

40from qg_test_utils import make_test_quantum_graph 

41 

42TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

43 

44 

45class TestSingleQuantumClustering(unittest.TestCase): 

46 """Tests for single_quantum_clustering method.""" 

47 

48 def setUp(self): 

49 self.qgraph = make_test_quantum_graph() 

50 

51 def tearDown(self): 

52 pass 

53 

54 def testClustering(self): 

55 """Test valid single quantum clustering.""" 

56 # Note: the cluster config should be ignored. 

57 config = BpsConfig( 

58 { 

59 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

60 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}}, 

61 } 

62 ) 

63 

64 cqg = single_quantum_clustering(config, self.qgraph, "single") 

65 self.assertIsNotNone(cqg) 

66 self.assertIn(cqg.name, "single") 

67 self.assertEqual(len(cqg), len(self.qgraph)) 

68 

69 def testClusteringNoTemplate(self): 

70 """Test valid single quantum clustering wihtout a template for the 

71 cluster names. 

72 """ 

73 # Note: the cluster config should be ignored. 

74 config = BpsConfig({"cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}}}) 

75 

76 cqg = single_quantum_clustering(config, self.qgraph, "single-no-template") 

77 self.assertIsNotNone(cqg) 

78 self.assertIn(cqg.name, "single-no-template") 

79 self.assertEqual(len(cqg), len(self.qgraph)) 

80 

81 

82class TestDimensionClustering(unittest.TestCase): 

83 """Tests for dimension_clustering method.""" 

84 

85 def setUp(self): 

86 self.qgraph = make_test_quantum_graph() 

87 

88 def tearDown(self): 

89 pass 

90 

91 def testClusterAllInOne(self): 

92 """All tasks in one cluster.""" 

93 name = "all-in-one" 

94 config = BpsConfig( 

95 { 

96 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

97 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}}, 

98 } 

99 ) 

100 answer = { 

101 "name": name, 

102 "nodes": { 

103 "cl1_1_2": { 

104 "label": "cl1", 

105 "dims": {"D1": 1, "D2": 2}, 

106 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

107 }, 

108 "cl1_3_4": { 

109 "label": "cl1", 

110 "dims": {"D1": 3, "D2": 4}, 

111 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

112 }, 

113 }, 

114 "edges": [], 

115 } 

116 

117 cqg = dimension_clustering(config, self.qgraph, name) 

118 check_cqg(cqg, answer) 

119 

120 def testClusterTemplate(self): 

121 """Test uses clusterTemplate value to name clusters.""" 

122 name = "cluster-template" 

123 config = BpsConfig( 

124 { 

125 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}", 

126 "cluster": { 

127 "cl1": { 

128 "clusterTemplate": "ct_{D1}_{D2}_{D3}_{D4}", 

129 "pipetasks": "T1, T2, T3, T4", 

130 "dimensions": "D1, D2", 

131 } 

132 }, 

133 } 

134 ) 

135 # Note: clusterTemplate can produce trailing underscore 

136 answer = { 

137 "name": name, 

138 "nodes": { 

139 "ct_1_2_": { 

140 "label": "cl1", 

141 "dims": {"D1": 1, "D2": 2}, 

142 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

143 }, 

144 "ct_3_4_": { 

145 "label": "cl1", 

146 "dims": {"D1": 3, "D2": 4}, 

147 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

148 }, 

149 }, 

150 "edges": [], 

151 } 

152 

153 cqg = dimension_clustering(config, self.qgraph, name) 

154 check_cqg(cqg, answer) 

155 

156 def testClusterNoDims(self): 

157 """Test if clusters have no dimensions.""" 

158 name = "cluster-no-dims" 

159 config = BpsConfig( 

160 { 

161 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}", 

162 "cluster": { 

163 "cl1": { 

164 "pipetasks": "T1, T2", 

165 }, 

166 "cl2": { 

167 "pipetasks": "T3, T4", 

168 }, 

169 }, 

170 } 

171 ) 

172 answer = { 

173 "name": name, 

174 "nodes": { 

175 "cl1": {"label": "cl1", "dims": {}, "counts": {"T1": 2, "T2": 2}}, 

176 "cl2": {"label": "cl2", "dims": {}, "counts": {"T3": 2, "T4": 2}}, 

177 }, 

178 "edges": [("cl1", "cl2")], 

179 } 

180 

181 cqg = dimension_clustering(config, self.qgraph, name) 

182 check_cqg(cqg, answer) 

183 

184 def testClusterTaskRepeat(self): 

185 """Can't have PipelineTask in more than one cluster.""" 

186 name = "task-repeat" 

187 config = BpsConfig( 

188 { 

189 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}", 

190 "cluster": { 

191 "cl1": { 

192 "pipetasks": "T1, T2", 

193 }, 

194 "cl2": { 

195 "pipetasks": "T2, T3, T4", 

196 }, 

197 }, 

198 } 

199 ) 

200 

201 with self.assertRaises(RuntimeError): 

202 _ = dimension_clustering(config, self.qgraph, name) 

203 

204 def testClusterMissingDimValue(self): 

205 """Quantum can't be missing a value for a clustering dimension.""" 

206 config = BpsConfig( 

207 { 

208 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

209 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, NotThere"}}, 

210 } 

211 ) 

212 

213 with self.assertRaises(RuntimeError): 

214 _ = dimension_clustering(config, self.qgraph, "missing-dim-value") 

215 

216 def testClusterEqualDim1(self): 

217 """Test equalDimensions using right half.""" 

218 name = "equal-dim" 

219 config = BpsConfig( 

220 { 

221 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

222 "cluster": { 

223 "cl1": { 

224 "pipetasks": "T1, T2, T3, T4", 

225 "dimensions": "D1, NotThere", 

226 "equalDimensions": "NotThere:D2", 

227 } 

228 }, 

229 } 

230 ) 

231 answer = { 

232 "name": name, 

233 "nodes": { 

234 "cl1_1_2": { 

235 "label": "cl1", 

236 "dims": {"D1": 1, "NotThere": 2}, 

237 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

238 }, 

239 "cl1_3_4": { 

240 "label": "cl1", 

241 "dims": {"D1": 3, "NotThere": 4}, 

242 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

243 }, 

244 }, 

245 "edges": [], 

246 } 

247 

248 cqg = dimension_clustering(config, self.qgraph, name) 

249 check_cqg(cqg, answer) 

250 

251 def testClusterEqualDim2(self): 

252 """Test equalDimensions using left half.""" 

253 name = "equal-dim-2" 

254 config = BpsConfig( 

255 { 

256 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

257 "cluster": { 

258 "cl1": { 

259 "pipetasks": "T1, T2, T3, T4", 

260 "dimensions": "D1, NotThere", 

261 "equalDimensions": "D2:NotThere", 

262 } 

263 }, 

264 } 

265 ) 

266 answer = { 

267 "name": name, 

268 "nodes": { 

269 "cl1_1_2": { 

270 "label": "cl1", 

271 "dims": {"D1": 1, "NotThere": 2}, 

272 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

273 }, 

274 "cl1_3_4": { 

275 "label": "cl1", 

276 "dims": {"D1": 3, "NotThere": 4}, 

277 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

278 }, 

279 }, 

280 "edges": [], 

281 } 

282 

283 cqg = dimension_clustering(config, self.qgraph, name) 

284 check_cqg(cqg, answer) 

285 

286 def testClusterMult(self): 

287 """Test multiple tasks in multiple clusters.""" 

288 name = "cluster-mult" 

289 config = BpsConfig( 

290 { 

291 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

292 "cluster": { 

293 "cl1": {"pipetasks": "T1, T2", "dimensions": "D1, D2"}, 

294 "cl2": {"pipetasks": "T3, T4", "dimensions": "D1, D2"}, 

295 }, 

296 } 

297 ) 

298 answer = { 

299 "name": name, 

300 "nodes": { 

301 "cl1_1_2": {"label": "cl1", "dims": {"D1": 1, "D2": 2}, "counts": {"T1": 1, "T2": 1}}, 

302 "cl1_3_4": {"label": "cl1", "dims": {"D1": 3, "D2": 4}, "counts": {"T1": 1, "T2": 1}}, 

303 "cl2_1_2": {"label": "cl2", "dims": {"D1": 1, "D2": 2}, "counts": {"T3": 1, "T4": 1}}, 

304 "cl2_3_4": {"label": "cl2", "dims": {"D1": 3, "D2": 4}, "counts": {"T3": 1, "T4": 1}}, 

305 }, 

306 "edges": [("cl1_3_4", "cl2_3_4"), ("cl1_1_2", "cl2_1_2")], 

307 } 

308 

309 cqg = dimension_clustering(config, self.qgraph, name) 

310 check_cqg(cqg, answer) 

311 

312 def testClusterPart(self): 

313 """Test will use templateDataId if no clusterTemplate.""" 

314 name = "cluster-part" 

315 config = BpsConfig( 

316 { 

317 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

318 "cluster": { 

319 "cl1": {"pipetasks": "T1, T2", "dimensions": "D1, D2"}, 

320 }, 

321 } 

322 ) 

323 answer = { 

324 "name": name, 

325 "nodes": { 

326 "cl1_1_2": {"label": "cl1", "dims": {"D1": 1, "D2": 2}, "counts": {"T1": 1, "T2": 1}}, 

327 "cl1_3_4": {"label": "cl1", "dims": {"D1": 3, "D2": 4}, "counts": {"T1": 1, "T2": 1}}, 

328 "NODENAME_T3_1_2_": {"label": "T3", "dims": {"D1": 1, "D2": 2}, "counts": {"T3": 1}}, 

329 "NODENAME_T3_3_4_": {"label": "T3", "dims": {"D1": 3, "D2": 4}, "counts": {"T3": 1}}, 

330 "NODENAME_T4_1_2_": {"label": "T4", "dims": {"D1": 1, "D2": 2}, "counts": {"T4": 1}}, 

331 "NODENAME_T4_3_4_": {"label": "T4", "dims": {"D1": 3, "D2": 4}, "counts": {"T4": 1}}, 

332 }, 

333 "edges": [("cl1_1_2", "NODENAME_T3_1_2_"), ("cl1_3_4", "NODENAME_T3_3_4_")], 

334 } 

335 

336 cqg = dimension_clustering(config, self.qgraph, name) 

337 check_cqg(cqg, answer) 

338 

339 def testClusterPartNoTemplate(self): 

340 """No templateDataId nor clusterTemplate (use cluster label).""" 

341 name = "cluster-part-no-template" 

342 config = BpsConfig( 

343 { 

344 "cluster": { 

345 "cl1": {"pipetasks": "T1, T2"}, 

346 } 

347 } 

348 ) 

349 answer = { 

350 "name": name, 

351 "nodes": { 

352 "cl1": {"label": "cl1", "dims": {}, "counts": {"T1": 2, "T2": 2}}, 

353 "NODEONLY_T3_{'D1': 1, 'D2': 2}": { 

354 "label": "T3", 

355 "dims": {"D1": 1, "D2": 2}, 

356 "counts": {"T3": 1}, 

357 }, 

358 "NODEONLY_T3_{'D1': 3, 'D2': 4}": { 

359 "label": "T3", 

360 "dims": {"D1": 3, "D2": 4}, 

361 "counts": {"T3": 1}, 

362 }, 

363 "NODEONLY_T4_{'D1': 1, 'D2': 2}": { 

364 "label": "T4", 

365 "dims": {"D1": 1, "D2": 2}, 

366 "counts": {"T4": 1}, 

367 }, 

368 "NODEONLY_T4_{'D1': 3, 'D2': 4}": { 

369 "label": "T4", 

370 "dims": {"D1": 3, "D2": 4}, 

371 "counts": {"T4": 1}, 

372 }, 

373 }, 

374 "edges": [("cl1", "NODEONLY_T3_{'D1': 1, 'D2': 2}"), ("cl1", "NODEONLY_T3_{'D1': 3, 'D2': 4}")], 

375 } 

376 

377 cqg = dimension_clustering(config, self.qgraph, name) 

378 check_cqg(cqg, answer) 

379 

380 def testClusterExtra(self): 

381 """Clustering includes labels of pipetasks that aren't in QGraph. 

382 They should just be ignored. 

383 """ 

384 name = "extra" 

385 config = BpsConfig( 

386 { 

387 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

388 "cluster": { 

389 "cl1": {"pipetasks": "T1, Extra1, T2, Extra2, T3, T4", "dimensions": "D1, D2"}, 

390 }, 

391 } 

392 ) 

393 answer = { 

394 "name": name, 

395 "nodes": { 

396 "cl1_1_2": { 

397 "label": "cl1", 

398 "dims": {"D1": 1, "D2": 2}, 

399 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

400 }, 

401 "cl1_3_4": { 

402 "label": "cl1", 

403 "dims": {"D1": 3, "D2": 4}, 

404 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1}, 

405 }, 

406 }, 

407 "edges": [], 

408 } 

409 

410 cqg = dimension_clustering(config, self.qgraph, name) 

411 check_cqg(cqg, answer) 

412 

413 def testClusterRepeat(self): 

414 """A PipelineTask appears in more than one cluster definition.""" 

415 config = BpsConfig( 

416 { 

417 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

418 "cluster": { 

419 "cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}, 

420 "cl2": {"pipetasks": "T2", "dimensions": "D1, D2"}, 

421 }, 

422 } 

423 ) 

424 

425 with self.assertRaises(RuntimeError): 

426 _ = dimension_clustering(config, self.qgraph, "repeat-task") 

427 

428 def testClusterDepends(self): 

429 """Part of a chain of PipelineTask appears in different cluster.""" 

430 config = BpsConfig( 

431 { 

432 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

433 "cluster": { 

434 "cl1": {"pipetasks": "T1, T3, T4", "dimensions": "D1, D2"}, 

435 "cl2": {"pipetasks": "T2", "dimensions": "D1, D2"}, 

436 }, 

437 } 

438 ) 

439 

440 with self.assertRaises(RuntimeError): 

441 _ = dimension_clustering(config, self.qgraph, "task-depends") 

442 

443 def testClusterOrder(self): 

444 """Ensure clusters method is in topological order as some 

445 uses require to always have processed parent before 

446 children. 

447 """ 

448 config = BpsConfig( 

449 { 

450 "templateDataId": "{D1}_{D2}_{D3}_{D4}", 

451 "cluster": { 

452 "cl2": {"pipetasks": "T2, T3", "dimensions": "D1, D2"}, 

453 }, 

454 } 

455 ) 

456 

457 cqg = dimension_clustering(config, self.qgraph, "task-cluster-order") 

458 processed = set() 

459 for cluster in cqg.clusters(): 

460 for parent in cqg.predecessors(cluster.name): 

461 self.assertIn( 

462 parent.name, 

463 processed, 

464 f"clusters() returned {cluster.name} before its parent {parent.name}", 

465 ) 

466 processed.add(cluster.name) 

467 

468 

469if __name__ == "__main__": 469 ↛ 470line 469 didn't jump to line 470, because the condition on line 469 was never true

470 unittest.main()