Coverage for tests/test_quantum_clustering_funcs.py: 26%
111 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-17 02:51 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-17 02:51 -0700
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
27"""Unit tests for the clustering methods.
28"""
30# Turn off "doesn't conform to snake_case naming style" because matching
31# the unittest casing.
32# pylint: disable=invalid-name
34import os
35import unittest
37from cqg_test_utils import check_cqg
38from lsst.ctrl.bps import BpsConfig
39from lsst.ctrl.bps.quantum_clustering_funcs import dimension_clustering, single_quantum_clustering
40from qg_test_utils import make_test_quantum_graph
42TESTDIR = os.path.abspath(os.path.dirname(__file__))
45class TestSingleQuantumClustering(unittest.TestCase):
46 """Tests for single_quantum_clustering method."""
48 def setUp(self):
49 self.qgraph = make_test_quantum_graph()
51 def tearDown(self):
52 pass
54 def testClustering(self):
55 """Test valid single quantum clustering."""
56 # Note: the cluster config should be ignored.
57 config = BpsConfig(
58 {
59 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
60 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}},
61 }
62 )
64 cqg = single_quantum_clustering(config, self.qgraph, "single")
65 self.assertIsNotNone(cqg)
66 self.assertIn(cqg.name, "single")
67 self.assertEqual(len(cqg), len(self.qgraph))
69 def testClusteringNoTemplate(self):
70 """Test valid single quantum clustering wihtout a template for the
71 cluster names.
72 """
73 # Note: the cluster config should be ignored.
74 config = BpsConfig({"cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}}})
76 cqg = single_quantum_clustering(config, self.qgraph, "single-no-template")
77 self.assertIsNotNone(cqg)
78 self.assertIn(cqg.name, "single-no-template")
79 self.assertEqual(len(cqg), len(self.qgraph))
82class TestDimensionClustering(unittest.TestCase):
83 """Tests for dimension_clustering method."""
85 def setUp(self):
86 self.qgraph = make_test_quantum_graph()
88 def tearDown(self):
89 pass
91 def testClusterAllInOne(self):
92 """All tasks in one cluster."""
93 name = "all-in-one"
94 config = BpsConfig(
95 {
96 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
97 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"}},
98 }
99 )
100 answer = {
101 "name": name,
102 "nodes": {
103 "cl1_1_2": {
104 "label": "cl1",
105 "dims": {"D1": 1, "D2": 2},
106 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
107 },
108 "cl1_3_4": {
109 "label": "cl1",
110 "dims": {"D1": 3, "D2": 4},
111 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
112 },
113 },
114 "edges": [],
115 }
117 cqg = dimension_clustering(config, self.qgraph, name)
118 check_cqg(cqg, answer)
120 def testClusterTemplate(self):
121 """Test uses clusterTemplate value to name clusters."""
122 name = "cluster-template"
123 config = BpsConfig(
124 {
125 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}",
126 "cluster": {
127 "cl1": {
128 "clusterTemplate": "ct_{D1}_{D2}_{D3}_{D4}",
129 "pipetasks": "T1, T2, T3, T4",
130 "dimensions": "D1, D2",
131 }
132 },
133 }
134 )
135 # Note: clusterTemplate can produce trailing underscore
136 answer = {
137 "name": name,
138 "nodes": {
139 "ct_1_2_": {
140 "label": "cl1",
141 "dims": {"D1": 1, "D2": 2},
142 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
143 },
144 "ct_3_4_": {
145 "label": "cl1",
146 "dims": {"D1": 3, "D2": 4},
147 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
148 },
149 },
150 "edges": [],
151 }
153 cqg = dimension_clustering(config, self.qgraph, name)
154 check_cqg(cqg, answer)
156 def testClusterNoDims(self):
157 """Test if clusters have no dimensions."""
158 name = "cluster-no-dims"
159 config = BpsConfig(
160 {
161 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}",
162 "cluster": {
163 "cl1": {
164 "pipetasks": "T1, T2",
165 },
166 "cl2": {
167 "pipetasks": "T3, T4",
168 },
169 },
170 }
171 )
172 answer = {
173 "name": name,
174 "nodes": {
175 "cl1": {"label": "cl1", "dims": {}, "counts": {"T1": 2, "T2": 2}},
176 "cl2": {"label": "cl2", "dims": {}, "counts": {"T3": 2, "T4": 2}},
177 },
178 "edges": [("cl1", "cl2")],
179 }
181 cqg = dimension_clustering(config, self.qgraph, name)
182 check_cqg(cqg, answer)
184 def testClusterTaskRepeat(self):
185 """Can't have PipelineTask in more than one cluster."""
186 name = "task-repeat"
187 config = BpsConfig(
188 {
189 "templateDataId": "tdid_{D1}_{D2}_{D3}_{D4}",
190 "cluster": {
191 "cl1": {
192 "pipetasks": "T1, T2",
193 },
194 "cl2": {
195 "pipetasks": "T2, T3, T4",
196 },
197 },
198 }
199 )
201 with self.assertRaises(RuntimeError):
202 _ = dimension_clustering(config, self.qgraph, name)
204 def testClusterMissingDimValue(self):
205 """Quantum can't be missing a value for a clustering dimension."""
206 config = BpsConfig(
207 {
208 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
209 "cluster": {"cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, NotThere"}},
210 }
211 )
213 with self.assertRaises(RuntimeError):
214 _ = dimension_clustering(config, self.qgraph, "missing-dim-value")
216 def testClusterEqualDim1(self):
217 """Test equalDimensions using right half."""
218 name = "equal-dim"
219 config = BpsConfig(
220 {
221 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
222 "cluster": {
223 "cl1": {
224 "pipetasks": "T1, T2, T3, T4",
225 "dimensions": "D1, NotThere",
226 "equalDimensions": "NotThere:D2",
227 }
228 },
229 }
230 )
231 answer = {
232 "name": name,
233 "nodes": {
234 "cl1_1_2": {
235 "label": "cl1",
236 "dims": {"D1": 1, "NotThere": 2},
237 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
238 },
239 "cl1_3_4": {
240 "label": "cl1",
241 "dims": {"D1": 3, "NotThere": 4},
242 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
243 },
244 },
245 "edges": [],
246 }
248 cqg = dimension_clustering(config, self.qgraph, name)
249 check_cqg(cqg, answer)
251 def testClusterEqualDim2(self):
252 """Test equalDimensions using left half."""
253 name = "equal-dim-2"
254 config = BpsConfig(
255 {
256 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
257 "cluster": {
258 "cl1": {
259 "pipetasks": "T1, T2, T3, T4",
260 "dimensions": "D1, NotThere",
261 "equalDimensions": "D2:NotThere",
262 }
263 },
264 }
265 )
266 answer = {
267 "name": name,
268 "nodes": {
269 "cl1_1_2": {
270 "label": "cl1",
271 "dims": {"D1": 1, "NotThere": 2},
272 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
273 },
274 "cl1_3_4": {
275 "label": "cl1",
276 "dims": {"D1": 3, "NotThere": 4},
277 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
278 },
279 },
280 "edges": [],
281 }
283 cqg = dimension_clustering(config, self.qgraph, name)
284 check_cqg(cqg, answer)
286 def testClusterMult(self):
287 """Test multiple tasks in multiple clusters."""
288 name = "cluster-mult"
289 config = BpsConfig(
290 {
291 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
292 "cluster": {
293 "cl1": {"pipetasks": "T1, T2", "dimensions": "D1, D2"},
294 "cl2": {"pipetasks": "T3, T4", "dimensions": "D1, D2"},
295 },
296 }
297 )
298 answer = {
299 "name": name,
300 "nodes": {
301 "cl1_1_2": {"label": "cl1", "dims": {"D1": 1, "D2": 2}, "counts": {"T1": 1, "T2": 1}},
302 "cl1_3_4": {"label": "cl1", "dims": {"D1": 3, "D2": 4}, "counts": {"T1": 1, "T2": 1}},
303 "cl2_1_2": {"label": "cl2", "dims": {"D1": 1, "D2": 2}, "counts": {"T3": 1, "T4": 1}},
304 "cl2_3_4": {"label": "cl2", "dims": {"D1": 3, "D2": 4}, "counts": {"T3": 1, "T4": 1}},
305 },
306 "edges": [("cl1_3_4", "cl2_3_4"), ("cl1_1_2", "cl2_1_2")],
307 }
309 cqg = dimension_clustering(config, self.qgraph, name)
310 check_cqg(cqg, answer)
312 def testClusterPart(self):
313 """Test will use templateDataId if no clusterTemplate."""
314 name = "cluster-part"
315 config = BpsConfig(
316 {
317 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
318 "cluster": {
319 "cl1": {"pipetasks": "T1, T2", "dimensions": "D1, D2"},
320 },
321 }
322 )
323 answer = {
324 "name": name,
325 "nodes": {
326 "cl1_1_2": {"label": "cl1", "dims": {"D1": 1, "D2": 2}, "counts": {"T1": 1, "T2": 1}},
327 "cl1_3_4": {"label": "cl1", "dims": {"D1": 3, "D2": 4}, "counts": {"T1": 1, "T2": 1}},
328 "NODENAME_T3_1_2_": {"label": "T3", "dims": {"D1": 1, "D2": 2}, "counts": {"T3": 1}},
329 "NODENAME_T3_3_4_": {"label": "T3", "dims": {"D1": 3, "D2": 4}, "counts": {"T3": 1}},
330 "NODENAME_T4_1_2_": {"label": "T4", "dims": {"D1": 1, "D2": 2}, "counts": {"T4": 1}},
331 "NODENAME_T4_3_4_": {"label": "T4", "dims": {"D1": 3, "D2": 4}, "counts": {"T4": 1}},
332 },
333 "edges": [("cl1_1_2", "NODENAME_T3_1_2_"), ("cl1_3_4", "NODENAME_T3_3_4_")],
334 }
336 cqg = dimension_clustering(config, self.qgraph, name)
337 check_cqg(cqg, answer)
339 def testClusterPartNoTemplate(self):
340 """No templateDataId nor clusterTemplate (use cluster label)."""
341 name = "cluster-part-no-template"
342 config = BpsConfig(
343 {
344 "cluster": {
345 "cl1": {"pipetasks": "T1, T2"},
346 }
347 }
348 )
349 answer = {
350 "name": name,
351 "nodes": {
352 "cl1": {"label": "cl1", "dims": {}, "counts": {"T1": 2, "T2": 2}},
353 "NODEONLY_T3_{'D1': 1, 'D2': 2}": {
354 "label": "T3",
355 "dims": {"D1": 1, "D2": 2},
356 "counts": {"T3": 1},
357 },
358 "NODEONLY_T3_{'D1': 3, 'D2': 4}": {
359 "label": "T3",
360 "dims": {"D1": 3, "D2": 4},
361 "counts": {"T3": 1},
362 },
363 "NODEONLY_T4_{'D1': 1, 'D2': 2}": {
364 "label": "T4",
365 "dims": {"D1": 1, "D2": 2},
366 "counts": {"T4": 1},
367 },
368 "NODEONLY_T4_{'D1': 3, 'D2': 4}": {
369 "label": "T4",
370 "dims": {"D1": 3, "D2": 4},
371 "counts": {"T4": 1},
372 },
373 },
374 "edges": [("cl1", "NODEONLY_T3_{'D1': 1, 'D2': 2}"), ("cl1", "NODEONLY_T3_{'D1': 3, 'D2': 4}")],
375 }
377 cqg = dimension_clustering(config, self.qgraph, name)
378 check_cqg(cqg, answer)
380 def testClusterExtra(self):
381 """Clustering includes labels of pipetasks that aren't in QGraph.
382 They should just be ignored.
383 """
384 name = "extra"
385 config = BpsConfig(
386 {
387 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
388 "cluster": {
389 "cl1": {"pipetasks": "T1, Extra1, T2, Extra2, T3, T4", "dimensions": "D1, D2"},
390 },
391 }
392 )
393 answer = {
394 "name": name,
395 "nodes": {
396 "cl1_1_2": {
397 "label": "cl1",
398 "dims": {"D1": 1, "D2": 2},
399 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
400 },
401 "cl1_3_4": {
402 "label": "cl1",
403 "dims": {"D1": 3, "D2": 4},
404 "counts": {"T1": 1, "T2": 1, "T3": 1, "T4": 1},
405 },
406 },
407 "edges": [],
408 }
410 cqg = dimension_clustering(config, self.qgraph, name)
411 check_cqg(cqg, answer)
413 def testClusterRepeat(self):
414 """A PipelineTask appears in more than one cluster definition."""
415 config = BpsConfig(
416 {
417 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
418 "cluster": {
419 "cl1": {"pipetasks": "T1, T2, T3, T4", "dimensions": "D1, D2"},
420 "cl2": {"pipetasks": "T2", "dimensions": "D1, D2"},
421 },
422 }
423 )
425 with self.assertRaises(RuntimeError):
426 _ = dimension_clustering(config, self.qgraph, "repeat-task")
428 def testClusterDepends(self):
429 """Part of a chain of PipelineTask appears in different cluster."""
430 config = BpsConfig(
431 {
432 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
433 "cluster": {
434 "cl1": {"pipetasks": "T1, T3, T4", "dimensions": "D1, D2"},
435 "cl2": {"pipetasks": "T2", "dimensions": "D1, D2"},
436 },
437 }
438 )
440 with self.assertRaises(RuntimeError):
441 _ = dimension_clustering(config, self.qgraph, "task-depends")
443 def testClusterOrder(self):
444 """Ensure clusters method is in topological order as some
445 uses require to always have processed parent before
446 children.
447 """
448 config = BpsConfig(
449 {
450 "templateDataId": "{D1}_{D2}_{D3}_{D4}",
451 "cluster": {
452 "cl2": {"pipetasks": "T2, T3", "dimensions": "D1, D2"},
453 },
454 }
455 )
457 cqg = dimension_clustering(config, self.qgraph, "task-cluster-order")
458 processed = set()
459 for cluster in cqg.clusters():
460 for parent in cqg.predecessors(cluster.name):
461 self.assertIn(
462 parent.name,
463 processed,
464 f"clusters() returned {cluster.name} before its parent {parent.name}",
465 )
466 processed.add(cluster.name)
469if __name__ == "__main__": 469 ↛ 470line 469 didn't jump to line 470, because the condition on line 469 was never true
470 unittest.main()