Coverage for tests/test_executors.py: 16%
418 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-12 12:21 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-12 12:21 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
28"""Simple unit test for cmdLineFwk module.
29"""
31import faulthandler
32import logging
33import multiprocessing
34import os
35import signal
36import sys
37import time
38import unittest
39import warnings
40from multiprocessing import Manager
42import networkx as nx
43import psutil
44from lsst.ctrl.mpexec import (
45 ExecutionStatus,
46 MPGraphExecutor,
47 MPGraphExecutorError,
48 MPTimeoutError,
49 QuantumExecutor,
50 QuantumReport,
51 SingleQuantumExecutor,
52)
53from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
54from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
55from lsst.pipe.base import NodeId
56from lsst.pipe.base.tests.simpleQGraph import AddTaskFactoryMock, makeSimpleQGraph
58logging.basicConfig(level=logging.DEBUG)
60_LOG = logging.getLogger(__name__)
62TESTDIR = os.path.abspath(os.path.dirname(__file__))
65class QuantumExecutorMock(QuantumExecutor):
66 """Mock class for QuantumExecutor"""
68 def __init__(self, mp=False):
69 self.quanta = []
70 if mp:
71 # in multiprocess mode use shared list
72 manager = Manager()
73 self.quanta = manager.list()
74 self.report = None
75 self._execute_called = False
77 def execute(self, taskDef, quantum):
78 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
79 self._execute_called = True
80 if taskDef.taskClass:
81 try:
82 # only works for one of the TaskMock classes below
83 taskDef.taskClass().runQuantum()
84 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label)
85 except Exception as exc:
86 self.report = QuantumReport.from_exception(
87 exception=exc,
88 dataId=quantum.dataId,
89 taskLabel=taskDef.label,
90 )
91 raise
92 self.quanta.append(quantum)
93 return quantum
95 def getReport(self):
96 if not self._execute_called:
97 raise RuntimeError("getReport called before execute")
98 return self.report
100 def getDataIds(self, field):
101 """Return values for dataId field for each visited quanta."""
102 return [quantum.dataId[field] for quantum in self.quanta]
105class QuantumMock:
106 """Mock equivalent of a `~lsst.daf.butler.Quantum`."""
108 def __init__(self, dataId):
109 self.dataId = dataId
111 def __eq__(self, other):
112 return self.dataId == other.dataId
114 def __hash__(self):
115 # dict.__eq__ is order-insensitive
116 return hash(tuple(sorted(kv for kv in self.dataId.items())))
119class QuantumIterDataMock:
120 """Simple class to mock QuantumIterData."""
122 def __init__(self, index, taskDef, **dataId):
123 self.index = index
124 self.taskDef = taskDef
125 self.quantum = QuantumMock(dataId)
126 self.dependencies = set()
127 self.nodeId = NodeId(index, "DummyBuildString")
130class QuantumGraphMock:
131 """Mock for quantum graph."""
133 def __init__(self, qdata):
134 self._graph = nx.DiGraph()
135 previous = qdata[0]
136 for node in qdata[1:]:
137 self._graph.add_edge(previous, node)
138 previous = node
140 def __iter__(self):
141 yield from nx.topological_sort(self._graph)
143 def __len__(self):
144 return len(self._graph)
146 def findTaskDefByLabel(self, label):
147 for q in self:
148 if q.taskDef.label == label:
149 return q.taskDef
151 def getQuantaForTask(self, taskDef):
152 nodes = self.getNodesForTask(taskDef)
153 return {q.quantum for q in nodes}
155 def getNodesForTask(self, taskDef):
156 quanta = set()
157 for q in self:
158 if q.taskDef == taskDef:
159 quanta.add(q)
160 return quanta
162 @property
163 def graph(self):
164 return self._graph
166 def findCycle(self):
167 return []
169 def determineInputsToQuantumNode(self, node):
170 result = set()
171 for n in node.dependencies:
172 for otherNode in self:
173 if otherNode.index == n:
174 result.add(otherNode)
175 return result
178class TaskMockMP:
179 """Simple mock class for task supporting multiprocessing."""
181 canMultiprocess = True
183 def runQuantum(self):
184 _LOG.debug("TaskMockMP.runQuantum")
185 pass
188class TaskMockFail:
189 """Simple mock class for task which fails."""
191 canMultiprocess = True
193 def runQuantum(self):
194 _LOG.debug("TaskMockFail.runQuantum")
195 raise ValueError("expected failure")
198class TaskMockCrash:
199 """Simple mock class for task which fails."""
201 canMultiprocess = True
203 def runQuantum(self):
204 _LOG.debug("TaskMockCrash.runQuantum")
205 # Disable fault handler to suppress long scary traceback.
206 faulthandler.disable()
207 signal.raise_signal(signal.SIGILL)
210class TaskMockLongSleep:
211 """Simple mock class for task which "runs" for very long time."""
213 canMultiprocess = True
215 def runQuantum(self):
216 _LOG.debug("TaskMockLongSleep.runQuantum")
217 time.sleep(100.0)
220class TaskMockNoMP:
221 """Simple mock class for task not supporting multiprocessing."""
223 canMultiprocess = False
226class TaskDefMock:
227 """Simple mock class for task definition in a pipeline."""
229 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
230 self.taskName = taskName
231 self.config = config
232 self.taskClass = taskClass
233 self.label = label
235 def __str__(self):
236 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
239def _count_status(report, status):
240 """Count number of quanta witha a given status."""
241 return len([qrep for qrep in report.quantaReports if qrep.status is status])
244class MPGraphExecutorTestCase(unittest.TestCase):
245 """A test case for MPGraphExecutor class"""
247 def test_mpexec_nomp(self):
248 """Make simple graph and execute"""
249 taskDef = TaskDefMock()
250 qgraph = QuantumGraphMock(
251 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
252 )
254 # run in single-process mode
255 qexec = QuantumExecutorMock()
256 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
257 mpexec.execute(qgraph)
258 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
259 report = mpexec.getReport()
260 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
261 self.assertIsNone(report.exitCode)
262 self.assertIsNone(report.exceptionInfo)
263 self.assertEqual(len(report.quantaReports), 3)
264 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
265 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
266 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
267 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
269 def test_mpexec_mp(self):
270 """Make simple graph and execute"""
271 taskDef = TaskDefMock()
272 qgraph = QuantumGraphMock(
273 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
274 )
276 methods = ["spawn"]
277 if sys.platform == "linux":
278 methods.append("forkserver")
280 for method in methods:
281 with self.subTest(startMethod=method):
282 # Run in multi-process mode, the order of results is not
283 # defined.
284 qexec = QuantumExecutorMock(mp=True)
285 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
286 mpexec.execute(qgraph)
287 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
288 report = mpexec.getReport()
289 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
290 self.assertIsNone(report.exitCode)
291 self.assertIsNone(report.exceptionInfo)
292 self.assertEqual(len(report.quantaReports), 3)
293 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
294 self.assertTrue(all(qrep.exitCode == 0 for qrep in report.quantaReports))
295 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
296 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
298 def test_mpexec_nompsupport(self):
299 """Try to run MP for task that has no MP support which should fail"""
300 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
301 qgraph = QuantumGraphMock(
302 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
303 )
305 # run in multi-process mode
306 qexec = QuantumExecutorMock()
307 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
308 with self.assertRaisesRegex(MPGraphExecutorError, "Task Task does not support multiprocessing"):
309 mpexec.execute(qgraph)
311 def test_mpexec_fixup(self):
312 """Make simple graph and execute, add dependencies by executing fixup
313 code.
314 """
315 taskDef = TaskDefMock()
317 for reverse in (False, True):
318 qgraph = QuantumGraphMock(
319 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
320 )
322 qexec = QuantumExecutorMock()
323 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
324 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, executionGraphFixup=fixup)
325 mpexec.execute(qgraph)
327 expected = [0, 1, 2]
328 if reverse:
329 expected = list(reversed(expected))
330 self.assertEqual(qexec.getDataIds("detector"), expected)
332 def test_mpexec_timeout(self):
333 """Fail due to timeout"""
334 taskDef = TaskDefMock()
335 taskDefSleep = TaskDefMock(taskClass=TaskMockLongSleep)
336 qgraph = QuantumGraphMock(
337 [
338 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
339 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
340 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
341 ]
342 )
344 # with failFast we'll get immediate MPTimeoutError
345 qexec = QuantumExecutorMock(mp=True)
346 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
347 with self.assertRaises(MPTimeoutError):
348 mpexec.execute(qgraph)
349 report = mpexec.getReport()
350 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
351 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
352 self.assertGreater(len(report.quantaReports), 0)
353 self.assertEqual(_count_status(report, ExecutionStatus.TIMEOUT), 1)
354 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
355 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
357 # with failFast=False exception happens after last task finishes
358 qexec = QuantumExecutorMock(mp=True)
359 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
360 with self.assertRaises(MPTimeoutError):
361 mpexec.execute(qgraph)
362 # We expect two tasks (0 and 2) to finish successfully and one task to
363 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
364 # finish on time, so expect more timeouts and issue a warning.
365 detectorIds = set(qexec.getDataIds("detector"))
366 self.assertLess(len(detectorIds), 3)
367 if detectorIds != {0, 2}:
368 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
369 report = mpexec.getReport()
370 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
371 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
372 self.assertGreater(len(report.quantaReports), 0)
373 self.assertGreater(_count_status(report, ExecutionStatus.TIMEOUT), 0)
374 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
375 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
377 def test_mpexec_failure(self):
378 """Failure in one task should not stop other tasks"""
379 taskDef = TaskDefMock()
380 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
381 qgraph = QuantumGraphMock(
382 [
383 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
384 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
385 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
386 ]
387 )
389 qexec = QuantumExecutorMock(mp=True)
390 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
391 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
392 mpexec.execute(qgraph)
393 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
394 report = mpexec.getReport()
395 self.assertEqual(report.status, ExecutionStatus.FAILURE)
396 self.assertEqual(
397 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
398 )
399 self.assertGreater(len(report.quantaReports), 0)
400 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
401 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
402 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
403 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
405 def test_mpexec_failure_dep(self):
406 """Failure in one task should skip dependents"""
407 taskDef = TaskDefMock()
408 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
409 qdata = [
410 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
411 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
412 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
413 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
414 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
415 ]
416 qdata[2].dependencies.add(1)
417 qdata[4].dependencies.add(3)
418 qdata[4].dependencies.add(2)
420 qgraph = QuantumGraphMock(qdata)
422 qexec = QuantumExecutorMock(mp=True)
423 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
424 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
425 mpexec.execute(qgraph)
426 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
427 report = mpexec.getReport()
428 self.assertEqual(report.status, ExecutionStatus.FAILURE)
429 self.assertEqual(
430 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
431 )
432 # Dependencies of failed tasks do not appear in quantaReports
433 self.assertGreater(len(report.quantaReports), 0)
434 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
435 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
436 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
437 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
438 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
440 def test_mpexec_failure_dep_nomp(self):
441 """Failure in one task should skip dependents, in-process version"""
442 taskDef = TaskDefMock()
443 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
444 qdata = [
445 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
446 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
447 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
448 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
449 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
450 ]
451 qdata[2].dependencies.add(1)
452 qdata[4].dependencies.add(3)
453 qdata[4].dependencies.add(2)
455 qgraph = QuantumGraphMock(qdata)
457 qexec = QuantumExecutorMock()
458 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
459 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
460 mpexec.execute(qgraph)
461 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
462 report = mpexec.getReport()
463 self.assertEqual(report.status, ExecutionStatus.FAILURE)
464 self.assertEqual(
465 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
466 )
467 # Dependencies of failed tasks do not appear in quantaReports
468 self.assertGreater(len(report.quantaReports), 0)
469 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
470 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
471 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
472 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
473 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
475 def test_mpexec_failure_failfast(self):
476 """Fast fail stops quickly.
478 Timing delay of task #3 should be sufficient to process
479 failure and raise exception.
480 """
481 taskDef = TaskDefMock()
482 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
483 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
484 qdata = [
485 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
486 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
487 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
488 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
489 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
490 ]
491 qdata[1].dependencies.add(0)
492 qdata[2].dependencies.add(1)
493 qdata[4].dependencies.add(3)
494 qdata[4].dependencies.add(2)
496 qgraph = QuantumGraphMock(qdata)
498 qexec = QuantumExecutorMock(mp=True)
499 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
500 with self.assertRaisesRegex(MPGraphExecutorError, "failed, exit code=1"):
501 mpexec.execute(qgraph)
502 self.assertCountEqual(qexec.getDataIds("detector"), [0])
503 report = mpexec.getReport()
504 self.assertEqual(report.status, ExecutionStatus.FAILURE)
505 self.assertEqual(
506 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
507 )
508 # Dependencies of failed tasks do not appear in quantaReports
509 self.assertGreater(len(report.quantaReports), 0)
510 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
511 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
512 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
514 def test_mpexec_crash(self):
515 """Check task crash due to signal"""
516 taskDef = TaskDefMock()
517 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
518 qgraph = QuantumGraphMock(
519 [
520 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
521 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
522 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
523 ]
524 )
526 qexec = QuantumExecutorMock(mp=True)
527 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
528 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
529 mpexec.execute(qgraph)
530 report = mpexec.getReport()
531 self.assertEqual(report.status, ExecutionStatus.FAILURE)
532 self.assertEqual(
533 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
534 )
535 # Dependencies of failed tasks do not appear in quantaReports
536 self.assertGreater(len(report.quantaReports), 0)
537 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
538 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
539 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
540 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
542 def test_mpexec_crash_failfast(self):
543 """Check task crash due to signal with --fail-fast"""
544 taskDef = TaskDefMock()
545 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
546 qgraph = QuantumGraphMock(
547 [
548 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
549 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
550 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
551 ]
552 )
554 qexec = QuantumExecutorMock(mp=True)
555 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
556 with self.assertRaisesRegex(MPGraphExecutorError, "failed, killed by signal 4 .Illegal instruction"):
557 mpexec.execute(qgraph)
558 report = mpexec.getReport()
559 self.assertEqual(report.status, ExecutionStatus.FAILURE)
560 self.assertEqual(
561 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
562 )
563 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
564 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
565 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
567 def test_mpexec_num_fd(self):
568 """Check that number of open files stays reasonable"""
569 taskDef = TaskDefMock()
570 qgraph = QuantumGraphMock(
571 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)]
572 )
574 this_proc = psutil.Process()
575 num_fds_0 = this_proc.num_fds()
577 # run in multi-process mode, the order of results is not defined
578 qexec = QuantumExecutorMock(mp=True)
579 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
580 mpexec.execute(qgraph)
582 num_fds_1 = this_proc.num_fds()
583 # They should be the same but allow small growth just in case.
584 # Without DM-26728 fix the difference would be equal to number of
585 # quanta (20).
586 self.assertLess(num_fds_1 - num_fds_0, 5)
589class SingleQuantumExecutorTestCase(unittest.TestCase):
590 """Tests for SingleQuantumExecutor implementation."""
592 instrument = "lsst.pipe.base.tests.simpleQGraph.SimpleInstrument"
594 def setUp(self):
595 self.root = makeTestTempDir(TESTDIR)
597 def tearDown(self):
598 removeTestTempDir(self.root)
600 def test_simple_execute(self) -> None:
601 """Run execute() method in simplest setup."""
602 nQuanta = 1
603 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, instrument=self.instrument)
605 nodes = list(qgraph)
606 self.assertEqual(len(nodes), nQuanta)
607 node = nodes[0]
609 taskFactory = AddTaskFactoryMock()
610 executor = SingleQuantumExecutor(butler, taskFactory)
611 executor.execute(node.taskDef, node.quantum)
612 self.assertEqual(taskFactory.countExec, 1)
614 # There must be one dataset of task's output connection
615 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
616 self.assertEqual(len(refs), 1)
618 def test_skip_existing_execute(self) -> None:
619 """Run execute() method twice, with skip_existing_in."""
620 nQuanta = 1
621 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, instrument=self.instrument)
623 nodes = list(qgraph)
624 self.assertEqual(len(nodes), nQuanta)
625 node = nodes[0]
627 taskFactory = AddTaskFactoryMock()
628 executor = SingleQuantumExecutor(butler, taskFactory)
629 executor.execute(node.taskDef, node.quantum)
630 self.assertEqual(taskFactory.countExec, 1)
632 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
633 self.assertEqual(len(refs), 1)
634 dataset_id_1 = refs[0].id
636 # Re-run it with skipExistingIn, it should not run.
637 assert butler.run is not None
638 executor = SingleQuantumExecutor(butler, taskFactory, skipExistingIn=[butler.run])
639 executor.execute(node.taskDef, node.quantum)
640 self.assertEqual(taskFactory.countExec, 1)
642 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
643 self.assertEqual(len(refs), 1)
644 dataset_id_2 = refs[0].id
645 self.assertEqual(dataset_id_1, dataset_id_2)
647 def test_clobber_outputs_execute(self) -> None:
648 """Run execute() method twice, with clobber_outputs."""
649 nQuanta = 1
650 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, instrument=self.instrument)
652 nodes = list(qgraph)
653 self.assertEqual(len(nodes), nQuanta)
654 node = nodes[0]
656 taskFactory = AddTaskFactoryMock()
657 executor = SingleQuantumExecutor(butler, taskFactory)
658 executor.execute(node.taskDef, node.quantum)
659 self.assertEqual(taskFactory.countExec, 1)
661 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
662 self.assertEqual(len(refs), 1)
663 dataset_id_1 = refs[0].id
665 original_dataset = butler.get(refs[0])
667 # Remove the dataset ourself, and replace it with something
668 # different so we can check later whether it got replaced.
669 butler.pruneDatasets([refs[0]], disassociate=False, unstore=True, purge=False)
670 replacement = original_dataset + 10
671 butler.put(replacement, refs[0])
673 # Re-run it with clobberOutputs and skipExistingIn, it should not
674 # clobber but should skip instead.
675 assert butler.run is not None
676 executor = SingleQuantumExecutor(
677 butler, taskFactory, skipExistingIn=[butler.run], clobberOutputs=True
678 )
679 executor.execute(node.taskDef, node.quantum)
680 self.assertEqual(taskFactory.countExec, 1)
682 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
683 self.assertEqual(len(refs), 1)
684 dataset_id_2 = refs[0].id
685 self.assertEqual(dataset_id_1, dataset_id_2)
687 second_dataset = butler.get(refs[0])
688 self.assertEqual(list(second_dataset), list(replacement))
690 # Re-run it with clobberOutputs but without skipExistingIn, it should
691 # clobber.
692 assert butler.run is not None
693 executor = SingleQuantumExecutor(butler, taskFactory, clobberOutputs=True)
694 executor.execute(node.taskDef, node.quantum)
695 self.assertEqual(taskFactory.countExec, 2)
697 refs = list(butler.registry.queryDatasets("add_dataset1", collections=butler.run))
698 self.assertEqual(len(refs), 1)
699 dataset_id_3 = refs[0].id
701 third_dataset = butler.get(refs[0])
702 self.assertEqual(list(third_dataset), list(original_dataset))
704 # No change in UUID even after replacement
705 self.assertEqual(dataset_id_1, dataset_id_3)
708def setup_module(module):
709 """Force spawn to be used if no method given explicitly.
711 This can be removed when Python 3.14 changes the default.
712 """
713 multiprocessing.set_start_method("spawn", force=True)
716if __name__ == "__main__":
717 # Do not need to force start mode when running standalone.
718 multiprocessing.set_start_method("spawn")
719 unittest.main()