From bcb093e69baf7e75742140eb2aaa097dc2824458 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:12:44 +0200 Subject: [PATCH] test: relaunch an mpiexec that never started, for every test and for the MPI_INIT flake too Open MPI 5 on the macOS Intel runner fails to start a launch now and then: either PMIx cannot find a network interface ("ptl_tool: problems getting address", exit 213) or a rank's MPI_Init cannot reach the others ("It looks like MPI_INIT failed ... PML add procs failed", exit 14). #88's two Intel runs each lost one test to the second message, once in test_pro_cli (two ranks) and once in test/multilevel (four ranks). The retry that caught the first message lived in test/multilevel/conftest.py, so it covered neither. The retry now lives in test/conftest.py, wraps every subprocess.run from any test, and matches both messages; nothing has run when either appears, so relaunching is safe, and any other failure is returned as it is. The multilevel conftest keeps only its rank-slot gate, layered over the root retry so a relaunch holds the same slots. test_mpi_launch_retry.py drives the retry with a fake launcher that fails a given number of times with a given message (PROTEUS_TEST_MPI_RETRY_DELAY=0 makes it quick). Co-Authored-By: Claude Fable 5.1 --- test/conftest.py | 37 +++++++++++++ test/multilevel/conftest.py | 29 +++------- test/test_mpi_launch_retry.py | 101 ++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 21 deletions(-) create mode 100644 test/test_mpi_launch_retry.py diff --git a/test/conftest.py b/test/conftest.py index 3e26cc3a60..860da6bea7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,8 @@ """Test-suite wide fixtures.""" import os +import subprocess import sys +import time import pytest @@ -37,6 +39,41 @@ def pytest_configure(config): Comm.argv = runtime.petsc_argv() +# mpiexec itself sometimes fails to start on a CI runner (the macOS Intel runner with Open MPI 5, +# mostly). Either PMIx cannot find a network interface for a moment ("ptl_tool: problems getting +# address for index 0 (kernel index -1)", exit status 213), or a rank's MPI_Init cannot reach the +# others ("It looks like MPI_INIT failed for some reason ... PML add procs failed --> Returned +# "Not found" (-13)", exit status 14). Every launch in that second fails, whatever it was testing, +# and no rank has run anything of the job when it happens, so launching it again is safe; any +# other failure is left alone. The messages are Open MPI's, so only its launches are ever retried. +MPI_LAUNCH_FAILURES = ('ptl_tool: problems getting address', 'It looks like MPI_INIT failed') + + +def mpi_launch_never_started(result): + """True for a ``subprocess.CompletedProcess`` whose mpiexec failed before any rank ran.""" + err = getattr(result, 'stderr', None) + if isinstance(err, bytes): + err = err.decode(errors='replace') + return (result.returncode != 0 and isinstance(err, str) + and any(text in err for text in MPI_LAUNCH_FAILURES)) + + +@pytest.fixture(autouse=True) +def _retry_an_mpi_launch_that_never_started(monkeypatch): + """Every ``subprocess.run`` from a test: launch again, up to three times in all, if an mpiexec + never started (MPI_LAUNCH_FAILURES). The pause before a relaunch is + PROTEUS_TEST_MPI_RETRY_DELAY seconds (default 3), doubled before the last attempt.""" + run = subprocess.run + + def run_with_retry(cmd, *args, **kwargs): + for attempt in range(3): + result = run(cmd, *args, **kwargs) + if attempt == 2 or not mpi_launch_never_started(result): + return result + time.sleep(float(os.environ.get('PROTEUS_TEST_MPI_RETRY_DELAY', 3.0)) * (attempt + 1)) + monkeypatch.setattr(subprocess, 'run', run_with_retry) + + @pytest.fixture(autouse=True) def _restore_petsc_options(): """Put PETSc's options database back as it was before each test. diff --git a/test/multilevel/conftest.py b/test/multilevel/conftest.py index fdc8ff587f..085fa46e21 100644 --- a/test/multilevel/conftest.py +++ b/test/multilevel/conftest.py @@ -91,32 +91,19 @@ def _rank_slots(n): f.close() -# mpiexec itself sometimes fails to start on a CI runner: PMIx cannot find a network interface for a -# moment ("ptl_tool: problems getting address for index 0 (kernel index -1)", exit status 213), and -# every launch in that second fails, whatever it was testing. Nothing of the job has run when that -# happens, so launching it again is safe; any other failure is left alone. -_LAUNCH_FAILURE = 'ptl_tool: problems getting address' - - @pytest.fixture(autouse=True) -def _launch_policy(monkeypatch): - """Every ``subprocess.run`` of an mpiexec from these tests: hold its rank slots while it runs - (mode 'slots') and launch again, up to three times, if it never started.""" +def _launch_policy(monkeypatch, _retry_an_mpi_launch_that_never_started): + """Every ``subprocess.run`` of an mpiexec from these tests holds its rank slots while it runs + (mode 'slots'). It wraps the retry test/conftest.py puts on every launch, so a launch that + never started (an Open MPI runner flake) is launched again under the same slots.""" run = subprocess.run def launch(cmd, *args, **kwargs): ranks = _ranks(cmd) if _MODE == 'slots' else None - for attempt in range(3): - if ranks is None: - result = run(cmd, *args, **kwargs) - else: - with _rank_slots(ranks): - result = run(cmd, *args, **kwargs) - err = getattr(result, 'stderr', None) - if (result.returncode == 0 or not isinstance(err, str) or _LAUNCH_FAILURE not in err - or attempt == 2): - return result - time.sleep(3.0 * (attempt + 1)) + if ranks is None: + return run(cmd, *args, **kwargs) + with _rank_slots(ranks): + return run(cmd, *args, **kwargs) monkeypatch.setattr(subprocess, 'run', launch) yield diff --git a/test/test_mpi_launch_retry.py b/test/test_mpi_launch_retry.py new file mode 100644 index 0000000000..5703dcf519 --- /dev/null +++ b/test/test_mpi_launch_retry.py @@ -0,0 +1,101 @@ +"""The relaunch of an mpiexec that never started (test/conftest.py). + +A fake launcher stands in for mpiexec: it counts its calls in a file, fails the first +``fail`` of them with the stderr text given, and succeeds after that. The retry in the +root conftest wraps every ``subprocess.run`` from a test, so calling it here goes +through the retry. +""" +import subprocess +import sys +import textwrap + +import pytest + +OMPI_MPI_INIT_FAILED = textwrap.dedent('''\ + -------------------------------------------------------------------------- + It looks like MPI_INIT failed for some reason; your parallel process is + likely to abort. There are many reasons that a parallel process can + fail during MPI_INIT; some of which are due to configuration or environment + problems. This failure appears to be an internal failure; here's some + additional information (which may only be relevant to an Open MPI + developer): + + PML add procs failed + --> Returned "Not found" (-13) instead of "Success" (0) + -------------------------------------------------------------------------- + [host:00000] *** An error occurred in MPI_Init_thread + [host:00000] *** MPI_ERRORS_ARE_FATAL (processes in this communicator will now abort, + prterun detected that one or more processes exited with non-zero status, + thus causing the job to be terminated. The first process to do so was: + + Process name: [prterun-host-58199@1,1] + Exit code: 14 + -------------------------------------------------------------------------- + ''') +OMPI_PTL_TOOL = ('[host:58199] ptl_tool: problems getting address for index 0 (kernel index -1)\n') +SOMETHING_ELSE = 'Traceback (most recent call last):\n ...\nAssertionError: the job itself failed\n' + +FAKE_LAUNCHER = textwrap.dedent('''\ + import os, sys + counter, fail, message, status = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]) + calls = int(open(counter).read()) + 1 if os.path.exists(counter) else 1 + with open(counter, 'w') as f: + f.write(str(calls)) + if calls <= fail: + sys.stderr.write(open(message).read()) + sys.exit(status) + print('RESULT started on call', calls) + ''') + + +@pytest.fixture +def launch(tmp_path, monkeypatch): + monkeypatch.setenv('PROTEUS_TEST_MPI_RETRY_DELAY', '0') + script = tmp_path / 'fake_mpiexec.py' + script.write_text(FAKE_LAUNCHER) + counter = tmp_path / 'calls' + + def _launch(fail, text, status, **kwargs): + message = tmp_path / 'message' + message.write_text(text) + result = subprocess.run([sys.executable, str(script), str(counter), str(fail), str(message), + str(status)], capture_output=True, text=True, **kwargs) + return result, int(counter.read_text()) + return _launch + + +def test_an_mpi_init_failure_is_launched_again(launch): + result, calls = launch(2, OMPI_MPI_INIT_FAILED, 14) + assert result.returncode == 0 and 'started on call 3' in result.stdout + assert calls == 3 + + +def test_a_pmix_address_failure_is_launched_again(launch): + result, calls = launch(1, OMPI_PTL_TOOL, 213) + assert result.returncode == 0 and 'started on call 2' in result.stdout + assert calls == 2 + + +def test_three_attempts_at_most(launch): + result, calls = launch(10, OMPI_MPI_INIT_FAILED, 14) + assert result.returncode == 14 + assert 'MPI_INIT failed' in result.stderr + assert calls == 3 + + +def test_a_failure_of_the_job_itself_is_not_launched_again(launch): + result, calls = launch(10, SOMETHING_ELSE, 1) + assert result.returncode == 1 + assert calls == 1 + + +def test_a_launch_whose_stderr_is_not_captured_is_not_launched_again(tmp_path, monkeypatch): + monkeypatch.setenv('PROTEUS_TEST_MPI_RETRY_DELAY', '0') + script = tmp_path / 'fake_mpiexec.py' + script.write_text(FAKE_LAUNCHER) + counter, message = tmp_path / 'calls', tmp_path / 'message' + message.write_text(OMPI_MPI_INIT_FAILED) + result = subprocess.run([sys.executable, str(script), str(counter), '10', str(message), '14'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + assert result.returncode == 14 + assert int(counter.read_text()) == 1