ci: optimize self-hosted KubeVirt runners and CI pipeline

.github/workflows/e2e.yaml

@@ -48,7 +48,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-24.04
+          - os: arc-runner-kubevirt-small
             arch: amd64
           - os: ubuntu-24.04-arm
             arch: arm64
@@ -91,7 +91,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-24.04
+          - os: arc-runner-kubevirt-small
             arch: amd64
           - os: ubuntu-24.04-arm
             arch: arm64
@@ -136,7 +136,7 @@ jobs:
   build-python-wheels:
     needs: changes
     if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
-    runs-on: ubuntu-24.04
+    runs-on: arc-runner-kubevirt-small
     timeout-minutes: 15
     steps:
       - name: Checkout repository
@@ -180,7 +180,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-24.04
+          - os: arc-runner-kubevirt-large
             arch: amd64
           - os: ubuntu-24.04-arm
             arch: arm64
@@ -245,7 +245,7 @@ jobs:
   e2e-compat-old-controller:
     needs: changes
     if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch'
-    runs-on: ubuntu-24.04
+    runs-on: arc-runner-kubevirt-large
     timeout-minutes: 60
     steps:
       - name: Checkout repository
@@ -284,7 +284,7 @@ jobs:
   e2e-compat-old-client:
     needs: [changes, build-controller-image, build-operator-image, build-python-wheels]
     if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch'
-    runs-on: ubuntu-24.04
+    runs-on: arc-runner-kubevirt-large
     timeout-minutes: 60
     steps:
       - name: Checkout repository

.github/workflows/python-tests.yaml

@@ -42,8 +42,9 @@ jobs:
     if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch'
     runs-on: ${{ matrix.runs-on }}
     strategy:
+      fail-fast: false
       matrix:
-        runs-on: [ubuntu-24.04, macos-15]
+        runs-on: [arc-runner-kubevirt-small, macos-15]
         # Floor: oldest Python in supported platforms (RHEL 9 appstream)
         # Ceiling: newest Python in latest Fedora
         # Review on each RHEL/Fedora release
@@ -61,48 +62,44 @@ jobs:
           version: ${{ steps.uv.outputs.version }}
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Qemu (Linux)
+      - name: Setup Linux dependencies
         if: runner.os == 'Linux'
         run: |
+          # udev rules and kernel modules (idempotent)
           echo <<EOF | sudo tee /etc/udev/rules.d/99-kvm.rules
           KERNEL=="kvm", GROUP="kvm", MODE="0666"
           KERNEL=="vhost-vsock", GROUP="kvm", MODE="0666"
           KERNEL=="vhost-net", GROUP="kvm", MODE="0666"
           EOF
-
           sudo udevadm control --reload-rules
-
-          sudo modprobe vhost_vsock
-          sudo modprobe vhost_net
-          sudo chmod 0666 /dev/kvm /dev/vhost-vsock /dev/vhost-net
-
-          sudo apt-get update
-          sudo apt-get install -y qemu-system-arm qemu-system-x86
-
-      - name: Install libgpiod-dev (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libgpiod-dev liblgpio-dev
-
-      - name: Install nftables and dnsmasq (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y nftables dnsmasq-base isc-dhcp-client dnsutils
-
-      - name: Load kernel modules for DUT network tests (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          for mod in veth bridge nf_nat nf_conntrack nft_masq nft_nat nft_chain_nat; do
+          for mod in vhost_vsock vhost_net veth bridge nf_nat nf_conntrack nft_masq nft_nat nft_chain_nat; do
             sudo modprobe "$mod" 2>/dev/null || true
           done
-
-      - name: Install Renode (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          wget https://github.com/renode/renode/releases/download/v1.16.1/renode_1.16.1_amd64.deb -O /tmp/renode.deb
-          sudo apt-get install -y /tmp/renode.deb
+          sudo chmod 0666 /dev/kvm /dev/vhost-vsock /dev/vhost-net 2>/dev/null || true
+
+          # collect missing packages
+          pkgs=()
+          command -v qemu-system-arm &>/dev/null  || pkgs+=(qemu-system-arm)
+          command -v qemu-system-x86_64 &>/dev/null || pkgs+=(qemu-system-x86)
+          dpkg -s libgpiod-dev &>/dev/null         || pkgs+=(libgpiod-dev)
+          dpkg -s liblgpio-dev &>/dev/null          || pkgs+=(liblgpio-dev)
+          command -v nft &>/dev/null                || pkgs+=(nftables)
+          command -v dnsmasq &>/dev/null            || pkgs+=(dnsmasq-base)
+          command -v dhclient &>/dev/null           || pkgs+=(isc-dhcp-client)
+          command -v dig &>/dev/null                || pkgs+=(dnsutils)
+          command -v renode &>/dev/null              || pkgs+=(renode)
+          command -v rpm2cpio &>/dev/null            || pkgs+=(rpm2cpio)
+          command -v cpio &>/dev/null                || pkgs+=(cpio)
+
+          if [ ${#pkgs[@]} -gt 0 ]; then
+            # add renode repo if needed
+            if [[ " ${pkgs[*]} " == *" renode "* ]]; then
+              wget -qO /tmp/renode.deb https://github.com/renode/renode/releases/download/v1.16.1/renode_1.16.1_amd64.deb
+              pkgs=("${pkgs[@]/renode//tmp/renode.deb}")
+            fi
+            sudo apt-get update
+            sudo apt-get install -y "${pkgs[@]}"
+          fi
 
       - name: Install Qemu (macOS)
         if: runner.os == 'macOS'
@@ -138,9 +135,18 @@ jobs:
       - name: Run pytest
         working-directory: python
         env:
-          PYTEST_ADDOPTS: "--cov-report=xml"
+          PYTEST_ADDOPTS: "--cov-report=xml --log-level=CRITICAL --log-cli-level=CRITICAL"
         run: |
-          make test
+          make test -j4 LOGS_DIR=${{ runner.temp }}/test-logs
+
+      - name: Upload test logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: test-logs-${{ matrix.runs-on }}-py${{ matrix.python-version }}
+          path: ${{ runner.temp }}/test-logs/
+          if-no-files-found: ignore
+          retention-days: 7
 
       # Diff-coverage is only checked on Linux. Several packages (e.g.
       # jumpstarter-driver-dut-network) are Linux-only, so their tests are
@@ -156,7 +162,7 @@ jobs:
             echo "::error::No coverage.xml files found"
             exit 1
           fi
-          uv run diff-cover $coverage_files --compare-branch=origin/${{ github.base_ref }} --fail-under=80 --exclude '*_pb2.py' '*_pb2_grpc.py' '**/conftest.py'
+          uv run diff-cover $coverage_files --compare-branch=origin/${{ github.base_ref }} --fail-under=80 --exclude '*_pb2.py' '*_pb2_grpc.py' '**/conftest.py' '**/test_*.py' '**/*_test.py'
 
   # https://github.com/orgs/community/discussions/26822
   pytest:

e2e/test/dut_network_test.go

@@ -121,11 +121,12 @@ var _ = Describe("DUT Network E2E Tests", Label("dut-network"), Ordered, func()
 		setupNetworkNamespaces()
 
 		configPath := filepath.Join(exporterDir, "exporter-dut-network.yaml")
-		tracker.StartDirectExporter(configPath, listenerPort, "", false)
+		tracker.StartDirectExporter(configPath, listenerPort, "", true)
 		WaitForDirectExporterReady(listenerPort, "")
 	})
 
 	AfterAll(func() {
+		tracker.DumpLogs(50)
 		tracker.StopAll()
 		teardownNetworkNamespaces()
 

e2e/test/utils.go

@@ -439,7 +439,8 @@ func (pt *ProcessTracker) StartDirectExporter(configFile string, port int, passp
 
 	var stderrBuf *logBuffer
 	if captureStderr {
-		stderrBuf = pt.getOrCreateLog("direct-exporter-stderr")
+		logName := "direct-exporter-stderr-" + strconv.Itoa(port)
+		stderrBuf = pt.getOrCreateLog(logName)
 		cmd.Stderr = stderrBuf
 	}
 

python/Makefile

@@ -80,14 +80,41 @@ docs-test: docs-generate-crds docs-generate-grpc
 docs-linkcheck: docs-generate-crds docs-generate-grpc
 	uv run --isolated --all-packages --group docs $(MAKE) -C docs linkcheck
 
+ifdef LOGS_DIR
+pkg-test-%: packages/%
+	@mkdir -p $(LOGS_DIR)
+	@rm -f $(LOGS_DIR)/$*.failed
+	@bash -c 'set -o pipefail; \
+	PYTHONUNBUFFERED=1 uv run --isolated --directory $< pytest 2>&1 | tee $(LOGS_DIR)/$*.log; \
+	rc=$$?; \
+	if [ $$rc -ne 0 ] && [ $$rc -ne 5 ]; then \
+		touch $(LOGS_DIR)/$*.failed; \
+	fi; \
+	true'
+else
 pkg-test-%: packages/%
 	uv run --isolated --directory $< pytest || [ $$? -eq 5 ]
+endif
 
 pkg-ty-%: packages/%
 	uv run --isolated --directory $< ty check .
 
 pkg-test-all: $(addprefix pkg-test-,$(PKG_TARGETS))
 
+ifdef LOGS_DIR
+test-report:
+	@failed=0; \
+	for f in $(LOGS_DIR)/*.failed; do \
+		[ -f "$$f" ] || continue; \
+		pkg=$$(basename "$$f" .failed); \
+		echo ""; \
+		echo "========== FAILED: $$pkg =========="; \
+		cat "$(LOGS_DIR)/$$pkg.log"; \
+		failed=1; \
+	done; \
+	[ $$failed -eq 0 ] && echo "All package tests passed." || exit 1
+endif
+
 pkg-ty-all: $(addprefix pkg-ty-,$(PKG_TARGETS))
 
 build:
@@ -126,7 +153,11 @@ clean-docs:
 
 clean: clean-docs clean-venv clean-build clean-test
 
+ifdef LOGS_DIR
+test: pkg-test-all docs-test docs-test-grpc test-report
+else
 test: pkg-test-all docs-test docs-test-grpc
+endif
 
 ty: pkg-ty-all
 
@@ -137,8 +168,8 @@ lint-fix:
 	uv run ruff check --fix
 
 .PHONY: default help docs docs-all docs-serve docs-serve-all docs-clean docs-test \
-	docs-check-grpc docs-test-grpc docs-linkcheck docs-generate-grpc pkg-test-all pkg-ty-all build generate sync \
-	clean clean-venv clean-build clean-test clean-all test test-all ty-all docs \
+	docs-check-grpc docs-test-grpc docs-linkcheck docs-generate-grpc pkg-test-all pkg-test-all-parallel pkg-ty-all build generate sync \
+	clean clean-venv clean-build clean-test clean-all test test-report test-all ty-all docs \
 	lint lint-fix \
 	pkg-ty-jumpstarter \
 	pkg-ty-jumpstarter-cli-admin \

python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/driver.py

@@ -1,6 +1,7 @@
 import asyncio
 import asyncio.subprocess
 import ipaddress
+import os
 import shutil
 import socket
 import subprocess
@@ -173,16 +174,18 @@ def _check_system_requirements(self) -> None:
         if sys.platform != "linux":
             raise RuntimeError("DutNetwork driver requires Linux (network namespaces, nftables)")
 
+        search_path = os.environ.get("PATH", "") + os.pathsep + "/usr/sbin" + os.pathsep + "/sbin"
+
         missing = []
-        if not shutil.which("ip"):
+        if not shutil.which("ip", path=search_path):
             missing.append("ip (iproute2)")
-        if not shutil.which("nft") and not self._nat_disabled():
+        if not shutil.which("nft", path=search_path) and not self._nat_disabled():
             missing.append("nft (nftables)")
-        if not shutil.which("dnsmasq") and self.dhcp_enabled:
+        if not shutil.which("dnsmasq", path=search_path) and self.dhcp_enabled:
             missing.append("dnsmasq")
-        if not shutil.which("sysctl") and not self._nat_disabled():
+        if not shutil.which("sysctl", path=search_path) and not self._nat_disabled():
             missing.append("sysctl")
-        if not shutil.which("tcpdump") and self.enable_tcpdump:
+        if not shutil.which("tcpdump", path=search_path) and self.enable_tcpdump:
             missing.append("tcpdump")
 
         if missing:

python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/iproute.py

@@ -1,13 +1,21 @@
 """Interface management via iproute2 commands with NetworkManager awareness."""
 
 import logging
+import os
 import shutil
 import subprocess
 
 from ._privilege import sudo_cmd
 
 logger = logging.getLogger(__name__)
 
+_SBIN_PATH = os.environ.get("PATH", "") + os.pathsep + "/usr/sbin" + os.pathsep + "/sbin"
+
+
+def _resolve_tool(name: str) -> str:
+    """Resolve a tool name to its full path, searching /usr/sbin and /sbin."""
+    return shutil.which(name, path=_SBIN_PATH) or name
+
 
 def _run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
     """Run a read-only command (no sudo)."""
@@ -81,7 +89,7 @@ def remove_ip_alias(interface: str, ip: str, prefix_len: int) -> None:
 
 def get_interface_forwarding(iface: str) -> str:
     """Return the current per-interface forwarding value ("0" or "1")."""
-    result = _run(["sysctl", "-n", f"net.ipv4.conf.{iface}.forwarding"], check=False)
+    result = _run([_resolve_tool("sysctl"), "-n", f"net.ipv4.conf.{iface}.forwarding"], check=False)
     return result.stdout.strip() or "0"
 
 
@@ -94,7 +102,7 @@ def set_interface_forwarding(iface: str, enabled: bool) -> None:
     """
     value = "1" if enabled else "0"
     logger.info("Setting net.ipv4.conf.%s.forwarding=%s", iface, value)
-    _run_priv(["sysctl", "-w", f"net.ipv4.conf.{iface}.forwarding={value}"])
+    _run_priv([_resolve_tool("sysctl"), "-w", f"net.ipv4.conf.{iface}.forwarding={value}"])
 
 
 def detect_upstream_interface() -> str | None:

python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_iproute.py

@@ -96,24 +96,28 @@ def test_nm_set_unmanaged_calls_nmcli_when_present(self):
 class TestGetInterfaceForwarding:
     def test_returns_current_value(self):
         fake = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n")
-        with patch.object(iproute, "_run", return_value=fake):
+        with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \
+             patch.object(iproute, "_run", return_value=fake):
             assert iproute.get_interface_forwarding("eth0") == "1"
 
     def test_returns_zero_on_failure(self):
         fake = subprocess.CompletedProcess(args=[], returncode=1, stdout="")
-        with patch.object(iproute, "_run", return_value=fake):
+        with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \
+             patch.object(iproute, "_run", return_value=fake):
             assert iproute.get_interface_forwarding("eth0") == "0"
 
     def test_uses_correct_sysctl_key(self):
         fake = subprocess.CompletedProcess(args=[], returncode=0, stdout="0\n")
-        with patch.object(iproute, "_run", return_value=fake) as mock_run:
+        with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \
+             patch.object(iproute, "_run", return_value=fake) as mock_run:
             iproute.get_interface_forwarding("eth0")
             mock_run.assert_called_once_with(
                 ["sysctl", "-n", "net.ipv4.conf.eth0.forwarding"], check=False
             )
 
     def test_set_interface_forwarding(self):
-        with patch.object(iproute, "_run_priv") as mock:
+        with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \
+             patch.object(iproute, "_run_priv") as mock:
             iproute.set_interface_forwarding("eth0", True)
             mock.assert_called_once_with(
                 ["sysctl", "-w", "net.ipv4.conf.eth0.forwarding=1"]

python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_tcpdump.py

@@ -73,7 +73,7 @@ def test_tcpdump_missing_binary_raises_when_enabled(self, tmp_path: Path):
                  patch(f"{_DRIVER_MODULE}.dnsmasq") as mock_dnsmasq:
                 mock_sys.platform = "linux"
 
-                def which_side_effect(cmd):
+                def which_side_effect(cmd, **kwargs):
                     if cmd == "tcpdump":
                         return None
                     return "/usr/bin/fake"
@@ -108,7 +108,7 @@ def test_tcpdump_missing_binary_ok_when_disabled(self, tmp_path: Path):
              patch(f"{_DRIVER_MODULE}.dnsmasq") as mock_dnsmasq:
             mock_sys.platform = "linux"
 
-            def which_side_effect(cmd):
+            def which_side_effect(cmd, **kwargs):
                 if cmd == "tcpdump":
                     return None
                 return "/usr/bin/fake"

python/packages/jumpstarter-driver-renode/jumpstarter_driver_renode/monitor.py

@@ -33,7 +33,7 @@ def add_expected_prompt(self, name: str) -> None:
         """Register a machine name so its prompt is recognised."""
         self._expected_prompts.add(name.encode())
 
-    async def connect(self, host: str, port: int, timeout: float = 10) -> None:
+    async def connect(self, host: str, port: int, timeout: float = 45) -> None:
         """Connect to the Renode monitor, retrying until the prompt appears."""
         with fail_after(timeout):
             while True: