diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 12f92a9cd..db87dc7dd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -48,7 +48,7 @@ jobs: strategy: matrix: include: - - os: ubuntu-24.04 + - os: arc-runner-kubevirt-small arch: amd64 - os: ubuntu-24.04-arm arch: arm64 @@ -91,7 +91,7 @@ jobs: strategy: matrix: include: - - os: ubuntu-24.04 + - os: arc-runner-kubevirt-small arch: amd64 - os: ubuntu-24.04-arm arch: arm64 @@ -136,7 +136,7 @@ jobs: build-python-wheels: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') - runs-on: ubuntu-24.04 + runs-on: arc-runner-kubevirt-small timeout-minutes: 15 steps: - name: Checkout repository @@ -180,7 +180,7 @@ jobs: strategy: matrix: include: - - os: ubuntu-24.04 + - os: arc-runner-kubevirt-large arch: amd64 - os: ubuntu-24.04-arm arch: arm64 @@ -245,7 +245,7 @@ jobs: e2e-compat-old-controller: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-24.04 + runs-on: arc-runner-kubevirt-large timeout-minutes: 60 steps: - name: Checkout repository @@ -284,7 +284,7 @@ jobs: e2e-compat-old-client: needs: [changes, build-controller-image, build-operator-image, build-python-wheels] if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-24.04 + runs-on: arc-runner-kubevirt-large timeout-minutes: 60 steps: - name: Checkout repository diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index 9334de073..8c02973d8 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -42,8 +42,9 @@ jobs: if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' runs-on: ${{ matrix.runs-on }} strategy: + fail-fast: false matrix: - runs-on: [ubuntu-24.04, macos-15] + runs-on: [arc-runner-kubevirt-small, macos-15] # Floor: oldest Python in supported platforms (RHEL 9 appstream) # Ceiling: newest Python in latest Fedora # Review on each RHEL/Fedora release @@ -61,48 +62,44 @@ jobs: version: ${{ steps.uv.outputs.version }} python-version: ${{ matrix.python-version }} - - name: Install Qemu (Linux) + - name: Setup Linux dependencies if: runner.os == 'Linux' run: | + # udev rules and kernel modules (idempotent) echo </dev/null || true done - - - name: Install Renode (Linux) - if: runner.os == 'Linux' - run: | - wget https://github.com/renode/renode/releases/download/v1.16.1/renode_1.16.1_amd64.deb -O /tmp/renode.deb - sudo apt-get install -y /tmp/renode.deb + sudo chmod 0666 /dev/kvm /dev/vhost-vsock /dev/vhost-net 2>/dev/null || true + + # collect missing packages + pkgs=() + command -v qemu-system-arm &>/dev/null || pkgs+=(qemu-system-arm) + command -v qemu-system-x86_64 &>/dev/null || pkgs+=(qemu-system-x86) + dpkg -s libgpiod-dev &>/dev/null || pkgs+=(libgpiod-dev) + dpkg -s liblgpio-dev &>/dev/null || pkgs+=(liblgpio-dev) + command -v nft &>/dev/null || pkgs+=(nftables) + command -v dnsmasq &>/dev/null || pkgs+=(dnsmasq-base) + command -v dhclient &>/dev/null || pkgs+=(isc-dhcp-client) + command -v dig &>/dev/null || pkgs+=(dnsutils) + command -v renode &>/dev/null || pkgs+=(renode) + command -v rpm2cpio &>/dev/null || pkgs+=(rpm2cpio) + command -v cpio &>/dev/null || pkgs+=(cpio) + + if [ ${#pkgs[@]} -gt 0 ]; then + # add renode repo if needed + if [[ " ${pkgs[*]} " == *" renode "* ]]; then + wget -qO /tmp/renode.deb https://github.com/renode/renode/releases/download/v1.16.1/renode_1.16.1_amd64.deb + pkgs=("${pkgs[@]/renode//tmp/renode.deb}") + fi + sudo apt-get update + sudo apt-get install -y "${pkgs[@]}" + fi - name: Install Qemu (macOS) if: runner.os == 'macOS' @@ -138,9 +135,18 @@ jobs: - name: Run pytest working-directory: python env: - PYTEST_ADDOPTS: "--cov-report=xml" + PYTEST_ADDOPTS: "--cov-report=xml --log-level=CRITICAL --log-cli-level=CRITICAL" run: | - make test + make test -j4 LOGS_DIR=${{ runner.temp }}/test-logs + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-logs-${{ matrix.runs-on }}-py${{ matrix.python-version }} + path: ${{ runner.temp }}/test-logs/ + if-no-files-found: ignore + retention-days: 7 # Diff-coverage is only checked on Linux. Several packages (e.g. # jumpstarter-driver-dut-network) are Linux-only, so their tests are @@ -156,7 +162,7 @@ jobs: echo "::error::No coverage.xml files found" exit 1 fi - uv run diff-cover $coverage_files --compare-branch=origin/${{ github.base_ref }} --fail-under=80 --exclude '*_pb2.py' '*_pb2_grpc.py' '**/conftest.py' + uv run diff-cover $coverage_files --compare-branch=origin/${{ github.base_ref }} --fail-under=80 --exclude '*_pb2.py' '*_pb2_grpc.py' '**/conftest.py' '**/test_*.py' '**/*_test.py' # https://github.com/orgs/community/discussions/26822 pytest: diff --git a/e2e/test/dut_network_test.go b/e2e/test/dut_network_test.go index 1b3fe5cda..b579d5f88 100644 --- a/e2e/test/dut_network_test.go +++ b/e2e/test/dut_network_test.go @@ -121,11 +121,12 @@ var _ = Describe("DUT Network E2E Tests", Label("dut-network"), Ordered, func() setupNetworkNamespaces() configPath := filepath.Join(exporterDir, "exporter-dut-network.yaml") - tracker.StartDirectExporter(configPath, listenerPort, "", false) + tracker.StartDirectExporter(configPath, listenerPort, "", true) WaitForDirectExporterReady(listenerPort, "") }) AfterAll(func() { + tracker.DumpLogs(50) tracker.StopAll() teardownNetworkNamespaces() diff --git a/e2e/test/utils.go b/e2e/test/utils.go index 75334e0fe..545b2a0b5 100644 --- a/e2e/test/utils.go +++ b/e2e/test/utils.go @@ -439,7 +439,8 @@ func (pt *ProcessTracker) StartDirectExporter(configFile string, port int, passp var stderrBuf *logBuffer if captureStderr { - stderrBuf = pt.getOrCreateLog("direct-exporter-stderr") + logName := "direct-exporter-stderr-" + strconv.Itoa(port) + stderrBuf = pt.getOrCreateLog(logName) cmd.Stderr = stderrBuf } diff --git a/python/Makefile b/python/Makefile index 2a0a32cb9..7b956be93 100644 --- a/python/Makefile +++ b/python/Makefile @@ -80,14 +80,41 @@ docs-test: docs-generate-crds docs-generate-grpc docs-linkcheck: docs-generate-crds docs-generate-grpc uv run --isolated --all-packages --group docs $(MAKE) -C docs linkcheck +ifdef LOGS_DIR +pkg-test-%: packages/% + @mkdir -p $(LOGS_DIR) + @rm -f $(LOGS_DIR)/$*.failed + @bash -c 'set -o pipefail; \ + PYTHONUNBUFFERED=1 uv run --isolated --directory $< pytest 2>&1 | tee $(LOGS_DIR)/$*.log; \ + rc=$$?; \ + if [ $$rc -ne 0 ] && [ $$rc -ne 5 ]; then \ + touch $(LOGS_DIR)/$*.failed; \ + fi; \ + true' +else pkg-test-%: packages/% uv run --isolated --directory $< pytest || [ $$? -eq 5 ] +endif pkg-ty-%: packages/% uv run --isolated --directory $< ty check . pkg-test-all: $(addprefix pkg-test-,$(PKG_TARGETS)) +ifdef LOGS_DIR +test-report: + @failed=0; \ + for f in $(LOGS_DIR)/*.failed; do \ + [ -f "$$f" ] || continue; \ + pkg=$$(basename "$$f" .failed); \ + echo ""; \ + echo "========== FAILED: $$pkg =========="; \ + cat "$(LOGS_DIR)/$$pkg.log"; \ + failed=1; \ + done; \ + [ $$failed -eq 0 ] && echo "All package tests passed." || exit 1 +endif + pkg-ty-all: $(addprefix pkg-ty-,$(PKG_TARGETS)) build: @@ -126,7 +153,11 @@ clean-docs: clean: clean-docs clean-venv clean-build clean-test +ifdef LOGS_DIR +test: pkg-test-all docs-test docs-test-grpc test-report +else test: pkg-test-all docs-test docs-test-grpc +endif ty: pkg-ty-all @@ -137,8 +168,8 @@ lint-fix: uv run ruff check --fix .PHONY: default help docs docs-all docs-serve docs-serve-all docs-clean docs-test \ - docs-check-grpc docs-test-grpc docs-linkcheck docs-generate-grpc pkg-test-all pkg-ty-all build generate sync \ - clean clean-venv clean-build clean-test clean-all test test-all ty-all docs \ + docs-check-grpc docs-test-grpc docs-linkcheck docs-generate-grpc pkg-test-all pkg-test-all-parallel pkg-ty-all build generate sync \ + clean clean-venv clean-build clean-test clean-all test test-report test-all ty-all docs \ lint lint-fix \ pkg-ty-jumpstarter \ pkg-ty-jumpstarter-cli-admin \ diff --git a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/driver.py b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/driver.py index 8f43b65d5..7a7a9961b 100644 --- a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/driver.py +++ b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/driver.py @@ -1,6 +1,7 @@ import asyncio import asyncio.subprocess import ipaddress +import os import shutil import socket import subprocess @@ -173,16 +174,18 @@ def _check_system_requirements(self) -> None: if sys.platform != "linux": raise RuntimeError("DutNetwork driver requires Linux (network namespaces, nftables)") + search_path = os.environ.get("PATH", "") + os.pathsep + "/usr/sbin" + os.pathsep + "/sbin" + missing = [] - if not shutil.which("ip"): + if not shutil.which("ip", path=search_path): missing.append("ip (iproute2)") - if not shutil.which("nft") and not self._nat_disabled(): + if not shutil.which("nft", path=search_path) and not self._nat_disabled(): missing.append("nft (nftables)") - if not shutil.which("dnsmasq") and self.dhcp_enabled: + if not shutil.which("dnsmasq", path=search_path) and self.dhcp_enabled: missing.append("dnsmasq") - if not shutil.which("sysctl") and not self._nat_disabled(): + if not shutil.which("sysctl", path=search_path) and not self._nat_disabled(): missing.append("sysctl") - if not shutil.which("tcpdump") and self.enable_tcpdump: + if not shutil.which("tcpdump", path=search_path) and self.enable_tcpdump: missing.append("tcpdump") if missing: diff --git a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/iproute.py b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/iproute.py index 526034faf..ad9a0219f 100644 --- a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/iproute.py +++ b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/iproute.py @@ -1,6 +1,7 @@ """Interface management via iproute2 commands with NetworkManager awareness.""" import logging +import os import shutil import subprocess @@ -8,6 +9,13 @@ logger = logging.getLogger(__name__) +_SBIN_PATH = os.environ.get("PATH", "") + os.pathsep + "/usr/sbin" + os.pathsep + "/sbin" + + +def _resolve_tool(name: str) -> str: + """Resolve a tool name to its full path, searching /usr/sbin and /sbin.""" + return shutil.which(name, path=_SBIN_PATH) or name + def _run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess: """Run a read-only command (no sudo).""" @@ -81,7 +89,7 @@ def remove_ip_alias(interface: str, ip: str, prefix_len: int) -> None: def get_interface_forwarding(iface: str) -> str: """Return the current per-interface forwarding value ("0" or "1").""" - result = _run(["sysctl", "-n", f"net.ipv4.conf.{iface}.forwarding"], check=False) + result = _run([_resolve_tool("sysctl"), "-n", f"net.ipv4.conf.{iface}.forwarding"], check=False) return result.stdout.strip() or "0" @@ -94,7 +102,7 @@ def set_interface_forwarding(iface: str, enabled: bool) -> None: """ value = "1" if enabled else "0" logger.info("Setting net.ipv4.conf.%s.forwarding=%s", iface, value) - _run_priv(["sysctl", "-w", f"net.ipv4.conf.{iface}.forwarding={value}"]) + _run_priv([_resolve_tool("sysctl"), "-w", f"net.ipv4.conf.{iface}.forwarding={value}"]) def detect_upstream_interface() -> str | None: diff --git a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_iproute.py b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_iproute.py index ea8a851d0..e19ba7ed2 100644 --- a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_iproute.py +++ b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_iproute.py @@ -96,24 +96,28 @@ def test_nm_set_unmanaged_calls_nmcli_when_present(self): class TestGetInterfaceForwarding: def test_returns_current_value(self): fake = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n") - with patch.object(iproute, "_run", return_value=fake): + with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \ + patch.object(iproute, "_run", return_value=fake): assert iproute.get_interface_forwarding("eth0") == "1" def test_returns_zero_on_failure(self): fake = subprocess.CompletedProcess(args=[], returncode=1, stdout="") - with patch.object(iproute, "_run", return_value=fake): + with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \ + patch.object(iproute, "_run", return_value=fake): assert iproute.get_interface_forwarding("eth0") == "0" def test_uses_correct_sysctl_key(self): fake = subprocess.CompletedProcess(args=[], returncode=0, stdout="0\n") - with patch.object(iproute, "_run", return_value=fake) as mock_run: + with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \ + patch.object(iproute, "_run", return_value=fake) as mock_run: iproute.get_interface_forwarding("eth0") mock_run.assert_called_once_with( ["sysctl", "-n", "net.ipv4.conf.eth0.forwarding"], check=False ) def test_set_interface_forwarding(self): - with patch.object(iproute, "_run_priv") as mock: + with patch.object(iproute, "_resolve_tool", return_value="sysctl"), \ + patch.object(iproute, "_run_priv") as mock: iproute.set_interface_forwarding("eth0", True) mock.assert_called_once_with( ["sysctl", "-w", "net.ipv4.conf.eth0.forwarding=1"] diff --git a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_tcpdump.py b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_tcpdump.py index ba4909ce2..9b89b9255 100644 --- a/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_tcpdump.py +++ b/python/packages/jumpstarter-driver-dut-network/jumpstarter_driver_dut_network/test_tcpdump.py @@ -73,7 +73,7 @@ def test_tcpdump_missing_binary_raises_when_enabled(self, tmp_path: Path): patch(f"{_DRIVER_MODULE}.dnsmasq") as mock_dnsmasq: mock_sys.platform = "linux" - def which_side_effect(cmd): + def which_side_effect(cmd, **kwargs): if cmd == "tcpdump": return None return "/usr/bin/fake" @@ -108,7 +108,7 @@ def test_tcpdump_missing_binary_ok_when_disabled(self, tmp_path: Path): patch(f"{_DRIVER_MODULE}.dnsmasq") as mock_dnsmasq: mock_sys.platform = "linux" - def which_side_effect(cmd): + def which_side_effect(cmd, **kwargs): if cmd == "tcpdump": return None return "/usr/bin/fake" diff --git a/python/packages/jumpstarter-driver-renode/jumpstarter_driver_renode/monitor.py b/python/packages/jumpstarter-driver-renode/jumpstarter_driver_renode/monitor.py index 83211f8e5..42826d4a1 100644 --- a/python/packages/jumpstarter-driver-renode/jumpstarter_driver_renode/monitor.py +++ b/python/packages/jumpstarter-driver-renode/jumpstarter_driver_renode/monitor.py @@ -33,7 +33,7 @@ def add_expected_prompt(self, name: str) -> None: """Register a machine name so its prompt is recognised.""" self._expected_prompts.add(name.encode()) - async def connect(self, host: str, port: int, timeout: float = 10) -> None: + async def connect(self, host: str, port: int, timeout: float = 45) -> None: """Connect to the Renode monitor, retrying until the prompt appears.""" with fail_after(timeout): while True: diff --git a/python/packages/jumpstarter-driver-uboot/jumpstarter_driver_uboot/driver_test.py b/python/packages/jumpstarter-driver-uboot/jumpstarter_driver_uboot/driver_test.py index 5e40b7c7a..d79800a61 100644 --- a/python/packages/jumpstarter-driver-uboot/jumpstarter_driver_uboot/driver_test.py +++ b/python/packages/jumpstarter-driver-uboot/jumpstarter_driver_uboot/driver_test.py @@ -1,5 +1,7 @@ import os import platform +import shutil +import subprocess import pytest import requests @@ -10,25 +12,45 @@ from .driver import UbootConsole from jumpstarter.common.utils import serve +UBOOT_RPM_URL = "https://kojipkgs.fedoraproject.org/packages/uboot-tools/2025.10/1.fc43/noarch/uboot-images-armv8-2025.10-1.fc43.noarch.rpm" + @pytest.fixture(scope="session") def uboot_image(tmpdir_factory): tmp_path = tmpdir_factory.mktemp("uboot-images") - - url = "https://kojipkgs.fedoraproject.org/packages/uboot-tools/2025.10/1.fc43/noarch/uboot-images-armv8-2025.10-1.fc43.noarch.rpm" - - with requests.get(url, stream=True) as r: - r.raise_for_status() - with (tmp_path / "uboot-images-armv8.rpm").open("wb") as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - - with rpmfile.open(tmp_path / "uboot-images-armv8.rpm") as rpm: - fd = rpm.extractfile("./usr/share/uboot/qemu_arm64/u-boot.bin") - with (tmp_path / "u-boot.bin").open("wb") as f: - f.write(fd.read()) - - yield tmp_path / "u-boot.bin" + rpm_path = tmp_path / "uboot-images-armv8.rpm" + bin_path = tmp_path / "u-boot.bin" + + print(f"\nDownloading u-boot RPM from {UBOOT_RPM_URL}") + try: + with requests.get(UBOOT_RPM_URL, stream=True, timeout=120) as r: + r.raise_for_status() + total = int(r.headers.get("content-length", 0)) + downloaded = 0 + with rpm_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + downloaded += len(chunk) + print(f"Downloaded {downloaded} bytes (expected {total})") + except requests.RequestException as e: + raise AssertionError(f"Failed to download u-boot RPM: {e}") from e + + print("Extracting u-boot.bin from RPM...") + if shutil.which("rpm2cpio") and shutil.which("cpio"): + subprocess.run( + f"rpm2cpio {rpm_path} | cpio -idm --quiet ./usr/share/uboot/qemu_arm64/u-boot.bin", + shell=True, cwd=str(tmp_path), check=True, + ) + extracted = tmp_path / "usr" / "share" / "uboot" / "qemu_arm64" / "u-boot.bin" + extracted.rename(bin_path) + else: + with rpmfile.open(rpm_path) as rpm: + fd = rpm.extractfile("./usr/share/uboot/qemu_arm64/u-boot.bin") + with bin_path.open("wb") as f: + f.write(fd.read()) + print(f"Extracted u-boot.bin ({bin_path.size()} bytes)") + + yield bin_path @pytest.mark.xfail(