backend-e2e.yml

name: Backend · E2E

# Three E2E gates, all against a throw-away GUID Postgres. ALL E2E tests live in CodeSpace.E2ETests
# (Category=E2E); the gates split it by the Surface trait because they need DIFFERENT runtimes:
#
#   1. e2e-http (Surface=Http) — the REAL ASP.NET pipeline (routing, [AllowAnonymous], model binding,
#      the GlobalExceptionFilter) in-memory via WebApplicationFactory: the actual HTTP status code a
#      webhook provider receives. Runs directly on the host runner, so it reaches Postgres at
#      localhost:5432 (the committed appsettings.json admin connection).
#
#   2. e2e-engine (Surface=Engine) — THE headline intelligent flow driven top-to-bottom as one engine run:
#      trigger.manual → llm.complete(responseSchema) planner → flow.map fan-out whose body is a REAL
#      agent.run that ACTUALLY EXECUTES (real AgentRunExecutor → real LocalProcessRunner → a fake
#      codex CLI process under REAL bubblewrap confinement → real completion → resume) → a synthesizer
#      that reduces the per-branch results. These engine-tier tests reuse the shared Postgres fixture +
#      seed infra from CodeSpace.IntegrationTests (referenced as a project). bubblewrap is installed +
#      Sandbox__RequireConfinement=true so the agent runs CONFINED (the production posture) and a degraded
#      sandbox fails HARD rather than silently running unconfined. It runs in a PRIVILEGED container
#      (mirroring sandbox-isolation.yml) so unprivileged user namespaces work despite ubuntu-latest's
#      AppArmor restriction — and so reaches Postgres by the service ALIAS (postgres:5432), overridden
#      via the TestPostgres__AdminConnectionString env var.
#
#   3. e2e-worker (Surface=Worker) — the PROCESSING role, which no other lane boots: a host with
#      HangfireHosting=Worker, its two real Hangfire servers draining a real Postgres queue, every
#      IRecurringJob registered by the real WorkerHangfireRegistrar and fired once. It needs no bubblewrap,
#      so it could have run inside e2e-http; it gets its own job because it is the only lane that EXECUTES
#      jobs. A Worker host holds ControlWorkerCount + ProcessorCount * 2 worker connections for its lifetime
#      (38 measured on a 12-core box — see FactoryHangfireRoleTests) and its ticks run the real sweeps,
#      filesystem reapers included. Its own Postgres service keeps that load and that blast radius off the
#      HTTP lane's database server, and its own check line keeps "do the recurring jobs still fire" legible
#      in the checks list rather than buried in a 159-test lane. In-process interference is handled
#      separately, by the assembly-wide DisableTestParallelization in AssemblyParallelization.cs.

on:
  push:
    branches: [main]
    paths:
      - 'backend/**'
      - '.github/workflows/backend-e2e.yml'
  pull_request:
    paths:
      - 'backend/**'
      - '.github/workflows/backend-e2e.yml'
  workflow_dispatch:

concurrency:
  group: backend-e2e-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  DOTNET_NOLOGO: 'true'
  # The DI container fail-fasts without a Variables master key outside Development (the settings arc's
  # guard). CI hosts get the SAME committed local-dev test key docker-compose.yml already carries —
  # a committed value, not a toggle; production deployments still supply their own real key.
  CODESPACE_VARIABLE_MASTER_KEY: 'bG9jYWwtZGV2LW9ubHkta2V5LW5vdC1mb3ItcHJvZCE='
  DOTNET_CLI_TELEMETRY_OPTOUT: 'true'
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 'true'

jobs:
  e2e-http:
    name: dotnet test (E2ETests · HTTP · Postgres)
    # Pinned, not ubuntu-latest: on ubuntu-26.04 an unprivileged bwrap works, so the suites switch to confined mode; they assume the unconfined posture (confinement is covered by sandbox-isolation.yml)
    runs-on: ubuntu-24.04
    timeout-minutes: 30

    services:
      postgres:
        image: postgres:18-alpine
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: '123456'
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d postgres"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 20

    steps:
      - uses: actions/checkout@v4

      - name: Set up .NET 10
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: '10.0.x'

      - name: Cache NuGet packages
        uses: actions/cache@v4
        with:
          path: ~/.nuget/packages
          key: nuget-e2e-${{ runner.os }}-${{ hashFiles('backend/**/*.csproj') }}
          restore-keys: nuget-e2e-${{ runner.os }}-

      - name: Restore
        run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj

      - name: Test (Surface=Http — the REAL ASP.NET pipeline via WebApplicationFactory)
        run: >
          dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj
          --no-restore
          --filter "Category=E2E&Surface=Http"
          --logger "console;verbosity=normal"
          --logger "trx;LogFileName=e2e-http.trx"
          --results-directory backend/TestResults
          --blame-hang-timeout 5m

      - name: Assert the HTTP E2E actually ran (no silent skip)
        # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a Surface-trait regression
        # could green this gate while exercising nothing. Require at least one HTTP E2E to have executed.
        run: |
          trx=backend/TestResults/e2e-http.trx
          test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; }
          executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
          echo "executed=${executed:-0}"
          if [ "${executed:-0}" -lt 1 ]; then
            echo "::error::Expected Category=E2E&Surface=Http tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?)."
            exit 1
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: backend-e2e-http-trx
          path: backend/TestResults/*.trx
          if-no-files-found: ignore

  e2e-engine:
    name: headline flow (planner → map → real agents → synthesizer)
    runs-on: ubuntu-latest
    timeout-minutes: 30

    # Runs in a PRIVILEGED container so unprivileged user namespaces work regardless of the host runner's
    # AppArmor policy — current ubuntu-latest (Ubuntu 24.04) ships kernel.apparmor_restrict_unprivileged_userns=1,
    # which denies unprivileged userns for an apt-installed bwrap. The bwrap-confined agent in this E2E needs
    # working userns, so we mirror the already-green sibling sandbox-isolation.yml (`docker run --privileged
    # dotnet/sdk:10.0 + apt install bubblewrap`) — the exact environment these tests were validated against.
    container:
      image: mcr.microsoft.com/dotnet/sdk:10.0
      options: --privileged

    env:
      # Fail-closed: the headline E2E runs a REAL agent.run branch through the real LocalProcessRunner,
      # which confines it under bubblewrap. With REQUIRE_SANDBOX=1 a host that can't sandbox (no bwrap /
      # no unprivileged userns) makes the run fail HARD instead of silently executing the agent unconfined
      # — so green here means the agent really ran AND was really confined.
      Sandbox__RequireConfinement: 'true'
      # Inside a job container the Postgres SERVICE container is reachable by its network ALIAS (`postgres`),
      # NOT via the host port mapping (`localhost:5432` is the job container itself). Override the test
      # admin connection string (TestPostgres:AdminConnectionString → env double-underscore form) to target
      # the service alias. The e2e-http job stays on `localhost` because it runs directly on the host runner.
      TestPostgres__AdminConnectionString: 'Host=postgres;Port=5432;Username=postgres;Password=123456;Database=postgres'

    services:
      postgres:
        image: postgres:18-alpine
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: '123456'
          POSTGRES_DB: postgres
        # No `ports:` mapping — inside a container job the service is reached by its alias on the Docker
        # network, so the host-port publish is unnecessary.
        options: >-
          --health-cmd "pg_isready -U postgres -d postgres"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 20

    steps:
      - uses: actions/checkout@v4

      - name: Install bubblewrap + util-linux (prlimit) + iproute2/nftables
        # The headline E2E spawns a REAL agent process through the production runner, which wraps it in
        # bubblewrap + prlimit (the production confinement posture). Running as root in the SDK container, so
        # no sudo. ca-certificates so the container can fetch packages; util-linux for prlimit; iproute2 and
        # nftables because the worker image ships them for allowlist runs. A network-off brokered agent needs
        # neither: it reaches its broker through the codespace-mcp relay, which the test project's build places
        # beside its assembly, and without which it is refused as sandbox_sealed_egress_unavailable.
        run: |
          apt-get update
          apt-get install -y --no-install-recommends bubblewrap util-linux ca-certificates iproute2 nftables
          echo "bwrap: $(bwrap --version)"
          echo "prlimit: $(prlimit --version)"

      - name: Probe — unprivileged user namespaces (informational)
        # Informational only: the authoritative gate is the test step below (REQUIRE_SANDBOX=1 → the run fails
        # HARD if confinement is unavailable, surfacing as the headline assertion failing). This just surfaces
        # a clear "userns works/doesn't" signal early without ever failing the job on a bwrap-flag quirk —
        # turning an opaque "run was Failed" into a diagnosable cause.
        run: |
          if bwrap --ro-bind / / --dev /dev --proc /proc --unshare-all --uid 0 true; then
            echo "✓ unprivileged user namespaces work"
          else
            echo "⚠ userns probe failed — the REQUIRE_SANDBOX=1 test step is authoritative and will fail hard if confinement is truly unavailable"
          fi

      - name: Cache NuGet packages
        uses: actions/cache@v4
        with:
          path: /root/.nuget/packages
          key: nuget-e2e-engine-${{ hashFiles('backend/**/*.csproj') }}
          restore-keys: nuget-e2e-engine-

      - name: Restore
        run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj

      - name: Test (Surface=Engine — the headline planner→map→real-agents→synthesizer flow)
        run: >
          dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj
          --no-restore
          --filter "Category=E2E&Surface=Engine"
          --logger "console;verbosity=normal"
          --logger "trx;LogFileName=e2e-engine.trx"
          --results-directory backend/TestResults
          --blame-hang-timeout 5m

      - name: Assert the headline E2E actually ran (no silent skip)
        # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a trait regression
        # could make this gate pass green while exercising nothing. Read the trx counters and require at
        # least the one headline-flow test to have executed (NOT skipped — the OS guard would skip it on a
        # non-Linux host, but this gate IS Linux, so a skip here means the real run was silently dodged).
        run: |
          trx=backend/TestResults/e2e-engine.trx
          test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; }
          executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
          passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
          echo "executed=${executed:-0} passed=${passed:-0}"
          if [ "${executed:-0}" -lt 1 ]; then
            echo "::error::Expected the Category=E2E&Surface=Engine headline-flow tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?) or it was silently skipped."
            exit 1
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: backend-e2e-engine-trx
          path: backend/TestResults/*.trx
          if-no-files-found: ignore

  e2e-worker:
    name: recurring jobs fire (worker host · Postgres)
    # Pinned, not ubuntu-latest: on ubuntu-26.04 an unprivileged bwrap works, so the suites switch to confined mode; they assume the unconfined posture (confinement is covered by sandbox-isolation.yml)
    runs-on: ubuntu-24.04
    timeout-minutes: 20

    services:
      postgres:
        image: postgres:18-alpine
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: '123456'
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d postgres"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 20

    steps:
      - uses: actions/checkout@v4

      - name: Set up .NET 10
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: '10.0.x'

      - name: Cache NuGet packages
        uses: actions/cache@v4
        with:
          path: ~/.nuget/packages
          key: nuget-e2e-${{ runner.os }}-${{ hashFiles('backend/**/*.csproj') }}
          restore-keys: nuget-e2e-${{ runner.os }}-

      - name: Restore
        run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj

      - name: Test (Surface=Worker — every recurring job registered and fired by the real worker host)
        run: >
          dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj
          --no-restore
          --filter "Category=E2E&Surface=Worker"
          --logger "console;verbosity=normal"
          --logger "trx;LogFileName=e2e-worker.trx"
          --results-directory backend/TestResults
          --blame-hang-timeout 10m

      - name: Assert the worker E2E actually ran (no silent skip)
        # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a Surface-trait regression could
        # green this gate while firing nothing — which is the exact condition this gate was added to end.
        run: |
          trx=backend/TestResults/e2e-worker.trx
          test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; }
          executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
          echo "executed=${executed:-0}"
          if [ "${executed:-0}" -lt 1 ]; then
            echo "::error::Expected Category=E2E&Surface=Worker tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?)."
            exit 1
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: backend-e2e-worker-trx
          path: backend/TestResults/*.trx
          if-no-files-found: ignore