fix: retry git fetches that stall

.github/workflows/e2e.yml

@@ -26,12 +26,18 @@ concurrency:
   group: e2e-${{ github.ref }}
   cancel-in-progress: true
 
-# The Linux job runs the suite natively, the way the dev container runs it
-# locally: Go at the dev container's version (.devcontainer/Dockerfile) plus
-# the runner's stock Docker, with the same TESTCONTAINERS_RYUK_DISABLED knob.
-# The Windows jobs keep the compose runner image
-# (tests/e2e/windows/Dockerfile-runner), which carries its own Go toolchain,
-# so a Windows host needs only Docker. See docs/e2e.md.
+# Both jobs run the suite natively: Go at the dev container's version
+# (.devcontainer/Dockerfile) plus the runner's Docker. The Linux job runs it
+# the way the dev container runs it locally, with the same
+# TESTCONTAINERS_RYUK_DISABLED knob. The Windows jobs skip the compose
+# runner image local Windows hosts use (tests/e2e/windows/Dockerfile-runner):
+# baking Go into a Windows image costs minutes per run. See docs/e2e.md.
+
+env:
+  GO_VERSION: '1.26.8'
+  # Runners are discarded with whatever still runs on them, so the suites
+  # tear containers down in the background (common.AsyncCleanup).
+  E2E_ASYNC_CLEANUP: 'true'
 
 jobs:
   e2e-linux:
@@ -53,7 +59,7 @@ jobs:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with:
-          go-version: '1.26.8'
+          go-version: ${{ env.GO_VERSION }}
           cache: false
       - run: go test -v -timeout 55m -run TestE2E ./linux
         working-directory: tests/e2e
@@ -66,27 +72,90 @@ jobs:
       matrix:
         # TODO See https://github.com/actions/runner-images/issues/13729
         runs-on: [windows-latest, windows-11-arm]
-        include:
-          - runs-on: windows-latest
-            compose: tests/e2e/windows/compose.amd64.yaml
-          - runs-on: windows-11-arm
-            compose: tests/e2e/windows/compose.arm64.yaml
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
+    defaults:
+      run:
+        shell: pwsh
+    env:
+      # windows-latest runs six combos, slowest first (common.SlowestFirst):
+      # with four at a time the short ones left over finish within the first
+      # wave. windows-11-arm runs two.
+      E2E_MAX_JOBS: '4'
     steps:
       - uses: actions/checkout@v7
-      # TODO For compiling moby only. Because https://github.com/actions/runner-images/blob/win11-arm64/20260920.174/images/windows/Windows11-Arm64-Readme.md is still using `Go 1.24.13`.
+      # The engine's data-root takes every image pull, layer commit and
+      # in-container Flutter install, and the runners' C: OS disk is slow
+      # remote storage. windows-latest keeps the workspace on a local D:
+      # disk: move the preinstalled engine's data-root next to it.
+      - if: matrix.runs-on == 'windows-latest'
+        name: Move the Docker data-root to the workspace disk
+        run: |
+          $dataRoot = (Split-Path -Qualifier $env:GITHUB_WORKSPACE) + '\docker'
+          $config = Join-Path $env:ProgramData 'docker\config\daemon.json'
+          $settings = if (Test-Path $config) { Get-Content -Raw $config | ConvertFrom-Json } else { [pscustomobject]@{} }
+          $settings | Add-Member -NotePropertyName 'data-root' -NotePropertyValue $dataRoot -Force
+          New-Item -ItemType Directory -Force -Path (Split-Path $config) | Out-Null
+          $settings | ConvertTo-Json | Set-Content -Path $config -Encoding ascii
+          Restart-Service docker
+          for ($attempt = 1; $attempt -le 30; $attempt++) {
+            $current = docker info --format '{{.DockerRootDir}}' 2>$null
+            if ($LASTEXITCODE -eq 0) { break }
+            Start-Sleep -Seconds 2
+          }
+          if ($current -ne $dataRoot) { throw "FAIL the engine runs off '$current', want '$dataRoot'" }
+          Write-Output "PASS the Docker data-root is $current"
+      # windows-11-arm has no D:, but carries an unformatted local NVMe disk
+      # (actions/runner-images#14088): format it for the engine's data-root.
       - if: matrix.runs-on == 'windows-11-arm'
+        id: disk
+        name: Format the local NVMe disk
+        run: |
+          $disk = Get-Disk | Where-Object PartitionStyle -eq 'RAW' | Select-Object -First 1
+          if ($disk) {
+            $volume = $disk | Initialize-Disk -PartitionStyle GPT -PassThru |
+              New-Partition -AssignDriveLetter -UseMaximumSize |
+              Format-Volume -FileSystem NTFS -Confirm:$false
+            "data-root=$($volume.DriveLetter):\vfox-docker\data" >> $env:GITHUB_OUTPUT
+            Write-Output "PASS formatted raw disk $($disk.Number) as $($volume.DriveLetter):"
+          }
+          else {
+            Write-Output '::warning::no raw local disk; the Docker data-root stays on C:'
+          }
+      # Building the engine from source takes minutes; the script reuses the
+      # binaries restored here.
+      - if: matrix.runs-on == 'windows-11-arm'
+        id: moby
+        uses: actions/cache@v6
+        with:
+          path: C:\vfox-docker\bin
+          key: moby-windows-arm64-go${{ env.GO_VERSION }}-${{ hashFiles('tests/e2e/windows/install-docker-arm64.ps1') }}
+      # Go is needed this early only to build the engine; otherwise it is set
+      # up below, while the base image pulls.
+      - if: matrix.runs-on == 'windows-11-arm' && steps.moby.outputs.cache-hit != 'true'
         uses: actions/setup-go@v7
         with:
-          go-version: 1.26.8
+          go-version: ${{ env.GO_VERSION }}
           cache: false
       - if: matrix.runs-on == 'windows-11-arm'
-        run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1
-      - if: matrix.runs-on == 'windows-11-arm'
-        name: Install Compose CLI plugin
+        run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1 -DataRoot '${{ steps.disk.outputs.data-root }}'
+      # Unpacking servercore is the longest setup step and needs nothing but
+      # the engine: start it now so that setting up Go and compiling the
+      # suite overlap it. The suite's own pull of the image joins this one.
+      # No output redirection: Start-Process then goes through ShellExecute,
+      # so the pull does not inherit this step's output pipes, and the runner
+      # only kills what still holds them when the step ends.
+      - name: Pull the base image in the background
         run: |
-          New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.docker\cli-plugins" | Out-Null
-          curl.exe -fsSL "https://github.com/docker/compose/releases/download/v5.5.1/docker-compose-windows-aarch64.exe" -o "$env:USERPROFILE\.docker\cli-plugins\docker-compose.exe"
-          docker compose version
-      - run: docker compose -f ${{ matrix.compose }} run --rm --build e2e
+          $arch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'amd64' }
+          $match = Select-String -Path tests/e2e/windows/Dockerfile -Pattern '^ARG BASE_IMAGE=(.+)-amd64$'
+          if (-not $match) { throw 'FAIL tests/e2e/windows/Dockerfile has no ARG BASE_IMAGE=...-amd64 line' }
+          $image = $match.Matches[0].Groups[1].Value + "-$arch"
+          Start-Process docker -ArgumentList 'pull', $image
+          Write-Output "Pulling $image in the background"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - run: go test -v -timeout 55m -run TestE2E ./windows
+        working-directory: tests/e2e

docs/arm64.md

@@ -47,7 +47,9 @@ run on Windows ARM64.
 ## Linux ARM64 and Windows ARM64
 
 - **`git` is required.** The checkout runs `git init` / `fetch` / `checkout`
-  against `https://github.com/flutter/flutter.git`.
+  against `https://github.com/flutter/flutter.git`. A fetch that stalls (under
+  1 KB/s for a minute) is abandoned and retried, up to three attempts;
+  `GIT_HTTP_LOW_SPEED_LIMIT` / `GIT_HTTP_LOW_SPEED_TIME` change the threshold.
 - **The first `flutter` run is slow.** It downloads the native Dart SDK and
   builds the Flutter tool, so it needs a network connection.
 - **No checksum is verified.** Git history is the integrity check.

docs/e2e.md

@@ -3,10 +3,10 @@
 The offline Lua hook tests (`tests/hooks_test.lua`) run in the **Test Plugin** workflow.
 The end-to-end suite runs in containers against real vfox on Linux and Windows;
 each `vfox` x `flavor` x `mirror` combination gets its own throwaway
-container. On Linux, CI runs the suite natively on the stock runners (Docker
-plus Go from `actions/setup-go`), the way the repository dev container
-reproduces it locally. On Windows it runs through a Docker Compose runner
-image, since dev containers cannot host Windows container images (see below).
+container. CI runs both suites natively on the stock runners (Docker plus Go
+from `actions/setup-go`). Locally, the repository dev container reproduces the
+Linux run; Windows hosts use a Docker Compose runner image instead, since dev
+containers cannot host Windows container images (see below).
 
 ## Dev container
 
@@ -40,7 +40,10 @@ devcontainer exec --workspace-folder . bash
 host daemon and waits for the suite to dial it back, which a
 Docker-out-of-Docker container cannot do. The suites terminate every
 container explicitly via `t.Cleanup` instead; leftovers from killed runs
-share the `vfox-flutter-e2e-` name prefix for pruning. Rootless Docker users
+share the `vfox-flutter-e2e-` name prefix for pruning. CI sets
+`E2E_ASYNC_CLEANUP=true` so that teardown runs in the background instead of
+holding up each combination, since its runners are discarded anyway; the
+last containers may then outlive the suite. Rootless Docker users
 export `DOCKER_SOCK="$XDG_RUNTIME_DIR/docker.sock"` on the host before
 opening the container. On Windows hosts, keeping the checkout inside WSL 2
 (e.g. under `\\wsl$`) keeps file and bind-mount access fast; a checkout on
@@ -89,6 +92,28 @@ named pipe; the host requirement is Docker in Windows-container mode plus the
 Compose v5 plugin (installed once per machine below). Go never touches the
 host, so it cannot conflict with a vfox-managed toolchain.
 
+CI skips that runner image, which costs minutes per run to build, and runs
+`go test` on the runner with Go from `actions/setup-go`, like the Linux job.
+It also moves the engine's data-root off the runners' slow C: OS disk,
+where image pulls, layer commits and the in-container Flutter installs would
+otherwise crawl: next to the workspace on `windows-latest`'s local D: disk,
+and onto the local NVMe disk `windows-11-arm` ships unformatted. With that,
+`E2E_MAX_JOBS` is 4 there. Both suites start the combinations slowest first
+(the CN mirror, then the official SDK), so the last ones to start are short.
+Unpacking the servercore base image remains the longest setup step, so CI
+starts pulling it in the background before it sets up Go and compiles the
+suite.
+
+The target image (`tests/e2e/windows/Dockerfile`) unpacks PowerShell 7 and
+MinGit from their portable zips instead of running their installers: Git
+for Windows' installer is an x86 program, which arm64 containers only run
+under emulation. MinGit is the same `git.exe` without bash, Perl and the
+credential manager, none of which vfox or Flutter calls.
+
+The `main` combinations share one `vfox.exe`: the suite builds it from vfox's
+main branch with its own Go toolchain while the image builds, and copies it
+into each `main` container before the container starts.
+
 The `windows-11-arm` runner has no Windows Hypervisor Platform feature,
 so Hyper-V isolation is unavailable; process isolation needs only the Containers feature.
 
@@ -165,7 +190,11 @@ docker compose -f tests/e2e/windows/compose.arm64.yaml run --rm --build e2e
 
 On arm64, Go is also what builds the engine, so `install-docker-arm64.ps1`
 runs before the suite; the compose runner image then carries its own pinned
-Go for the tests, and the host Go is not used again. Scope the run down with
+Go for the tests, and the host Go is not used again. The script keeps the
+engine binaries in `C:\vfox-docker\bin` and reuses them on later runs while
+they match its pinned version (CI caches that directory); `-DataRoot` moves
+the engine's images and containers off the default `C:\vfox-docker\data`.
+Scope the run down with
 `$env:VFOX_VERSION = 'latest'; $env:FLAVOR = 'official'` in front of the
 compose command, as on the other platforms.
 

docs/ohos.md

@@ -41,6 +41,9 @@ vfox search flutter
 - **Mirrors don't apply.** `FLUTTER_STORAGE_BASE_URL` and `VFOX_FLUTTER_GITHUB_MIRROR` (see `mirror.md`) only affect
   official SDK downloads; OpenHarmony builds are cloned from gitcode, so both variables
   are ignored.
+- **Stalled clones are retried.** A fetch from gitcode that stalls (under 1 KB/s for a
+  minute) is abandoned and retried, up to three attempts; `GIT_HTTP_LOW_SPEED_LIMIT` /
+  `GIT_HTTP_LOW_SPEED_TIME` change the threshold.
 
 ### Building OpenHarmony applications
 

lib/git.lua

@@ -18,6 +18,11 @@ local VFOX_DIR = ".vfox"
 local TMP_DIR = "tmp"
 local ENGINE_PIN = "bin/internal/engine.version"
 local FETCH_ATTEMPTS = 3
+-- A stalled connection never fails on its own, which would leave the retries
+-- above unused: abort a fetch that moves under 1 KB/s for a minute, so the next
+-- attempt reconnects. GIT_HTTP_LOW_SPEED_LIMIT and GIT_HTTP_LOW_SPEED_TIME
+-- still override both values.
+local FETCH_STALL_GUARD = "-c http.lowSpeedLimit=1000 -c http.lowSpeedTime=60"
 local GITHUB_MIRROR_ENV = "VFOX_FLUTTER_GITHUB_MIRROR"
 local GH_BASE = "https://github.com/"
 
@@ -137,7 +142,7 @@ end
 
 function M.fetch(root, remote, ref)
     for _ = 1, FETCH_ATTEMPTS do
-        if git(root, "fetch -q --depth 1 " .. remote .. " " .. ref) then
+        if git(root, FETCH_STALL_GUARD .. " fetch -q --depth 1 " .. remote .. " " .. ref) then
             return true
         end
     end

tests/e2e/common/common.go

@@ -28,8 +28,10 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 )
@@ -180,6 +182,51 @@ func MatrixForArch(arch string) ([]Combo, error) {
 	return combos, nil
 }
 
+// SlowestFirst returns combos ordered by how long they usually take, the
+// slowest first, so a bounded run (RunCombos) ends on short combos instead
+// of starting a long one last: non-default mirrors (the CN mirror is far
+// from the CI runners) before the default one, official before ohos. The
+// order is otherwise kept.
+func SlowestFirst(combos []Combo) []Combo {
+	slowness := func(c Combo) int {
+		w := 0
+		if c.Mirror != "default" {
+			w += 2
+		}
+		if c.Flavor == "official" {
+			w++
+		}
+		return w
+	}
+	sorted := slices.Clone(combos)
+	slices.SortStableFunc(sorted, func(a, b Combo) int { return slowness(b) - slowness(a) })
+	return sorted
+}
+
+// RunCombos runs run as one subtest per combo, at most jobs at a time,
+// starting them in the given order: unlike t.Parallel behind a semaphore,
+// whose start order is up to the scheduler, so callers can put the slowest
+// combos first.
+func RunCombos(t *testing.T, combos []Combo, jobs int, run func(t *testing.T, c Combo)) {
+	t.Helper()
+	queue := make(chan Combo, len(combos))
+	for _, c := range combos {
+		queue <- c
+	}
+	close(queue)
+	var wg sync.WaitGroup
+	for range min(max(jobs, 1), len(combos)) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for c := range queue {
+				t.Run(c.Slug(), func(t *testing.T) { run(t, c) })
+			}
+		}()
+	}
+	wg.Wait()
+}
+
 // MaxJobs bounds parallel containers; honours $E2E_MAX_JOBS and falls back
 // to def when unset or invalid. Linux passes 3, Windows passes 2 (matching
 // the old e2e.ps1).
@@ -202,6 +249,29 @@ func RetryAttempts() int {
 	return 3
 }
 
+// AsyncCleanup reports whether containers are torn down in the background
+// instead of holding up the test that ran them; honours $E2E_ASYNC_CLEANUP.
+// CI sets it, since the runner is discarded with whatever still runs on
+// it; local runs keep waiting so no container outlives the suite.
+func AsyncCleanup() bool {
+	v, _ := strconv.ParseBool(strings.TrimSpace(os.Getenv("E2E_ASYNC_CLEANUP")))
+	return v
+}
+
+// Cleanup registers terminate to run once t finishes, in the background
+// under AsyncCleanup (the suite may then exit before the last containers
+// are gone).
+func Cleanup(t *testing.T, terminate func()) {
+	t.Helper()
+	t.Cleanup(func() {
+		if AsyncCleanup() {
+			go terminate()
+			return
+		}
+		terminate()
+	})
+}
+
 // ResolveFlutterVersion honours $FLUTTER_VERSION / $OHOS_VERSION, else the
 // defaults above.
 func ResolveFlutterVersion(t *testing.T, flavor string) string {