From fddf33c1d1fb1143c40dadb8ed49ab290f3de6f5 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Sun, 13 Sep 2026 08:22:26 +0800 Subject: [PATCH] Converge idle duplicate IPsec SAs after rolling upgrade --- .github/workflows/codeql-verification.yml | 2 +- .github/workflows/release.yml | 34 +++--- .github/workflows/security-release-gate.yml | 14 ++- README.md | 22 +++- backend/ipsec/ipsec.go | 64 ++++++++++- backend/ipsec/ipsec_test.go | 52 ++++++++- backend/ipsec/templates.go | 2 +- scripts/integration-two-node-ipsec | 121 +++++++++++++++----- scripts/integration-two-node-vxlan | 2 +- scripts/validate | 2 +- 10 files changed, 249 insertions(+), 66 deletions(-) diff --git a/.github/workflows/codeql-verification.yml b/.github/workflows/codeql-verification.yml index e53cacf..6efe9e1 100644 --- a/.github/workflows/codeql-verification.yml +++ b/.github/workflows/codeql-verification.yml @@ -53,7 +53,7 @@ jobs: cd "$GITHUB_WORKSPACE" go test -run '^$' ./... go test -run '^$' -tags=integration ./store - VERSION_OVERRIDE=0.14.32 ./scripts/build + VERSION_OVERRIDE=0.14.33 ./scripts/build - name: Analyze without publishing temporary alerts uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63fea89..d4bab60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release IPsec overlay -# A release is dispatched from main only after an annotated v0.14.32 tag and +# A release is dispatched from main only after an annotated v0.14.33 tag and # successful, same-commit Security release gate and CodeQL verification runs. # GHCR, registry attestations, and GitHub Releases are not an atomic transaction. # All source/product/runtime gates precede the image push. A later failure may @@ -9,11 +9,11 @@ on: workflow_dispatch: inputs: release_tag: - description: Existing annotated, numeric v0.14.32 tag at main HEAD + description: Existing annotated, numeric v0.14.33 tag at main HEAD required: true type: choice options: - - v0.14.32 + - v0.14.33 security_run_id: description: Successful same-SHA Security release gate run ID required: true @@ -35,7 +35,7 @@ permissions: attestations: write concurrency: - group: ipsec-vxlan-overlay-release-v0.14.32 + group: ipsec-vxlan-overlay-release-v0.14.33 cancel-in-progress: false jobs: @@ -48,8 +48,8 @@ jobs: CODEQL_RUN_ID: ${{ inputs.codeql_run_id }} RESUME_DIGEST: ${{ inputs.resume_digest }} GH_TOKEN: ${{ github.token }} - SERVICE_IMAGE: ghcr.io/pasturestack/ipsec-vxlan-overlay-network:v0.14.32 - LOCAL_IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.32 + SERVICE_IMAGE: ghcr.io/pasturestack/ipsec-vxlan-overlay-network:v0.14.33 + LOCAL_IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.33 TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 GOFLAGS: -mod=vendor GOWORK: off @@ -67,7 +67,7 @@ jobs: run: | set -euo pipefail test "$GITHUB_REF" = refs/heads/main - test "$RELEASE_TAG" = v0.14.32 + test "$RELEASE_TAG" = v0.14.33 [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] [[ "$SECURITY_RUN_ID" =~ ^[0-9]+$ ]] [[ "$CODEQL_RUN_ID" =~ ^[0-9]+$ ]] @@ -152,15 +152,15 @@ jobs: shell: bash run: | set -euo pipefail - VERSION_OVERRIDE=0.14.32 ./scripts/build + VERSION_OVERRIDE=0.14.33 ./scripts/build sha256sum --check evidence/product.sha256 if [ -z "$RESUME_DIGEST" ]; then - VERSION_OVERRIDE=0.14.32 TAG=0.14.32 REPO=local/pasturestack \ + VERSION_OVERRIDE=0.14.33 TAG=0.14.33 REPO=local/pasturestack \ IMAGE_REVISION="$SOURCE_SHA" ./scripts/package test "$(wc -l < dist/images)" -eq 1 grep -Fx "$LOCAL_IMAGE" dist/images test "$(docker image inspect "$LOCAL_IMAGE" \ - --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.32 + --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.33 test "$(docker image inspect "$LOCAL_IMAGE" \ --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$SOURCE_SHA" docker tag "$LOCAL_IMAGE" "$SERVICE_IMAGE" @@ -178,7 +178,7 @@ jobs: docker pull "$SERVICE_IMAGE@$RESUME_DIGEST" docker tag "$SERVICE_IMAGE@$RESUME_DIGEST" "$SERVICE_IMAGE" test "$(docker image inspect "$SERVICE_IMAGE" \ - --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.32 + --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.33 test "$(docker image inspect "$SERVICE_IMAGE" \ --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$SOURCE_SHA" verification_container="$(docker create "$SERVICE_IMAGE")" @@ -312,7 +312,7 @@ jobs: run: | set -euo pipefail release_dir="dist/release" - product_dir="$release_dir/ipsec-vxlan-overlay-network-0.14.32-linux-amd64" + product_dir="$release_dir/ipsec-vxlan-overlay-network-0.14.33-linux-amd64" mkdir -p "$product_dir" cp bin/ipsec-vxlan-overlay-network bin/ipsec-vxlan-overlay-topology \ LICENSE ORIGIN.md SECURITY.md THIRD_PARTY_NOTICES.md "$product_dir/" @@ -324,11 +324,11 @@ jobs: find "$product_dir" -exec touch -h -d "@$source_epoch" {} + tar --sort=name --mtime="@$source_epoch" --owner=0 --group=0 \ --numeric-owner -C "$release_dir" \ - -cJf "$release_dir/ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz" \ - ipsec-vxlan-overlay-network-0.14.32-linux-amd64 + -cJf "$release_dir/ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz" \ + ipsec-vxlan-overlay-network-0.14.33-linux-amd64 ( cd "$release_dir" - sha256sum ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz \ + sha256sum ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz \ source-sbom.cdx.json image-sbom.cdx.json product.sha256 \ runtime-security.json gate-security-summary.txt image-digest.txt \ release-identity.txt > SHA256SUMS @@ -352,9 +352,9 @@ jobs: run: | set -euo pipefail gh release create "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --verify-tag \ - --title 'IPsec Overlay v0.14.32' \ + --title 'IPsec Overlay v0.14.33' \ --notes "Source ${SOURCE_SHA}; Security gate run ${SECURITY_RUN_ID}; CodeQL run ${CODEQL_RUN_ID}. GHCR image ${SERVICE_IMAGE}@$(cat dist/release/image-digest.txt | sed 's/^.*@//')." \ - dist/release/ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz \ + dist/release/ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz \ dist/release/source-sbom.cdx.json \ dist/release/image-sbom.cdx.json \ dist/release/product.sha256 \ diff --git a/.github/workflows/security-release-gate.yml b/.github/workflows/security-release-gate.yml index 3dd8d57..37991ca 100644 --- a/.github/workflows/security-release-gate.yml +++ b/.github/workflows/security-release-gate.yml @@ -18,12 +18,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 120 env: - CANDIDATE_VERSION: 0.14.32 - VERSION_OVERRIDE: 0.14.32 - TAG: 0.14.32 + CANDIDATE_VERSION: 0.14.33 + VERSION_OVERRIDE: 0.14.33 + TAG: 0.14.33 IMAGE_REVISION: ${{ github.sha }} REPO: local/pasturestack - IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.32 + IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.33 GO_VERSION: 1.27.0 GO_LINUX_AMD64_SHA256: 675c26c449cbb18fc24b74650de1eabbae6e16f64326fd85a283fb3b58280685 TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 @@ -139,7 +139,9 @@ jobs: set -euo pipefail cd "$GITHUB_WORKSPACE" IMAGE="$IMAGE" ./scripts/integration-two-node-vxlan - IMAGE="$IMAGE" ./scripts/integration-two-node-ipsec + IMAGE="$IMAGE" \ + OLD_IMAGE='ghcr.io/pasturestack/ipsec-vxlan-overlay-network@sha256:4d8a51e04bdd27fea3cb2949158103d43e0d2470907c328f76f7a0c6ccec8608' \ + ./scripts/integration-two-node-ipsec - name: Run reachable Go vulnerability analysis shell: bash @@ -271,7 +273,7 @@ jobs: impact_statement:$impact }] | { "@context":"https://openvex.dev/ns/v0.2.0", - "@id":"https://github.com/PastureStack/ipsec-vxlan-overlay-network/security/openvex/dapper/0.14.32", + "@id":"https://github.com/PastureStack/ipsec-vxlan-overlay-network/security/openvex/dapper/0.14.33", author:"PastureStack Security", timestamp:$timestamp, version:1, diff --git a/README.md b/README.md index 14a45e2..0769e17 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ restart path. This does not change firewall ownership or the selected Docker firewall backend. Catalog pinning and live peer-restart verification remain separate gates. -The `v0.14.32` change is scoped to IPsec peer lifecycle. Each managed IKE +The `v0.14.32` change was scoped to IPsec peer lifecycle. Each managed IKE connection requests strongSwan's per-peer `unique=replace` policy, including older custom templates that omit the option; an explicit template policy is preserved. The health reconciler force-removes only a `DELETING` IKE_SA for @@ -57,8 +57,20 @@ which the same managed peer already has an established replacement with an installed CHILD_SA. It never terminates by connection name or changes host firewall rules. The isolated two-node regression test now forces concurrent initiation and a one-sided peer restart, and requires exactly one working SA -on each side. This source change is not proof that a Catalog or live host has -already been upgraded. +on each side within a bounded convergence window while encrypted traffic +continues. This source change is not proof that a Catalog or live host has +already been upgraded. A subsequent live rolling upgrade on two hosts showed +that `unique=replace` and cleanup of `DELETING` SAs alone did not converge: +both peers retained two established SAs, with only one carrying traffic. + +`v0.14.33` keeps the 30-second health reconciler as the owner of missing-SA +recovery and changes the default CHILD `close_action` to `none`, avoiding a +second automatic initiation path after peer close. Explicit custom CHILD +templates remain unchanged. After four health cycles, the reconciler may +terminate only an idle established duplicate for the same managed peer when +exactly one other installed association has carried traffic. Ambiguous or +recent pairs, unrelated peers, and other connections are left untouched. The +live two-host upgrade and traffic gates remain separate from this source fix. The release gate rejects Critical/High findings and secrets in the source, shipped binaries, and runtime image. It scans the disposable Dapper builder @@ -84,8 +96,8 @@ The build is containerized and requires Docker on a Linux AMD64 host: ```sh make test make validate -VERSION_OVERRIDE=0.14.32 make build -TAG=0.14.32 make package +VERSION_OVERRIDE=0.14.33 make build +TAG=0.14.33 make package ``` The package build downloads dependencies anonymously, verifies every standalone binary with SHA-256, pins the Ubuntu base image by digest, resolves every directly installed package from Canonical snapshot `20260808T000000Z` with the exact versions in `ubuntu-apt.lock`, and includes the corresponding strongSwan source archives in the image. Go dependencies are declared in `go.mod`, checksum-bound by `go.sum`, and committed in the standard module-aware `vendor` tree for offline builds. diff --git a/backend/ipsec/ipsec.go b/backend/ipsec/ipsec.go index 71ec200..fa9d2f6 100644 --- a/backend/ipsec/ipsec.go +++ b/backend/ipsec/ipsec.go @@ -30,6 +30,7 @@ const ( pidFile = "/var/run/charon.pid" ipsecHealthCheckInterval = 30 * time.Second + duplicateSAQuietPeriod = 120 * time.Second ipsecInitiateDelay = 5 * time.Second ipsecInitiateAttempts = 3 ipsecInitiateTimeout = "12" @@ -491,13 +492,14 @@ func (o *Overlay) reconcileIpsecHealth() error { o.scheduleInitiatesLocked(missingHosts, 0) o.Unlock() } - return o.reapDeletingDuplicateSAs(expectedHosts) + return o.reapDuplicateSAs(expectedHosts) } -// A peer may disappear while strongSwan is sending DELETE for a replaced SA. -// Once its replacement has an installed CHILD_SA, remove only the old local SA -// by unique ID; never terminate a connection by name or touch unrelated peers. -func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error { +// Reap only unambiguous redundant associations by unique local ID. A peer +// restart can leave a DELETING SA; an upgrade can also leave an ESTABLISHED +// pair where only one CHILD has carried traffic for at least four health +// cycles. Never terminate by connection name or touch another peer's SAs. +func (o *Overlay) reapDuplicateSAs(expectedHosts map[string]bool) error { client, err := getClient() if err != nil { return err @@ -508,7 +510,9 @@ func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error if err != nil { return err } - for _, id := range deletingDuplicateIKEIDs(sas, expectedHosts) { + ids := deletingDuplicateIKEIDs(sas, expectedHosts) + ids = append(ids, idleDuplicateIKEIDs(sas, expectedHosts)...) + for _, id := range ids { response, err := client.Request("terminate", map[string]interface{}{ "ike-id": id, "force": "yes", @@ -525,6 +529,54 @@ func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error return nil } +func idleDuplicateIKEIDs(sas []map[string]goStrongswanVici.IkeSa, expectedHosts map[string]bool) []string { + type candidate struct { + id string + traffic uint64 + } + byPeer := map[string][]candidate{} + for _, saMap := range sas { + for name, sa := range saMap { + if !expectedHosts[sa.Remote_host] || name != "conn-"+sa.Remote_host || sa.State != "ESTABLISHED" || + len(sa.Tasks_active) != 0 || len(sa.Tasks_queued) != 0 || sa.Local_id == "" || sa.Remote_id == "" { + continue + } + age, ageErr := strconv.ParseUint(sa.Established, 10, 64) + id, idErr := strconv.ParseUint(sa.Uniqueid, 10, 32) + if ageErr != nil || age < uint64(duplicateSAQuietPeriod/time.Second) || idErr != nil || id == 0 { + continue + } + var traffic uint64 + installed := false + for childName, child := range sa.Child_sas { + if child.State != "INSTALLED" || canonicalChildName(childName) != "child-"+sa.Remote_host || child.Reqid != reqIdStr { + continue + } + installed = true + traffic += child.GetBytesIn() + child.GetBytesOut() + } + if !installed { + continue + } + key := sa.Remote_host + "\x00" + sa.Local_id + "\x00" + sa.Remote_id + byPeer[key] = append(byPeer[key], candidate{id: sa.Uniqueid, traffic: traffic}) + } + } + var ids []string + for _, group := range byPeer { + // A two-SA pair with exactly one used tunnel is distinguishable from + // IKE reauthentication or a partially established peer. Leave all + // ambiguous groups alone rather than risking the live data path. + if len(group) == 2 && group[0].traffic == 0 && group[1].traffic > 0 { + ids = append(ids, group[0].id) + } else if len(group) == 2 && group[1].traffic == 0 && group[0].traffic > 0 { + ids = append(ids, group[1].id) + } + } + sort.Strings(ids) + return ids +} + func deletingDuplicateIKEIDs(sas []map[string]goStrongswanVici.IkeSa, expectedHosts map[string]bool) []string { healthy := map[string]bool{} for _, saMap := range sas { diff --git a/backend/ipsec/ipsec_test.go b/backend/ipsec/ipsec_test.go index 404a5ff..16e7325 100644 --- a/backend/ipsec/ipsec_test.go +++ b/backend/ipsec/ipsec_test.go @@ -30,9 +30,13 @@ func TestIKEConnectionUniquenessIsSentToVICI(t *testing.T) { if loaded["remote_addrs"].([]interface{})[0] != "192.0.2.20" { t.Fatalf("remote address was lost: %v", loaded["remote_addrs"]) } - if _, ok := loaded["children"].(map[string]interface{})["child-192.0.2.20"]; !ok { + child, ok := loaded["children"].(map[string]interface{})["child-192.0.2.20"] + if !ok { t.Fatal("CHILD_SA was lost from VICI request") } + if got := child.(map[string]interface{})["close_action"]; got != "none" { + t.Fatalf("VICI close_action = %v, want none", got) + } } func TestIKEConnectionCustomTemplateUniqueness(t *testing.T) { @@ -60,6 +64,52 @@ func TestIKEConnectionCustomTemplateUniqueness(t *testing.T) { } } +func TestDefaultChildCloseActionDefersRecoveryToHealthReconciliation(t *testing.T) { + templates := Templates{ConfigDir: t.TempDir()} + if err := templates.Reload(); err != nil { + t.Fatal(err) + } + if got := templates.NewChildSaConf().CloseAction; got != "none" { + t.Fatalf("default close_action = %q, want none", got) + } + configDir := t.TempDir() + if err := os.WriteFile(filepath.Join(configDir, childSaConfName), []byte(`{"close_action":"start"}`), 0600); err != nil { + t.Fatal(err) + } + templates = Templates{ConfigDir: configDir} + if err := templates.Reload(); err != nil { + t.Fatal(err) + } + if got := templates.NewChildSaConf().CloseAction; got != "start" { + t.Fatalf("explicit close_action = %q, want start", got) + } +} + +func TestIdleDuplicateIKEIDsOnlyReapsUnusedLongLivedManagedPeer(t *testing.T) { + makeSA := func(id, host, age, bytes string) goStrongswanVici.IkeSa { + return goStrongswanVici.IkeSa{ + Uniqueid: id, Remote_host: host, Local_id: "192.0.2.10", Remote_id: host, + State: "ESTABLISHED", Established: age, + Child_sas: map[string]goStrongswanVici.Child_sas{ + "child-" + host + "-" + id: {State: "INSTALLED", Reqid: reqIdStr, Bytes_in: bytes}, + }, + } + } + sas := []map[string]goStrongswanVici.IkeSa{ + {"conn-192.0.2.20": makeSA("3", "192.0.2.20", "180", "0")}, + {"conn-192.0.2.20": makeSA("4", "192.0.2.20", "180", "400")}, + {"conn-192.0.2.21": makeSA("5", "192.0.2.21", "180", "0")}, + {"conn-192.0.2.21": makeSA("6", "192.0.2.21", "180", "0")}, + {"conn-192.0.2.22": makeSA("7", "192.0.2.22", "20", "0")}, + {"conn-192.0.2.22": makeSA("8", "192.0.2.22", "20", "50")}, + {"other-192.0.2.20": makeSA("9", "192.0.2.20", "180", "0")}, + } + ids := idleDuplicateIKEIDs(sas, map[string]bool{"192.0.2.20": true, "192.0.2.21": true, "192.0.2.22": true}) + if len(ids) != 1 || ids[0] != "3" { + t.Fatalf("idle duplicate IDs = %v, want [3]", ids) + } +} + func TestDeletingDuplicateIKEIDsOnlyReapsReplacedPeer(t *testing.T) { sas := []map[string]goStrongswanVici.IkeSa{ {"conn-192.0.2.20": { diff --git a/backend/ipsec/templates.go b/backend/ipsec/templates.go index ca7e24f..8824cee 100644 --- a/backend/ipsec/templates.go +++ b/backend/ipsec/templates.go @@ -36,7 +36,7 @@ var ( "remote_ts": ["0.0.0.0/0"], "esp_proposals": ["aes128gcm16-modp2048", "aes-modp2048"], "start_action": "none", - "close_action": "start", + "close_action": "none", "mode": "tunnel", "policies": "no" }`) diff --git a/scripts/integration-two-node-ipsec b/scripts/integration-two-node-ipsec index bb0d8c8..b4b92fb 100755 --- a/scripts/integration-two-node-ipsec +++ b/scripts/integration-two-node-ipsec @@ -1,7 +1,9 @@ #!/bin/bash set -euo pipefail -image=${IMAGE:-local/pasturestack/ipsec-vxlan-overlay-network:0.14.32} +image=${IMAGE:-local/pasturestack/ipsec-vxlan-overlay-network:0.14.33} +old_image=${OLD_IMAGE:-} +startup_image=${old_image:-$image} test_id=$$ network="pasture-overlay-it-${test_id}" node_a="pasture-overlay-a-${test_id}" @@ -23,6 +25,9 @@ dump_logs() { } docker image inspect "$image" >/dev/null +if [ -n "$old_image" ]; then + docker pull "$old_image" >/dev/null +fi docker network create --driver bridge --subnet 172.31.253.0/24 "$network" >/dev/null mkdir -p "$work_dir/node-a/ipsec" "$work_dir/node-b/ipsec" @@ -57,7 +62,7 @@ start_node() { --ip "$underlay" \ --entrypoint /bin/bash \ -v "$config_dir:/test:ro" \ - "$image" \ + "$startup_image" \ -c " set -euo pipefail ip address add ${overlay}/32 dev eth0 @@ -92,6 +97,67 @@ if [ "$ready" != true ]; then exit 1 fi +if [ -n "$old_image" ]; then + # Attempt the live upgrade's duplicate starting state. strongSwan may + # converge these test-only initiations itself; the live gate remains the + # authority for that scenario, while this still tests a rolling handoff. + docker exec "$node_a" swanctl --initiate --ike conn-172.31.253.11 --timeout 20 >"$work_dir/old-initiate-a.log" 2>&1 & + old_extra_a=$! + docker exec "$node_b" swanctl --initiate --ike conn-172.31.253.10 --timeout 20 >"$work_dir/old-initiate-b.log" 2>&1 & + old_extra_b=$! + wait "$old_extra_a" || true + wait "$old_extra_b" || true + old_duplicate=false + for attempt in $(seq 1 30); do + count_a=$(docker exec "$node_a" swanctl --list-sas --raw 2>/dev/null | grep -c '^list-sa event' || true) + count_b=$(docker exec "$node_b" swanctl --list-sas --raw 2>/dev/null | grep -c '^list-sa event' || true) + if [ "$count_a" -ge 2 ] && [ "$count_b" -ge 2 ]; then + old_duplicate=true + break + fi + sleep 1 + done + echo "OLD_DUPLICATE_REPRODUCED=$old_duplicate" + if [ "$old_duplicate" != true ]; then + cat "$work_dir/old-initiate-a.log" "$work_dir/old-initiate-b.log" >&2 + fi + + # Roll one peer at a time, keeping the other peer alive. The candidate + # must restore encrypted traffic before and after the second handoff. + startup_image=$image + docker rm -f "$node_a" >/dev/null + start_node "$node_a" 172.31.253.10 10.42.253.10 10.42.253.11 "$work_dir/node-a" + first_handoff=false + for attempt in $(seq 1 60); do + if docker exec "$node_a" ping -c 1 -W 1 -I 10.42.253.10 10.42.253.11 >/dev/null 2>&1; then + first_handoff=true + break + fi + sleep 1 + done + if [ "$first_handoff" != true ]; then + dump_logs + echo "First rolling upgrade handoff did not restore encrypted traffic" >&2 + exit 1 + fi + docker rm -f "$node_b" >/dev/null + start_node "$node_b" 172.31.253.11 10.42.253.11 10.42.253.10 "$work_dir/node-b" + ready=false + for attempt in $(seq 1 60); do + if docker exec "$node_a" swanctl --list-sas --raw 2>/dev/null | grep -q 'child-172.31.253.11' && \ + docker exec "$node_b" swanctl --list-sas --raw 2>/dev/null | grep -q 'child-172.31.253.10'; then + ready=true + break + fi + sleep 1 + done + if [ "$ready" != true ]; then + dump_logs + echo "Second rolling upgrade handoff did not establish IPsec" >&2 + exit 1 + fi +fi + if ! docker exec "$node_a" ping -c 3 -W 2 -I 10.42.253.10 10.42.253.11 >/dev/null; then dump_logs docker exec "$node_a" ip -s xfrm state >&2 || true @@ -118,9 +184,29 @@ single_sa_on_both_nodes() { [ "$count_a" = 1 ] && [ "$count_b" = 1 ] } -if ! single_sa_on_both_nodes; then +wait_for_single_working_sa() { + local attempt + for attempt in $(seq 1 180); do + # Keep exercising the data path while the 120-second quiet period and + # 30-second health reconciler decide whether an old SA is truly idle. + if docker exec "$node_a" ping -c 1 -W 1 -I 10.42.253.10 10.42.253.11 >/dev/null 2>&1 && + single_sa_on_both_nodes; then + return 0 + fi + sleep 1 + done + return 1 +} + +dump_sas() { + docker exec "$node_a" swanctl --list-sas --raw >&2 || true + docker exec "$node_b" swanctl --list-sas --raw >&2 || true +} + +if ! wait_for_single_working_sa; then dump_logs - echo "Initial connection has duplicate IKE SAs" >&2 + dump_sas + echo "Initial connection did not converge to one working IKE SA" >&2 exit 1 fi @@ -132,17 +218,9 @@ docker exec "$node_b" swanctl --initiate --ike conn-172.31.253.10 --timeout 20 > extra_b=$! wait "$extra_a" || true wait "$extra_b" || true -single_after_race=false -for attempt in $(seq 1 30); do - if single_sa_on_both_nodes && \ - docker exec "$node_a" ping -c 1 -W 1 -I 10.42.253.10 10.42.253.11 >/dev/null 2>&1; then - single_after_race=true - break - fi - sleep 1 -done -if [ "$single_after_race" != true ]; then +if ! wait_for_single_working_sa; then dump_logs + dump_sas echo "Concurrent initiation left duplicate or unusable IKE SAs" >&2 exit 1 fi @@ -150,20 +228,9 @@ fi # Simulate a peer restart without resetting the surviving node. This is where # concurrent initiation previously left two long-lived SAs for one peer. docker restart --time 5 "$node_b" >/dev/null -recovered=false -for attempt in $(seq 1 120); do - if single_sa_on_both_nodes && \ - docker exec "$node_a" ping -c 1 -W 1 -I 10.42.253.10 10.42.253.11 >/dev/null 2>&1; then - recovered=true - break - fi - sleep 1 -done - -if [ "$recovered" != true ]; then +if ! wait_for_single_working_sa; then dump_logs - docker exec "$node_a" swanctl --list-sas >&2 || true - docker exec "$node_b" swanctl --list-sas >&2 || true + dump_sas echo "Peer restart did not converge to one working IKE SA" >&2 exit 1 fi diff --git a/scripts/integration-two-node-vxlan b/scripts/integration-two-node-vxlan index 06d613a..909a07b 100755 --- a/scripts/integration-two-node-vxlan +++ b/scripts/integration-two-node-vxlan @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -image=${IMAGE:-local/pasturestack/ipsec-vxlan-overlay-network:0.14.32} +image=${IMAGE:-local/pasturestack/ipsec-vxlan-overlay-network:0.14.33} test_id=$$ network="pasture-vxlan-it-${test_id}" node_a="pasture-vxlan-a-${test_id}" diff --git a/scripts/validate b/scripts/validate index cb2dd2a..9d35543 100755 --- a/scripts/validate +++ b/scripts/validate @@ -49,7 +49,7 @@ for script in scripts/* package/*.sh package/update-platform-ca; do bash -n "$script" done -version=${VERSION_OVERRIDE:-0.14.32} +version=${VERSION_OVERRIDE:-0.14.33} first=$(mktemp -d) trap 'rm -rf "$first"' EXIT VERSION_OVERRIDE="$version" ./scripts/build