Converge idle duplicate IPsec SAs after rolling upgrades

.github/workflows/codeql-verification.yml

@@ -53,7 +53,7 @@ jobs:
           cd "$GITHUB_WORKSPACE"
           go test -run '^$' ./...
           go test -run '^$' -tags=integration ./store
-          VERSION_OVERRIDE=0.14.32 ./scripts/build
+          VERSION_OVERRIDE=0.14.33 ./scripts/build
 
       - name: Analyze without publishing temporary alerts
         uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8

.github/workflows/release.yml

@@ -1,6 +1,6 @@
 name: Release IPsec overlay
 
-# A release is dispatched from main only after an annotated v0.14.32 tag and
+# A release is dispatched from main only after an annotated v0.14.33 tag and
 # successful, same-commit Security release gate and CodeQL verification runs.
 # GHCR, registry attestations, and GitHub Releases are not an atomic transaction.
 # All source/product/runtime gates precede the image push. A later failure may
@@ -9,11 +9,11 @@ on:
   workflow_dispatch:
     inputs:
       release_tag:
-        description: Existing annotated, numeric v0.14.32 tag at main HEAD
+        description: Existing annotated, numeric v0.14.33 tag at main HEAD
         required: true
         type: choice
         options:
-          - v0.14.32
+          - v0.14.33
       security_run_id:
         description: Successful same-SHA Security release gate run ID
         required: true
@@ -35,7 +35,7 @@ permissions:
   attestations: write
 
 concurrency:
-  group: ipsec-vxlan-overlay-release-v0.14.32
+  group: ipsec-vxlan-overlay-release-v0.14.33
   cancel-in-progress: false
 
 jobs:
@@ -48,8 +48,8 @@ jobs:
       CODEQL_RUN_ID: ${{ inputs.codeql_run_id }}
       RESUME_DIGEST: ${{ inputs.resume_digest }}
       GH_TOKEN: ${{ github.token }}
-      SERVICE_IMAGE: ghcr.io/pasturestack/ipsec-vxlan-overlay-network:v0.14.32
-      LOCAL_IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.32
+      SERVICE_IMAGE: ghcr.io/pasturestack/ipsec-vxlan-overlay-network:v0.14.33
+      LOCAL_IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.33
       TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969
       GOFLAGS: -mod=vendor
       GOWORK: off
@@ -67,7 +67,7 @@ jobs:
         run: |
           set -euo pipefail
           test "$GITHUB_REF" = refs/heads/main
-          test "$RELEASE_TAG" = v0.14.32
+          test "$RELEASE_TAG" = v0.14.33
           [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]
           [[ "$SECURITY_RUN_ID" =~ ^[0-9]+$ ]]
           [[ "$CODEQL_RUN_ID" =~ ^[0-9]+$ ]]
@@ -152,15 +152,15 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          VERSION_OVERRIDE=0.14.32 ./scripts/build
+          VERSION_OVERRIDE=0.14.33 ./scripts/build
           sha256sum --check evidence/product.sha256
           if [ -z "$RESUME_DIGEST" ]; then
-            VERSION_OVERRIDE=0.14.32 TAG=0.14.32 REPO=local/pasturestack \
+            VERSION_OVERRIDE=0.14.33 TAG=0.14.33 REPO=local/pasturestack \
               IMAGE_REVISION="$SOURCE_SHA" ./scripts/package
             test "$(wc -l < dist/images)" -eq 1
             grep -Fx "$LOCAL_IMAGE" dist/images
             test "$(docker image inspect "$LOCAL_IMAGE" \
-              --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.32
+              --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.33
             test "$(docker image inspect "$LOCAL_IMAGE" \
               --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$SOURCE_SHA"
             docker tag "$LOCAL_IMAGE" "$SERVICE_IMAGE"
@@ -178,7 +178,7 @@ jobs:
             docker pull "$SERVICE_IMAGE@$RESUME_DIGEST"
             docker tag "$SERVICE_IMAGE@$RESUME_DIGEST" "$SERVICE_IMAGE"
             test "$(docker image inspect "$SERVICE_IMAGE" \
-              --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.32
+              --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = 0.14.33
             test "$(docker image inspect "$SERVICE_IMAGE" \
               --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$SOURCE_SHA"
             verification_container="$(docker create "$SERVICE_IMAGE")"
@@ -312,7 +312,7 @@ jobs:
         run: |
           set -euo pipefail
           release_dir="dist/release"
-          product_dir="$release_dir/ipsec-vxlan-overlay-network-0.14.32-linux-amd64"
+          product_dir="$release_dir/ipsec-vxlan-overlay-network-0.14.33-linux-amd64"
           mkdir -p "$product_dir"
           cp bin/ipsec-vxlan-overlay-network bin/ipsec-vxlan-overlay-topology \
             LICENSE ORIGIN.md SECURITY.md THIRD_PARTY_NOTICES.md "$product_dir/"
@@ -324,11 +324,11 @@ jobs:
           find "$product_dir" -exec touch -h -d "@$source_epoch" {} +
           tar --sort=name --mtime="@$source_epoch" --owner=0 --group=0 \
             --numeric-owner -C "$release_dir" \
-            -cJf "$release_dir/ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz" \
-            ipsec-vxlan-overlay-network-0.14.32-linux-amd64
+            -cJf "$release_dir/ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz" \
+            ipsec-vxlan-overlay-network-0.14.33-linux-amd64
           (
             cd "$release_dir"
-            sha256sum ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz \
+            sha256sum ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz \
               source-sbom.cdx.json image-sbom.cdx.json product.sha256 \
               runtime-security.json gate-security-summary.txt image-digest.txt \
               release-identity.txt > SHA256SUMS
@@ -352,9 +352,9 @@ jobs:
         run: |
           set -euo pipefail
           gh release create "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --verify-tag \
-            --title 'IPsec Overlay v0.14.32' \
+            --title 'IPsec Overlay v0.14.33' \
             --notes "Source ${SOURCE_SHA}; Security gate run ${SECURITY_RUN_ID}; CodeQL run ${CODEQL_RUN_ID}. GHCR image ${SERVICE_IMAGE}@$(cat dist/release/image-digest.txt | sed 's/^.*@//')." \
-            dist/release/ipsec-vxlan-overlay-network-0.14.32-linux-amd64.tar.xz \
+            dist/release/ipsec-vxlan-overlay-network-0.14.33-linux-amd64.tar.xz \
             dist/release/source-sbom.cdx.json \
             dist/release/image-sbom.cdx.json \
             dist/release/product.sha256 \

.github/workflows/security-release-gate.yml

@@ -18,12 +18,12 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 120
     env:
-      CANDIDATE_VERSION: 0.14.32
-      VERSION_OVERRIDE: 0.14.32
-      TAG: 0.14.32
+      CANDIDATE_VERSION: 0.14.33
+      VERSION_OVERRIDE: 0.14.33
+      TAG: 0.14.33
       IMAGE_REVISION: ${{ github.sha }}
       REPO: local/pasturestack
-      IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.32
+      IMAGE: local/pasturestack/ipsec-vxlan-overlay-network:0.14.33
       GO_VERSION: 1.27.0
       GO_LINUX_AMD64_SHA256: 675c26c449cbb18fc24b74650de1eabbae6e16f64326fd85a283fb3b58280685
       TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969
@@ -139,7 +139,9 @@ jobs:
           set -euo pipefail
           cd "$GITHUB_WORKSPACE"
           IMAGE="$IMAGE" ./scripts/integration-two-node-vxlan
-          IMAGE="$IMAGE" ./scripts/integration-two-node-ipsec
+          IMAGE="$IMAGE" \
+            OLD_IMAGE='ghcr.io/pasturestack/ipsec-vxlan-overlay-network@sha256:4d8a51e04bdd27fea3cb2949158103d43e0d2470907c328f76f7a0c6ccec8608' \
+            ./scripts/integration-two-node-ipsec
 
       - name: Run reachable Go vulnerability analysis
         shell: bash
@@ -271,7 +273,7 @@ jobs:
               impact_statement:$impact
             }] | {
               "@context":"https://openvex.dev/ns/v0.2.0",
-              "@id":"https://github.com/PastureStack/ipsec-vxlan-overlay-network/security/openvex/dapper/0.14.32",
+              "@id":"https://github.com/PastureStack/ipsec-vxlan-overlay-network/security/openvex/dapper/0.14.33",
               author:"PastureStack Security",
               timestamp:$timestamp,
               version:1,

README.md

@@ -49,16 +49,28 @@ restart path. This does not change firewall ownership or the selected Docker
 firewall backend. Catalog pinning and live peer-restart verification remain
 separate gates.
 
-The `v0.14.32` change is scoped to IPsec peer lifecycle. Each managed IKE
+The `v0.14.32` change was scoped to IPsec peer lifecycle. Each managed IKE
 connection requests strongSwan's per-peer `unique=replace` policy, including
 older custom templates that omit the option; an explicit template policy is
 preserved. The health reconciler force-removes only a `DELETING` IKE_SA for
 which the same managed peer already has an established replacement with an
 installed CHILD_SA. It never terminates by connection name or changes host
 firewall rules. The isolated two-node regression test now forces concurrent
 initiation and a one-sided peer restart, and requires exactly one working SA
-on each side. This source change is not proof that a Catalog or live host has
-already been upgraded.
+on each side within a bounded convergence window while encrypted traffic
+continues. This source change is not proof that a Catalog or live host has
+already been upgraded. A subsequent live rolling upgrade on two hosts showed
+that `unique=replace` and cleanup of `DELETING` SAs alone did not converge:
+both peers retained two established SAs, with only one carrying traffic.
+
+`v0.14.33` keeps the 30-second health reconciler as the owner of missing-SA
+recovery and changes the default CHILD `close_action` to `none`, avoiding a
+second automatic initiation path after peer close. Explicit custom CHILD
+templates remain unchanged. After four health cycles, the reconciler may
+terminate only an idle established duplicate for the same managed peer when
+exactly one other installed association has carried traffic. Ambiguous or
+recent pairs, unrelated peers, and other connections are left untouched. The
+live two-host upgrade and traffic gates remain separate from this source fix.
 
 The release gate rejects Critical/High findings and secrets in the source,
 shipped binaries, and runtime image. It scans the disposable Dapper builder
@@ -84,8 +96,8 @@ The build is containerized and requires Docker on a Linux AMD64 host:
 ```sh
 make test
 make validate
-VERSION_OVERRIDE=0.14.32 make build
-TAG=0.14.32 make package
+VERSION_OVERRIDE=0.14.33 make build
+TAG=0.14.33 make package
 ```
 
 The package build downloads dependencies anonymously, verifies every standalone binary with SHA-256, pins the Ubuntu base image by digest, resolves every directly installed package from Canonical snapshot `20260808T000000Z` with the exact versions in `ubuntu-apt.lock`, and includes the corresponding strongSwan source archives in the image. Go dependencies are declared in `go.mod`, checksum-bound by `go.sum`, and committed in the standard module-aware `vendor` tree for offline builds.

backend/ipsec/ipsec.go

@@ -30,6 +30,7 @@ const (
 	pidFile  = "/var/run/charon.pid"
 
 	ipsecHealthCheckInterval = 30 * time.Second
+	duplicateSAQuietPeriod   = 120 * time.Second
 	ipsecInitiateDelay       = 5 * time.Second
 	ipsecInitiateAttempts    = 3
 	ipsecInitiateTimeout     = "12"
@@ -491,13 +492,14 @@ func (o *Overlay) reconcileIpsecHealth() error {
 		o.scheduleInitiatesLocked(missingHosts, 0)
 		o.Unlock()
 	}
-	return o.reapDeletingDuplicateSAs(expectedHosts)
+	return o.reapDuplicateSAs(expectedHosts)
 }
 
-// A peer may disappear while strongSwan is sending DELETE for a replaced SA.
-// Once its replacement has an installed CHILD_SA, remove only the old local SA
-// by unique ID; never terminate a connection by name or touch unrelated peers.
-func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error {
+// Reap only unambiguous redundant associations by unique local ID.  A peer
+// restart can leave a DELETING SA; an upgrade can also leave an ESTABLISHED
+// pair where only one CHILD has carried traffic for at least four health
+// cycles.  Never terminate by connection name or touch another peer's SAs.
+func (o *Overlay) reapDuplicateSAs(expectedHosts map[string]bool) error {
 	client, err := getClient()
 	if err != nil {
 		return err
@@ -508,7 +510,9 @@ func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error
 	if err != nil {
 		return err
 	}
-	for _, id := range deletingDuplicateIKEIDs(sas, expectedHosts) {
+	ids := deletingDuplicateIKEIDs(sas, expectedHosts)
+	ids = append(ids, idleDuplicateIKEIDs(sas, expectedHosts)...)
+	for _, id := range ids {
 		response, err := client.Request("terminate", map[string]interface{}{
 			"ike-id":  id,
 			"force":   "yes",
@@ -525,6 +529,54 @@ func (o *Overlay) reapDeletingDuplicateSAs(expectedHosts map[string]bool) error
 	return nil
 }
 
+func idleDuplicateIKEIDs(sas []map[string]goStrongswanVici.IkeSa, expectedHosts map[string]bool) []string {
+	type candidate struct {
+		id      string
+		traffic uint64
+	}
+	byPeer := map[string][]candidate{}
+	for _, saMap := range sas {
+		for name, sa := range saMap {
+			if !expectedHosts[sa.Remote_host] || name != "conn-"+sa.Remote_host || sa.State != "ESTABLISHED" ||
+				len(sa.Tasks_active) != 0 || len(sa.Tasks_queued) != 0 || sa.Local_id == "" || sa.Remote_id == "" {
+				continue
+			}
+			age, ageErr := strconv.ParseUint(sa.Established, 10, 64)
+			id, idErr := strconv.ParseUint(sa.Uniqueid, 10, 32)
+			if ageErr != nil || age < uint64(duplicateSAQuietPeriod/time.Second) || idErr != nil || id == 0 {
+				continue
+			}
+			var traffic uint64
+			installed := false
+			for childName, child := range sa.Child_sas {
+				if child.State != "INSTALLED" || canonicalChildName(childName) != "child-"+sa.Remote_host || child.Reqid != reqIdStr {
+					continue
+				}
+				installed = true
+				traffic += child.GetBytesIn() + child.GetBytesOut()
+			}
+			if !installed {
+				continue
+			}
+			key := sa.Remote_host + "\x00" + sa.Local_id + "\x00" + sa.Remote_id
+			byPeer[key] = append(byPeer[key], candidate{id: sa.Uniqueid, traffic: traffic})
+		}
+	}
+	var ids []string
+	for _, group := range byPeer {
+		// A two-SA pair with exactly one used tunnel is distinguishable from
+		// IKE reauthentication or a partially established peer.  Leave all
+		// ambiguous groups alone rather than risking the live data path.
+		if len(group) == 2 && group[0].traffic == 0 && group[1].traffic > 0 {
+			ids = append(ids, group[0].id)
+		} else if len(group) == 2 && group[1].traffic == 0 && group[0].traffic > 0 {
+			ids = append(ids, group[1].id)
+		}
+	}
+	sort.Strings(ids)
+	return ids
+}
+
 func deletingDuplicateIKEIDs(sas []map[string]goStrongswanVici.IkeSa, expectedHosts map[string]bool) []string {
 	healthy := map[string]bool{}
 	for _, saMap := range sas {

backend/ipsec/ipsec_test.go

@@ -30,9 +30,13 @@ func TestIKEConnectionUniquenessIsSentToVICI(t *testing.T) {
 	if loaded["remote_addrs"].([]interface{})[0] != "192.0.2.20" {
 		t.Fatalf("remote address was lost: %v", loaded["remote_addrs"])
 	}
-	if _, ok := loaded["children"].(map[string]interface{})["child-192.0.2.20"]; !ok {
+	child, ok := loaded["children"].(map[string]interface{})["child-192.0.2.20"]
+	if !ok {
 		t.Fatal("CHILD_SA was lost from VICI request")
 	}
+	if got := child.(map[string]interface{})["close_action"]; got != "none" {
+		t.Fatalf("VICI close_action = %v, want none", got)
+	}
 }
 
 func TestIKEConnectionCustomTemplateUniqueness(t *testing.T) {
@@ -60,6 +64,52 @@ func TestIKEConnectionCustomTemplateUniqueness(t *testing.T) {
 	}
 }
 
+func TestDefaultChildCloseActionDefersRecoveryToHealthReconciliation(t *testing.T) {
+	templates := Templates{ConfigDir: t.TempDir()}
+	if err := templates.Reload(); err != nil {
+		t.Fatal(err)
+	}
+	if got := templates.NewChildSaConf().CloseAction; got != "none" {
+		t.Fatalf("default close_action = %q, want none", got)
+	}
+	configDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(configDir, childSaConfName), []byte(`{"close_action":"start"}`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	templates = Templates{ConfigDir: configDir}
+	if err := templates.Reload(); err != nil {
+		t.Fatal(err)
+	}
+	if got := templates.NewChildSaConf().CloseAction; got != "start" {
+		t.Fatalf("explicit close_action = %q, want start", got)
+	}
+}
+
+func TestIdleDuplicateIKEIDsOnlyReapsUnusedLongLivedManagedPeer(t *testing.T) {
+	makeSA := func(id, host, age, bytes string) goStrongswanVici.IkeSa {
+		return goStrongswanVici.IkeSa{
+			Uniqueid: id, Remote_host: host, Local_id: "192.0.2.10", Remote_id: host,
+			State: "ESTABLISHED", Established: age,
+			Child_sas: map[string]goStrongswanVici.Child_sas{
+				"child-" + host + "-" + id: {State: "INSTALLED", Reqid: reqIdStr, Bytes_in: bytes},
+			},
+		}
+	}
+	sas := []map[string]goStrongswanVici.IkeSa{
+		{"conn-192.0.2.20": makeSA("3", "192.0.2.20", "180", "0")},
+		{"conn-192.0.2.20": makeSA("4", "192.0.2.20", "180", "400")},
+		{"conn-192.0.2.21": makeSA("5", "192.0.2.21", "180", "0")},
+		{"conn-192.0.2.21": makeSA("6", "192.0.2.21", "180", "0")},
+		{"conn-192.0.2.22": makeSA("7", "192.0.2.22", "20", "0")},
+		{"conn-192.0.2.22": makeSA("8", "192.0.2.22", "20", "50")},
+		{"other-192.0.2.20": makeSA("9", "192.0.2.20", "180", "0")},
+	}
+	ids := idleDuplicateIKEIDs(sas, map[string]bool{"192.0.2.20": true, "192.0.2.21": true, "192.0.2.22": true})
+	if len(ids) != 1 || ids[0] != "3" {
+		t.Fatalf("idle duplicate IDs = %v, want [3]", ids)
+	}
+}
+
 func TestDeletingDuplicateIKEIDsOnlyReapsReplacedPeer(t *testing.T) {
 	sas := []map[string]goStrongswanVici.IkeSa{
 		{"conn-192.0.2.20": {

backend/ipsec/templates.go

@@ -36,7 +36,7 @@ var (
 		"remote_ts": ["0.0.0.0/0"],
 		"esp_proposals":  ["aes128gcm16-modp2048", "aes-modp2048"],
 		"start_action": "none",
-		"close_action": "start",
+		"close_action": "none",
 		"mode": "tunnel",
 		"policies": "no"
 	}`)