fix(net): health-check unmanaged-Site gateways through the tunnel

cmd/unbounded-net-node/node_types.go

@@ -192,7 +192,7 @@ type gatewayPeerInfo struct {
 	LearnedRoutes          map[string]unboundednetv1alpha1.GatewayNodeRoute
 	PodCIDRs               []string // Gateway node's podCIDRs for specific routing
 	SkipPodCIDRRoutes      bool     // Keep tunnel reachability host routes but skip explicit PodCIDR routes
-	UnmanagedCNISite       bool     // Gateway's Site has manageCniPlugin=false, so it never gets a podCIDR
+	UnmanagedCNISite       bool     // Gateway's Site has manageCniPlugin=false, so it may have no podCIDR
 	TunnelProtocol         string   // Resolved tunnel protocol for this peer (WireGuard or GENEVE)
 	TunnelMTU              int      // Effective MTU after route, encapsulation, and CRD limits
 }

cmd/unbounded-net-node/peer_healthcheck.go

@@ -111,7 +111,7 @@ func registerPeersWithHealthCheck(
 
 	// Gateway peers.
 	for _, gwPeer := range gatewayPeers {
-		overlayIP := getHealthIPFromPodCIDRs(gwPeer.PodCIDRs)
+		overlayIP, viaIf := gatewayHealthTarget(gwPeer, state.nodePodCIDRs)
 		if overlayIP == "" {
 			continue
 		}
@@ -145,10 +145,10 @@ func registerPeersWithHealthCheck(
 		state.gatewayPeerHealthCheckEnabled[ifName] = true
 		state.mu.Unlock()
 
-		if err := state.healthCheckManager.AddPeer(gwPeer.Name, net.ParseIP(overlayIP), settings); err != nil {
+		if err := state.healthCheckManager.AddPeerVia(gwPeer.Name, net.ParseIP(overlayIP), viaIf, settings); err != nil {
 			registrationErrors = append(registrationErrors, fmt.Errorf("register gateway peer %s at %s: %w", gwPeer.Name, overlayIP, err))
 		} else {
-			klog.V(4).Infof("Healthcheck: registered gateway peer %s at %s (iface %s)", gwPeer.Name, overlayIP, ifName)
+			klog.V(4).Infof("Healthcheck: registered gateway peer %s at %s (iface %s, via %q)", gwPeer.Name, overlayIP, ifName, viaIf)
 		}
 	}
 

cmd/unbounded-net-node/peer_healthcheck_test.go

@@ -6,6 +6,7 @@ package main
 import (
 	"context"
 	"errors"
+	"net/netip"
 	"testing"
 	"time"
 
@@ -246,3 +247,119 @@ func TestReconciliationPassesFreshHealthProfilesBeforeStateCommit(t *testing.T)
 		t.Fatalf("registration used stale state: %+v %v", got, err)
 	}
 }
+
+func TestGatewayHealthTarget(t *testing.T) {
+	routed := []string{"10.32.130.0/23"}
+	v4Overlay := []string{"192.168.20.0/25"}
+
+	cases := map[string]struct {
+		peer       gatewayPeerInfo
+		local      []string
+		wantIP     string
+		wantViaDev string
+	}{
+		"overlay address from pod CIDR": {
+			peer:   gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, PodCIDRs: []string{"10.244.1.0/24"}},
+			wantIP: "10.244.1.1",
+		},
+		"unmanaged gateway probed at its IPv4 address through the tunnel": {
+			peer:       gatewayPeerInfo{InternalIPs: []string{"fd00::4", "10.0.0.4"}, RoutedCidrs: routed, UnmanagedCNISite: true},
+			local:      v4Overlay,
+			wantIP:     "10.0.0.4",
+			wantViaDev: unbounded0DeviceName,
+		},
+		"unmanaged IPv6-only gateway is not probed": {
+			peer:  gatewayPeerInfo{InternalIPs: []string{"fd00::4"}, RoutedCidrs: routed, UnmanagedCNISite: true},
+			local: v4Overlay,
+		},
+		"no IPv4 overlay address on this node to probe from": {
+			peer:  gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, RoutedCidrs: routed, UnmanagedCNISite: true},
+			local: []string{"fd10::/64"},
+		},
+		"managed gateway still waiting for its pod CIDR": {
+			peer:  gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, RoutedCidrs: routed},
+			local: v4Overlay,
+		},
+		"unmanaged gateway over WireGuard": {
+			peer:  gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, RoutedCidrs: routed, UnmanagedCNISite: true, TunnelProtocol: "WireGuard"},
+			local: v4Overlay,
+		},
+		"unmanaged gateway without routed CIDRs": {
+			peer:  gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, UnmanagedCNISite: true},
+			local: v4Overlay,
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			ip, via := gatewayHealthTarget(tc.peer, tc.local)
+			if ip != tc.wantIP || via != tc.wantViaDev {
+				t.Fatalf("gatewayHealthTarget() = (%q, %q), want (%q, %q)", ip, via, tc.wantIP, tc.wantViaDev)
+			}
+		})
+	}
+}
+
+// TestGatewayProbePins checks that the probe target gets exactly one BPF
+// entry: pinned when localCidrs exclude it, and not added a second time when
+// gatewayBPFPrefixes already has it (two nexthops would be health-filtered).
+func TestGatewayProbePins(t *testing.T) {
+	peer := gatewayPeerInfo{InternalIPs: []string{"10.0.0.4"}, RoutedCidrs: []string{"10.32.130.0/23"}, UnmanagedCNISite: true}
+	local := []string{"192.168.20.0/25"}
+
+	excluded := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/27")}
+	if got := gatewayProbePins(peer, excluded, local); len(got) != 1 || got[0] != "10.0.0.4/32" {
+		t.Fatalf("pins with the gateway IP in localCidrs = %v, want [10.0.0.4/32]", got)
+	}
+
+	if got := gatewayProbePins(peer, nil, local); len(got) != 0 {
+		t.Fatalf("pins without localCidrs = %v, want none (gatewayBPFPrefixes has the /32)", got)
+	}
+
+	managed := peer
+	managed.UnmanagedCNISite = false
+
+	if got := gatewayProbePins(managed, excluded, local); len(got) != 0 {
+		t.Fatalf("pins for a managed gateway = %v, want none", got)
+	}
+}
+
+// TestRegisterUnmanagedGatewayHealthCheck covers a gateway in a Site with
+// manageCniPlugin=false: it has no pod CIDR, and used to be skipped, so its
+// BPF entries were never health-checked or failed over.
+func TestRegisterUnmanagedGatewayHealthCheck(t *testing.T) {
+	manager, err := healthcheck.NewManager("local", 0, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer manager.Stop()
+
+	state := &wireGuardState{
+		healthCheckManager:            manager,
+		meshPeerHealthCheckEnabled:    make(map[string]bool),
+		gatewayPeerHealthCheckEnabled: make(map[string]bool),
+		nodePodCIDRs:                  []string{"192.168.20.0/25"},
+	}
+	gateways := []gatewayPeerInfo{{
+		Name:             "gw",
+		PoolName:         "pool",
+		InternalIPs:      []string{"10.0.0.4"},
+		RoutedCidrs:      []string{"10.32.130.0/23"},
+		UnmanagedCNISite: true,
+	}}
+
+	desired, err := registerPeersWithHealthCheck(nil, gateways, "remote", false, map[string]string{"remote": "site"}, nil, nil, nil, nil,
+		map[string]healthcheck.HealthCheckSettings{"site": healthcheck.DefaultSettings()}, state,
+		func(gatewayPeerInfo) string { return "geneve0" }, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !desired["gw"] {
+		t.Fatalf("unmanaged gateway was not registered for health checks: %v", desired)
+	}
+
+	if _, err := manager.GetPeerSettings("gw"); err != nil {
+		t.Fatalf("no health check session for the unmanaged gateway: %v", err)
+	}
+}

cmd/unbounded-net-node/tunnel_config.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/vishvananda/netlink"
@@ -499,6 +501,13 @@ func configureTunnelPeers(
 		// LAN. The host CIDR is a longest-prefix match so it overrides the
 		// supernet ECMP without disturbing pod-CIDR traffic.
 		addPeerBPFEntries(bpfEntries, gatewayBPFPrefixes(gwPeer, state.siteRouting.localCIDRs), underlayIP, uint32(cfg.GeneveVNI), peerIfIdx, peerProto, gwPeer.Name)
+
+		// A gateway without pod CIDRs is health-checked at its own IP through
+		// the tunnel (see gatewayHealthTarget). Pin that IP in the BPF map when
+		// localCidrs keep it out: only probes sent out of unbounded0 explicitly
+		// use this entry, so no kernel route is added and the GENEVE outer
+		// packet still takes the underlay.
+		addPeerBPFEntries(bpfEntries, gatewayProbePins(gwPeer, state.siteRouting.localCIDRs, state.nodePodCIDRs), underlayIP, uint32(cfg.GeneveVNI), peerIfIdx, peerProto, gwPeer.Name)
 	}
 
 	// Store entries for deferred reconcile (after VXLAN and WG entries are added).
@@ -540,8 +549,9 @@ func configureTunnelPeers(
 // gatewayPeerReadyForTunnel reports whether a gateway peer can be programmed
 // into the tunnel BPF map. A gateway in a managed Site without a podCIDR is
 // still waiting for CNI setup (it publishes its WireGuard key first), so it
-// stays out of the map. A gateway in a Site with manageCniPlugin=false never
-// gets a podCIDR, so its routed CIDRs are what make it usable.
+// stays out of the map. A gateway in a Site with manageCniPlugin=false gets
+// no podCIDR unless one is assigned explicitly, so its routed CIDRs are what
+// make it usable.
 func gatewayPeerReadyForTunnel(gwPeer gatewayPeerInfo) bool {
 	if len(gwPeer.InternalIPs) == 0 {
 		return false
@@ -554,6 +564,63 @@ func gatewayPeerReadyForTunnel(gwPeer gatewayPeerInfo) bool {
 	return gwPeer.UnmanagedCNISite && len(gwPeer.RoutedCidrs) > 0
 }
 
+// gatewayHealthTarget returns the address to health-check a shared-tunnel
+// gateway peer at, and the interface the probes must leave through ("" for
+// normal routing). Gateways with pod CIDRs answer on their overlay address.
+// A gateway in a Site with manageCniPlugin=false may have none; it is then probed at
+// its internal IPv4 address out of unbounded0: the eBPF program sends the
+// probe through the tunnel, where firewalls between Sites already allow the
+// encapsulation, instead of over the underlay on the health-check port.
+//
+// That needs an IPv4 overlay address on this node (localPodCIDRs) as the probe
+// source, so the reply also comes back through the tunnel. IPv6 has no on-link
+// fallback without a route through unbounded0, so IPv6-only gateways are not
+// probed this way.
+func gatewayHealthTarget(gwPeer gatewayPeerInfo, localPodCIDRs []string) (string, string) {
+	if ip := getHealthIPFromPodCIDRs(gwPeer.PodCIDRs); ip != "" {
+		return ip, ""
+	}
+
+	if gwPeer.TunnelProtocol == "WireGuard" || !gatewayPeerReadyForTunnel(gwPeer) || !hasIPv4CIDR(localPodCIDRs) {
+		return "", ""
+	}
+
+	for _, raw := range gwPeer.InternalIPs {
+		if ip := net.ParseIP(raw); ip != nil && ip.To4() != nil {
+			return ip.String(), unbounded0DeviceName
+		}
+	}
+
+	return "", ""
+}
+
+// gatewayProbePins returns the host CIDR to pin in the BPF map for a gateway
+// probed through the tunnel, unless gatewayBPFPrefixes already includes it
+// (a second nexthop on the same key would make the entry health-filtered).
+func gatewayProbePins(gwPeer gatewayPeerInfo, localCIDRs []netip.Prefix, localPodCIDRs []string) []string {
+	ip, viaIf := gatewayHealthTarget(gwPeer, localPodCIDRs)
+	if viaIf == "" {
+		return nil
+	}
+
+	pin := ipsToHostCIDRs([]string{ip})
+	if slices.Contains(gatewayBPFPrefixes(gwPeer, localCIDRs), pin[0]) {
+		return nil
+	}
+
+	return pin
+}
+
+func hasIPv4CIDR(cidrs []string) bool {
+	for _, cidr := range cidrs {
+		if prefix, err := netip.ParsePrefix(cidr); err == nil && prefix.Addr().Is4() {
+			return true
+		}
+	}
+
+	return false
+}
+
 // addPeerBPFEntries adds LPM trie entries for a set of CIDRs pointing to a
 // tunnel endpoint. Healthy=true is set on every new nexthop so it is
 // forwarded immediately; the healthcheck callback later clears Healthy if

internal/net/healthcheck/manager.go

@@ -118,6 +118,13 @@ func (m *Manager) Stop() {
 // AddPeer registers a new peer for health checking. If the manager is
 // already running, the session starts immediately.
 func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings HealthCheckSettings) error {
+	return m.AddPeerVia(peerHostname, overlayIP, "", settings)
+}
+
+// AddPeerVia registers a peer whose probes leave through the named interface
+// regardless of the routing table, for example a tunnel device whose eBPF
+// program forwards them to the peer. An empty viaIf routes probes normally.
+func (m *Manager) AddPeerVia(peerHostname string, overlayIP net.IP, viaIf string, settings HealthCheckSettings) error {
 	if err := settings.validate(); err != nil {
 		return err
 	}
@@ -127,7 +134,7 @@ func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings Health
 
 	m.mu.Lock()
 	if existing, exists := m.sessions[peerHostname]; exists {
-		if existing.overlayIP.Equal(overlayIP) {
+		if existing.overlayIP.Equal(overlayIP) && existing.viaIf == viaIf {
 			existing.updateSettings(settings)
 			m.mu.Unlock()
 
@@ -146,6 +153,7 @@ func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings Health
 	s := newSession(sessionConfig{
 		peerHostname:  peerHostname,
 		overlayIP:     overlayIP,
+		viaIf:         viaIf,
 		port:          m.port,
 		localHostname: m.localHostname,
 		settings:      settings,
@@ -161,7 +169,12 @@ func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings Health
 	}
 
 	updatePeerGauges(m.sessions)
-	klog.Infof("healthcheck: added peer %s (%s)", peerHostname, overlayIP)
+
+	if viaIf != "" {
+		klog.Infof("healthcheck: added peer %s (%s via %s)", peerHostname, overlayIP, viaIf)
+	} else {
+		klog.Infof("healthcheck: added peer %s (%s)", peerHostname, overlayIP)
+	}
 
 	return nil
 }

internal/net/healthcheck/sendvia_linux.go

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package healthcheck
+
+import (
+	"fmt"
+	"net"
+
+	"golang.org/x/sys/unix"
+)
+
+// writeVia sends data to addr out of the interface with index ifIndex. The
+// interface is set per packet with IP_PKTINFO/IPV6_PKTINFO, so the kernel uses
+// it even when the routing table would pick another one, while replies still
+// arrive on the shared socket.
+func writeVia(conn net.PacketConn, data []byte, addr *net.UDPAddr, ifIndex int) error {
+	udp, ok := conn.(*net.UDPConn)
+	if !ok {
+		return fmt.Errorf("connection is %T, not *net.UDPConn", conn)
+	}
+
+	var oob []byte
+	if addr.IP.To4() != nil {
+		oob = unix.PktInfo4(&unix.Inet4Pktinfo{Ifindex: int32(ifIndex)}) //nolint:gosec // kernel interface indexes fit in int32
+	} else {
+		oob = unix.PktInfo6(&unix.Inet6Pktinfo{Ifindex: uint32(ifIndex)}) //nolint:gosec // kernel interface indexes are positive
+	}
+
+	_, _, err := udp.WriteMsgUDP(data, oob, addr)
+
+	return err
+}