From f4629767145b9c4539b018ccaac749f5588dac6d Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 11 Nov 2026 16:16:30 +0300 Subject: [PATCH] feat(kg): make the check-in period configurable A node's WireGuard IP is assigned by its rank in the currently Ready set, and a node counts as Ready only while its last-seen annotation is within twice the check-in period (60s by default). On large clusters the per-node last-seen writes generate enough watch churn that liveness cannot propagate within that window, so every agent's Ready set collapses and most nodes collide on the first address of the subnet. Add a --check-in-period flag so operators can widen the freshness window on large clusters. The default is unchanged at 30s, so existing deployments behave exactly as before. Signed-off-by: IvanHunters --- cmd/kg/main.go | 4 +++- pkg/mesh/backend.go | 15 +++++++++++++-- pkg/mesh/mesh.go | 5 ++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/kg/main.go b/cmd/kg/main.go index 4472f7f7..7395b181 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -124,6 +124,7 @@ var ( internalCIDRsRaw []string subnet string resyncPeriod time.Duration + checkInPeriod time.Duration iptablesForwardRule bool prioritisePrivateAddr bool @@ -157,6 +158,7 @@ func init() { cmd.Flags().StringSliceVar(&internalCIDRsRaw, "internal-cidr", nil, "CIDRs to consider for internal IP auto-detection. If specified, only IPs within these CIDRs will be used. Can be specified multiple times.") cmd.Flags().StringVar(&subnet, "subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.") cmd.Flags().DurationVar(&resyncPeriod, "resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?") + cmd.Flags().DurationVar(&checkInPeriod, "check-in-period", mesh.DefaultCheckInPeriod, "How often should Kilo nodes refresh their last-seen annotation? A node is considered ready while its last-seen is within twice this value. Increase on large clusters where the default causes Ready-set churn and WireGuard IP collisions.") cmd.Flags().BoolVar(&iptablesForwardRule, "iptables-forward-rules", false, "Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!") cmd.Flags().BoolVar(&prioritisePrivateAddr, "prioritise-private-addresses", false, "Prefer to assign a private IP address to the node's endpoint.") @@ -291,7 +293,7 @@ func runRoot(_ *cobra.Command, _ []string) error { mtu = uint(v) } - m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, autoMTU, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry) + m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, autoMTU, resyncPeriod, checkInPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry) if err != nil { return fmt.Errorf("failed to create Kilo mesh: %v", err) } diff --git a/pkg/mesh/backend.go b/pkg/mesh/backend.go index eebbec2a..1cc749ff 100644 --- a/pkg/mesh/backend.go +++ b/pkg/mesh/backend.go @@ -25,8 +25,10 @@ import ( ) const ( - // checkInPeriod is how often nodes should check-in. - checkInPeriod = 30 * time.Second + // DefaultCheckInPeriod is the default check-in period: nodes refresh their + // last-seen annotation this often, and a node is considered Ready while its + // last-seen is within twice this value. + DefaultCheckInPeriod = 30 * time.Second // DefaultKiloInterface is the default interface created and used by Kilo. DefaultKiloInterface = "kilo0" // DefaultKiloPort is the default UDP port Kilo uses. @@ -35,6 +37,15 @@ const ( DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist" ) +// checkInPeriod is how often nodes should check in. It defaults to +// DefaultCheckInPeriod and may be overridden at startup through New (the +// --check-in-period flag). On large clusters the per-node last-seen writes +// create enough watch churn that liveness cannot propagate within the default +// window, which shrinks every agent's Ready set; a longer period widens the +// freshness window so the Ready set stays close to the full node set. See +// https://github.com/squat/kilo/issues/549. +var checkInPeriod = DefaultCheckInPeriod + // DefaultKiloSubnet is the default CIDR for Kilo. var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}} diff --git a/pkg/mesh/mesh.go b/pkg/mesh/mesh.go index 4cb1661d..e7c8f309 100644 --- a/pkg/mesh/mesh.go +++ b/pkg/mesh/mesh.go @@ -91,7 +91,10 @@ type Mesh struct { } // New returns a new Mesh instance. -func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, autoMTU bool, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { +func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, autoMTU bool, resyncPeriod time.Duration, checkIn time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { + if checkIn > 0 { + checkInPeriod = checkIn + } if err := os.MkdirAll(kiloPath, 0700); err != nil { return nil, fmt.Errorf("failed to create directory to store configuration: %v", err) }