diff --git a/cmd/kg/main.go b/cmd/kg/main.go index 4472f7f7..7395b181 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -124,6 +124,7 @@ var ( internalCIDRsRaw []string subnet string resyncPeriod time.Duration + checkInPeriod time.Duration iptablesForwardRule bool prioritisePrivateAddr bool @@ -157,6 +158,7 @@ func init() { cmd.Flags().StringSliceVar(&internalCIDRsRaw, "internal-cidr", nil, "CIDRs to consider for internal IP auto-detection. If specified, only IPs within these CIDRs will be used. Can be specified multiple times.") cmd.Flags().StringVar(&subnet, "subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.") cmd.Flags().DurationVar(&resyncPeriod, "resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?") + cmd.Flags().DurationVar(&checkInPeriod, "check-in-period", mesh.DefaultCheckInPeriod, "How often should Kilo nodes refresh their last-seen annotation? A node is considered ready while its last-seen is within twice this value. Increase on large clusters where the default causes Ready-set churn and WireGuard IP collisions.") cmd.Flags().BoolVar(&iptablesForwardRule, "iptables-forward-rules", false, "Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!") cmd.Flags().BoolVar(&prioritisePrivateAddr, "prioritise-private-addresses", false, "Prefer to assign a private IP address to the node's endpoint.") @@ -291,7 +293,7 @@ func runRoot(_ *cobra.Command, _ []string) error { mtu = uint(v) } - m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, autoMTU, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry) + m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, autoMTU, resyncPeriod, checkInPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry) if err != nil { return fmt.Errorf("failed to create Kilo mesh: %v", err) } diff --git a/pkg/mesh/backend.go b/pkg/mesh/backend.go index eebbec2a..1cc749ff 100644 --- a/pkg/mesh/backend.go +++ b/pkg/mesh/backend.go @@ -25,8 +25,10 @@ import ( ) const ( - // checkInPeriod is how often nodes should check-in. - checkInPeriod = 30 * time.Second + // DefaultCheckInPeriod is the default check-in period: nodes refresh their + // last-seen annotation this often, and a node is considered Ready while its + // last-seen is within twice this value. + DefaultCheckInPeriod = 30 * time.Second // DefaultKiloInterface is the default interface created and used by Kilo. DefaultKiloInterface = "kilo0" // DefaultKiloPort is the default UDP port Kilo uses. @@ -35,6 +37,15 @@ const ( DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist" ) +// checkInPeriod is how often nodes should check in. It defaults to +// DefaultCheckInPeriod and may be overridden at startup through New (the +// --check-in-period flag). On large clusters the per-node last-seen writes +// create enough watch churn that liveness cannot propagate within the default +// window, which shrinks every agent's Ready set; a longer period widens the +// freshness window so the Ready set stays close to the full node set. See +// https://github.com/squat/kilo/issues/549. +var checkInPeriod = DefaultCheckInPeriod + // DefaultKiloSubnet is the default CIDR for Kilo. var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}} diff --git a/pkg/mesh/mesh.go b/pkg/mesh/mesh.go index 4cb1661d..e7c8f309 100644 --- a/pkg/mesh/mesh.go +++ b/pkg/mesh/mesh.go @@ -91,7 +91,10 @@ type Mesh struct { } // New returns a new Mesh instance. -func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, autoMTU bool, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { +func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, autoMTU bool, resyncPeriod time.Duration, checkIn time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { + if checkIn > 0 { + checkInPeriod = checkIn + } if err := os.MkdirAll(kiloPath, 0700); err != nil { return nil, fmt.Errorf("failed to create directory to store configuration: %v", err) }