diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index f3325394eb..1532d61deb 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -107,7 +107,10 @@ jobs: Test_36_MultiContainerPerContainerBinding, Test_43_RelativeOpenPathResolution, Test_48_MultiSubtypeGroupedProfileDocument, - Test_49_EphemeralContainerFullTreatment + Test_49_EphemeralContainerFullTreatment, + Test_50_ServiceRefNetworkNeighbor, + Test_51_ServiceRefIngressR0012, + Test_53_DefaultLearnedNetworkFalsePositives ] steps: - name: Checkout code diff --git a/cmd/main.go b/cmd/main.go index e41b256727..bfec59e977 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,11 +39,13 @@ import ( "github.com/kubescape/node-agent/pkg/malwaremanager" malwaremanagerv1 "github.com/kubescape/node-agent/pkg/malwaremanager/v1" otelmetrics "github.com/kubescape/node-agent/pkg/metricsmanager/otel" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/networkstream" networkstreamv1 "github.com/kubescape/node-agent/pkg/networkstream/v1" "github.com/kubescape/node-agent/pkg/nodeprofilemanager" nodeprofilemanagerv1 "github.com/kubescape/node-agent/pkg/nodeprofilemanager/v1" "github.com/kubescape/node-agent/pkg/objectcache" + "github.com/kubescape/node-agent/pkg/objectcache/containerprofilecache" "github.com/kubescape/node-agent/pkg/objectcache/dnscache" "github.com/kubescape/node-agent/pkg/objectcache/k8scache" @@ -72,6 +74,8 @@ import ( "github.com/kubescape/node-agent/pkg/watcher/seccompprofilewatcher" goruntime "go.opentelemetry.io/contrib/instrumentation/runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + toolscache "k8s.io/client-go/tools/cache" ) func main() { @@ -321,6 +325,60 @@ func main() { ruleBindingCache.AddNotifier(&ruleBindingNotify) cpc := containerprofilecache.NewContainerProfileCache(cfg, storageClient, k8sObjectCache, metricsProvider) + // Resolve serviceRef/serviceSelector/entity network neighbors against live + // cluster state (Service ClusterIPs + endpoints, Node IPs + CNI gateway) at + // projection time. Gated behind networkServiceResolutionEnabled: the + // cluster-wide Service+EndpointSlice list+watch (one per DaemonSet node) is + // only paid where profiles actually use the feature. Services and + // EndpointSlices are watched cluster-wide (a profile may reference any + // namespace's Service); the Node informer is field-selected to this agent's + // own node — the "host" entity is local, and a cluster-wide Node watch on + // every DaemonSet pod is O(nodes^2) traffic for no benefit. A TransformFunc + // strips managedFields/annotations (and per-endpoint fields beyond + // Addresses) before objects enter the cache to keep its footprint small. + if cfg.EnableNetworkServiceResolution { + svcInformers := informers.NewSharedInformerFactory(k8sClient.GetKubernetesClient(), 0) + nodeInformers := informers.NewSharedInformerFactoryWithOptions( + k8sClient.GetKubernetesClient(), 0, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = "metadata.name=" + cfg.NodeName + }), + ) + svcInformer := svcInformers.Core().V1().Services().Informer() + sliceInformer := svcInformers.Discovery().V1().EndpointSlices().Informer() + _ = svcInformer.SetTransform(networkpeer.TrimService) + _ = sliceInformer.SetTransform(networkpeer.TrimEndpointSlice) + serviceLister := networkpeer.NewInformerLister( + svcInformers.Core().V1().Services().Lister(), + svcInformers.Discovery().V1().EndpointSlices().Lister(), + nodeInformers.Core().V1().Nodes().Lister(), + cfg.NodeName, + ) + // Advance the lister generation on any Service/EndpointSlice/Node + // change, so the reconciler re-projects serviceRef/entity profiles when + // the cluster view moves (endpoint churn, or caches that fill after + // startup). Per-event cost is a single atomic increment; the + // re-projection itself is coalesced onto the reconcile tick, and only + // serviceRef-using profiles are eligible. + bump := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(interface{}) { serviceLister.Bump() }, + UpdateFunc: func(_, _ interface{}) { serviceLister.Bump() }, + DeleteFunc: func(interface{}) { serviceLister.Bump() }, + } + _, _ = svcInformer.AddEventHandler(bump) + _, _ = sliceInformer.AddEventHandler(bump) + _, _ = nodeInformers.Core().V1().Nodes().Informer().AddEventHandler(bump) + // Start the informers and hand the lister over WITHOUT blocking on + // cache sync: node-agent's core startup (container watcher, profiling) + // must not wait on these, and a bounded wait here previously delayed + // learning enough to trip the tight completion budget of Test_22. The + // caches fill in the background; serviceRef/entity neighbors resolve on + // the next reconcile once populated. nil-until-set is a no-op in + // projection. + svcInformers.Start(ctx.Done()) + nodeInformers.Start(ctx.Done()) + cpc.SetServiceLister(serviceLister) + } cpc.Start(ctx) if cpm, ok := containerProfileManager.(*containerprofilemanagerv1.ContainerProfileManager); ok { cpm.SetCompletionNotifier(cpc) diff --git a/go.mod b/go.mod index 808f11127a..5bef7bf8f7 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/kubescape/backend v0.0.39 github.com/kubescape/go-logger v0.0.32 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.303 + github.com/kubescape/storage v0.0.320 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/oleiade/lane/v2 v2.0.0 @@ -472,7 +472,7 @@ require ( zombiezen.com/go/sqlite v1.4.0 // indirect ) -replace github.com/inspektor-gadget/inspektor-gadget => github.com/matthyx/inspektor-gadget v0.0.0-20260819074828-9494a925bd43 +replace github.com/inspektor-gadget/inspektor-gadget => github.com/kubescape/inspektor-gadget v0.0.0-20260826074832-06b0d12baca0 replace github.com/anchore/syft => github.com/kubescape/syft v1.32.0-ks.2 diff --git a/go.sum b/go.sum index b239690414..07f8e6a26f 100644 --- a/go.sum +++ b/go.sum @@ -891,10 +891,12 @@ github.com/kubescape/backend v0.0.39 h1:B1QRfKCSFlzuE+jWOnk/l7EpH71/Q3n14KKq0QSn github.com/kubescape/backend v0.0.39/go.mod h1:cMEGP8cXUZgY89YU4GRBGIla9HZW7grZsUtlCwvZgAE= github.com/kubescape/go-logger v0.0.32 h1:4mI+XJOV8VFCMewrEE9VIFEIOhzXokYT3nFpNfXf4fM= github.com/kubescape/go-logger v0.0.32/go.mod h1:Alj7JBQ8/WCxbXe8Ura6ZheSRK45E0p21M3xeqedX90= +github.com/kubescape/inspektor-gadget v0.0.0-20260826074832-06b0d12baca0 h1:kJzq1CnGP4kqAdyVWrb9TGlRpniwsIsVVMfo61OuzCQ= +github.com/kubescape/inspektor-gadget v0.0.0-20260826074832-06b0d12baca0/go.mod h1:cwCFczq1LJ6Frpur0Vr5Ncic77a9ihki07Xpwzy+ItI= github.com/kubescape/k8s-interface v0.0.214 h1:j7KP0/5VvYOoQdBGV2+gRM3qnR8PWLAGF8RM/k/DmJ0= github.com/kubescape/k8s-interface v0.0.214/go.mod h1:WNYUG93aZ5kDmuaRKFLtVhp18Yc6EfaHdD1gLYtVTN4= -github.com/kubescape/storage v0.0.303 h1:0nXI6E07lbWsg7iEH04vR4kwiekj//uCQl/La+8j4aM= -github.com/kubescape/storage v0.0.303/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= +github.com/kubescape/storage v0.0.320 h1:uKqc6SL9phBMbjiN20uTK1SVOINHuNrklWaxb2U09Ag= +github.com/kubescape/storage v0.0.320/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kubescape/syft v1.32.0-ks.2 h1:xdUksUmKEyyVKsTfJDYW8Z5HawVJtelsUolPOsWtDx0= github.com/kubescape/syft v1.32.0-ks.2/go.mod h1:E6Kd4iBM2ljUOUQvSt7hVK6vBwaHkMXwcvBZmGMSY5o= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= @@ -919,8 +921,6 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/matthyx/inspektor-gadget v0.0.0-20260819074828-9494a925bd43 h1:RNn7KJswYAsnFW+gZRIBNRrcQ3BSsyW5dpXR28aRkDE= -github.com/matthyx/inspektor-gadget v0.0.0-20260819074828-9494a925bd43/go.mod h1:cwCFczq1LJ6Frpur0Vr5Ncic77a9ihki07Xpwzy+ItI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= diff --git a/pkg/config/config.go b/pkg/config/config.go index cec9f41ab6..2653228ec4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,6 +83,8 @@ type Config struct { EnableMalwareDetection bool `mapstructure:"malwareDetectionEnabled"` EnableNetworkStreaming bool `mapstructure:"networkStreamingEnabled"` EnableNetworkTracing bool `mapstructure:"networkServiceEnabled"` + EnableNetworkServiceResolution bool `mapstructure:"networkServiceResolutionEnabled"` + AlertOnHostPeers bool `mapstructure:"alertOnHostPeers"` EnableNodeProfile bool `mapstructure:"nodeProfileServiceEnabled"` EnablePartialProfileGeneration bool `mapstructure:"partialProfileGenerationEnabled"` EnableMetricsExporter bool `mapstructure:"prometheusExporterEnabled"` @@ -185,6 +187,8 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("podName", os.Getenv(PodNameEnvVar)) viper.SetDefault("fimEnabled", false) viper.SetDefault("networkStreamingEnabled", false) + viper.SetDefault("networkServiceResolutionEnabled", true) + viper.SetDefault("alertOnHostPeers", false) viper.SetDefault("kubernetesMode", true) viper.SetDefault("networkStreamingInterval", 2*time.Minute) viper.SetDefault("workerPoolSize", 3000) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 754b342279..7624e5e2ce 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -36,6 +36,7 @@ func TestLoadConfig(t *testing.T) { EnableHttpDetection: false, EnableFIM: true, EnableNetworkStreaming: false, + EnableNetworkServiceResolution: true, EnableEmbeddedSboms: false, EnableHostSensor: true, HostSensorInterval: 1 * time.Minute, diff --git a/pkg/containerprofilemanager/v1/container_data.go b/pkg/containerprofilemanager/v1/container_data.go index 9ddb1ed555..0c7ed93467 100644 --- a/pkg/containerprofilemanager/v1/container_data.go +++ b/pkg/containerprofilemanager/v1/container_data.go @@ -222,7 +222,6 @@ func (cd *containerData) createNetworkNeighbor(networkEvent NetworkEvent, namesp } } else if networkEvent.Destination.Kind == EndpointKindService { - // For service, we need to retrieve it and use its selector svc, err := k8sClient.GetWorkload(networkEvent.Destination.Namespace, "Service", networkEvent.Destination.Name) // TODO: use IG inventory as this can generate a lot of API calls. if err != nil { logger.L().Warning("failed to get service", @@ -231,19 +230,16 @@ func (cd *containerData) createNetworkNeighbor(networkEvent NetworkEvent, namesp return nil } + // The ClusterIP is stable, so record it: detection matches it directly when the event carries the pre-DNAT address, covering selectorless services and the learn/detect label asymmetry. + neighborEntry.IPAddress = networkEvent.Destination.IPAddress + var selector map[string]string if svc.GetName() == "kubernetes" && svc.GetNamespace() == "default" { - // The default service has no selectors, in addition, we want to save the default service address selector = svc.GetLabels() - neighborEntry.IPAddress = networkEvent.Destination.IPAddress } else { selector = svc.GetServiceSelector() } - - if len(selector) == 0 { - // TODO: check if we need to handle services with no selectors - return nil - } else { + if len(selector) > 0 { neighborEntry.PodSelector = &metav1.LabelSelector{ MatchLabels: selector, } @@ -255,10 +251,6 @@ func (cd *containerData) createNetworkNeighbor(networkEvent NetworkEvent, namesp } } else { - if networkEvent.Destination.IPAddress == "127.0.0.1" { - // No need to generate for localhost - return nil - } neighborEntry.IPAddress = networkEvent.Destination.IPAddress if dnsResolverClient != nil { diff --git a/pkg/containerprofilemanager/v1/container_data_service_test.go b/pkg/containerprofilemanager/v1/container_data_service_test.go new file mode 100644 index 0000000000..228db42100 --- /dev/null +++ b/pkg/containerprofilemanager/v1/container_data_service_test.go @@ -0,0 +1,98 @@ +package containerprofilemanager + +import ( + "testing" + + "github.com/kubescape/k8s-interface/k8sinterface" + "github.com/kubescape/k8s-interface/workloadinterface" + "github.com/kubescape/node-agent/pkg/objectcache" + "github.com/kubescape/node-agent/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +type fakeServiceClient struct { + selector map[string]interface{} + labels map[string]interface{} +} + +func (f *fakeServiceClient) GetWorkload(namespace, _, name string) (k8sinterface.IWorkload, error) { + meta := map[string]interface{}{"name": name, "namespace": namespace} + if f.labels != nil { + meta["labels"] = f.labels + } + spec := map[string]interface{}{} + if f.selector != nil { + spec["selector"] = f.selector + } + return workloadinterface.NewWorkloadObj(map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": meta, + "spec": spec, + }), nil +} + +func (f *fakeServiceClient) CalculateWorkloadParentRecursive(w k8sinterface.IWorkload) (string, string, error) { + return w.GetKind(), w.GetName(), nil +} +func (f *fakeServiceClient) GetKubernetesClient() kubernetes.Interface { return nil } +func (f *fakeServiceClient) GetDynamicClient() dynamic.Interface { return nil } + +// A service neighbor records its stable ClusterIP so detection matches the peer +// by address (the learned selector alone misses on CNIs that keep the pre-DNAT +// ClusterIP on the wire, since IG stamps the service's metadata labels while the +// learned selector holds the pod selector), and a selectorless service is kept. +func TestCreateNetworkNeighbor_ServiceRecordsClusterIP(t *testing.T) { + const clusterIP = "10.43.12.34" + cd := &containerData{watchedContainerData: &objectcache.WatchedContainerData{Namespace: "default"}} + ev := NetworkEvent{ + Port: 80, + Protocol: "tcp", + PktType: utils.OutgoingPktType, + Destination: Destination{ + Kind: EndpointKindService, + Namespace: "default", + Name: "nginx", + IPAddress: clusterIP, + }, + } + + withSel := &fakeServiceClient{selector: map[string]interface{}{"app": "nginx"}} + n := cd.createNetworkNeighbor(ev, "default", withSel, nil) + require.NotNil(t, n) + assert.Equal(t, clusterIP, n.IPAddress, "a service neighbor must record its stable ClusterIP") + require.NotNil(t, n.PodSelector, "a service with a selector keeps its pod selector") + assert.Equal(t, "nginx", n.PodSelector.MatchLabels["app"]) + + noSel := &fakeServiceClient{} + n2 := cd.createNetworkNeighbor(ev, "default", noSel, nil) + require.NotNil(t, n2, "a selectorless service must not be dropped — the ClusterIP identifies it") + assert.Equal(t, clusterIP, n2.IPAddress) + assert.Nil(t, n2.PodSelector, "no selector to learn when the service defines none") +} + +// With the 127.0.0.1 learn-drop removed, loopback and its aliases are learned +// as address neighbors (subject to R0011/R0012) instead of being silently +// dropped — loopback is a real attack surface (localhost admin panels, sidecar +// pivots). Only the literal 127.0.0.1 changed behavior; the others were never +// dropped, and this pins that they all remain learnable. +func TestCreateNetworkNeighbor_LoopbackAliasesLearned(t *testing.T) { + cd := &containerData{watchedContainerData: &objectcache.WatchedContainerData{Namespace: "default"}} + for _, ip := range []string{"127.0.0.1", "127.0.0.53", "::1", "0.0.0.0"} { + ev := NetworkEvent{ + Port: 8080, + Protocol: "tcp", + PktType: utils.OutgoingPktType, + Destination: Destination{ + Kind: EndpointKindRaw, + IPAddress: ip, + }, + } + n := cd.createNetworkNeighbor(ev, "default", nil, nil) + require.NotNil(t, n, "loopback/localhost %s must be learned (guard removed), not dropped", ip) + assert.Equal(t, ip, n.IPAddress, "loopback %s recorded as an address neighbor", ip) + } +} diff --git a/pkg/networkpeer/exclude_namespaces_test.go b/pkg/networkpeer/exclude_namespaces_test.go new file mode 100644 index 0000000000..5ee8af1c46 --- /dev/null +++ b/pkg/networkpeer/exclude_namespaces_test.go @@ -0,0 +1,151 @@ +package networkpeer + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/config" +) + +// excludeNamespaces (Config.SkipNamespace) filters which WORKLOADS node-agent +// profiles; the selector resolver queries a cluster-wide Service/Node view that +// is unaware of it. These pin the resulting asymmetry so it is deliberate. +func excludedTopology() *fakeLister { + l := realFluxTopology() + l.services["kube-system/kube-dns"] = &ServiceInfo{ + Namespace: "kube-system", Name: "kube-dns", + Labels: map[string]string{"k8s-app": "kube-dns", "probe": "yes", "__ns__": "kube-system"}, + ClusterIPs: []string{"10.43.0.10"}, + EndpointIPs: []string{"10.42.0.5"}, + } + l.services["honey/storage"].Labels["probe"] = "yes" + return l +} + +func TestExcludeNamespaces_ServiceRefIntoExcludedNsStillResolves(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + if !cfg.SkipNamespace("kube-system") { + t.Fatal("precondition: kube-system must be an excluded namespace") + } + l := excludedTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l) + for _, ip := range []string{"10.43.0.10", "10.42.0.5"} { + if !Matches(tuples, ip, 53, "TCP") { + t.Errorf("serviceRef naming excluded ns %s:53 must still resolve — excludeNamespaces does not gate peer allowlisting", ip) + } + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}}, l); len(got) != 1 || got[0] != "kube-dns.kube-system.svc.cluster.local" { + t.Errorf("the excluded-ns Service FQDN is implied too: got %v", got) + } +} + +func TestExcludeNamespaces_ServiceSelectorFansIntoExcludedNs(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + l := excludedTopology() + + all := Resolve(PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, l) + if !Matches(all, "10.43.0.10", 53, "TCP") { + t.Error("a namespace-less serviceSelector fans into the excluded namespace (kube-dns) — exclusion does not scope fanout") + } + if !Matches(all, "10.43.70.156", 53, "TCP") { + t.Error("the same selector still resolves the monitored-ns Service (honey/storage)") + } + if !cfg.SkipNamespace("kube-system") { + t.Fatal("kube-system is excluded, yet its Service was just allowlisted above — the asymmetry under test") + } + + scoped := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"probe": "yes"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, + Ports: tcp(53), + }, l) + if Matches(scoped, "10.43.0.10", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must exclude the kube-system Service") + } + if !Matches(scoped, "10.43.70.156", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must still resolve honey/storage") + } +} + +// The only namespace-scoping the selectors offer is authored NamespaceLabels; +// the chart's excludeNamespaces is orthogonal to it. This grids the two axes so +// a reader sees excludeNamespaces never appears as an input to resolution. +func TestExcludeNamespaces_TruthTable(t *testing.T) { + l := excludedTopology() + const ( + excludedIP = "10.43.0.10" // kube-system/kube-dns ClusterIP + monitorIP = "10.43.70.156" // honey/storage ClusterIP + ) + cases := []struct { + name string + spec PeerSpec + wantExcludedPeer bool + wantMonitoredPeer bool + why string + }{ + { + "serviceRef-excluded-ns", + PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, + true, false, "explicit serviceRef into an excluded ns resolves", + }, + { + "serviceRef-monitored-ns", + PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}, Ports: tcp(53)}, + false, true, "serviceRef into a monitored ns resolves", + }, + { + "selector-no-nsLabels", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, + true, true, "unscoped selector fans across the exclusion boundary", + }, + { + "selector-nsLabels-honey", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, Ports: tcp(53)}, + false, true, "NamespaceLabels is the ONLY thing that scopes fanout", + }, + { + "selector-nsLabels-kube-system", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}, Ports: tcp(53)}, + true, false, "NamespaceLabels can deliberately TARGET an excluded ns", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tuples := Resolve(c.spec, l) + if got := Matches(tuples, excludedIP, 53, "TCP"); got != c.wantExcludedPeer { + t.Errorf("excluded-ns peer match=%v want %v — %s", got, c.wantExcludedPeer, c.why) + } + if got := Matches(tuples, monitorIP, 53, "TCP"); got != c.wantMonitoredPeer { + t.Errorf("monitored-ns peer match=%v want %v — %s", got, c.wantMonitoredPeer, c.why) + } + }) + } +} + +// The source/peer asymmetry, stated as one assertion pair under both the +// exclude-denylist and the include-allowlist forms of SkipNamespace. +func TestExcludeNamespaces_SourceSuppressedButPeerAllowlisted(t *testing.T) { + l := excludedTopology() + peerResolves := func() bool { + return Matches(Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l), "10.43.0.10", 53, "TCP") + } + for _, cfg := range []*config.Config{ + {ExcludeNamespaces: []string{"kube-system"}}, + {IncludeNamespaces: []string{"honey"}}, // allowlist form: kube-system is implicitly skipped + } { + if !cfg.SkipNamespace("kube-system") { + t.Fatal("a kube-system SOURCE workload must be skipped from profiling") + } + if !peerResolves() { + t.Error("yet a kube-system PEER remains resolvable/allowlistable — the resolver takes no namespace-exclusion input") + } + } +} + +// The host entity has no namespace, so excludeNamespaces cannot touch it. +func TestExcludeNamespaces_HostEntityOrthogonal(t *testing.T) { + l := excludedTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + if !Matches(tuples, "192.168.0.191", 10250, "TCP") { + t.Error("host entity resolves regardless of any excludeNamespaces setting (it names no namespace)") + } +} diff --git a/pkg/networkpeer/expand.go b/pkg/networkpeer/expand.go new file mode 100644 index 0000000000..f8958f8221 --- /dev/null +++ b/pkg/networkpeer/expand.go @@ -0,0 +1,171 @@ +package networkpeer + +import ( + "strings" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +// ExpandServiceNeighbors resolves every serviceRef / serviceSelector / entity +// neighbor in the list against the cluster view and returns equivalent +// synthesized ipAddresses neighbors (one per source neighbor, carrying the +// resolved IPs and the source neighbor's own ports). +// +// The synthesized neighbors are ordinary selector-free ipAddresses entries +// (plus, for Service-backed specs, the Service's cluster FQDN as a dnsName so a +// client dialling it by name is allowlisted too), so the existing +// port-sensitive address matcher and DNS matcher handle them with no further +// change — a serviceRef/host neighbor becomes exactly the narrow, resolved +// entry it stands for. Neighbors that resolve to nothing (unknown +// Service, selector matching nothing, unknown entity) contribute nothing — +// never a match-all. Callers append the result to the same direction (egress +// or ingress) before projecting the profile. +func ExpandServiceNeighbors(neighbors []v1beta1.NetworkNeighbor, profileNs string, l Lister) []v1beta1.NetworkNeighbor { + if l == nil { + return nil + } + var out []v1beta1.NetworkNeighbor + for i := range neighbors { + n := &neighbors[i] + spec, ok := specFromNeighbor(n, profileNs) + if !ok { + continue + } + ips := ResolveIPs(spec, l) + dnsNames := ResolveDNSNames(spec, l) + if len(ips) == 0 && len(dnsNames) == 0 { + continue + } + out = append(out, v1beta1.NetworkNeighbor{ + Identifier: n.Identifier + "-resolved", + Type: n.Type, + IPAddresses: ips, + DNSNames: dnsNames, + Ports: n.Ports, + }) + } + return out +} + +// WithResolvedServiceNeighbors returns cp with every serviceRef/serviceSelector/ +// entity neighbor expanded into equivalent selector-free ipAddresses neighbors +// (appended to the same direction), so the projection's existing address +// surface enforces them. It is a no-op — returning cp unchanged — when the +// lister is nil or nothing resolves, and it never mutates the input: a copy is +// made only when there is something to add. Call it immediately before +// projecting a ContainerProfile. +func WithResolvedServiceNeighbors(cp *v1beta1.ContainerProfile, l Lister) *v1beta1.ContainerProfile { + if cp == nil || l == nil { + return cp + } + egExtra := ExpandServiceNeighbors(cp.Spec.Egress, cp.Namespace, l) + inExtra := ExpandServiceNeighbors(cp.Spec.Ingress, cp.Namespace, l) + if len(egExtra) == 0 && len(inExtra) == 0 { + return cp + } + out := cp.DeepCopy() + out.Spec.Egress = append(out.Spec.Egress, egExtra...) + out.Spec.Ingress = append(out.Spec.Ingress, inExtra...) + return out +} + +// WithHostPeer appends a synthetic entity:host neighbor to both directions so the node's own IPs (kubelet probes, hostNetwork peers, masqueraded traffic) resolve into the address surface and never alert; node infrastructure is ambient, volatile and has no pod identity to learn, so it is allowlisted by default. +func WithHostPeer(cp *v1beta1.ContainerProfile) *v1beta1.ContainerProfile { + if cp == nil { + return cp + } + out := cp.DeepCopy() + host := v1beta1.NetworkNeighbor{Identifier: "host-entity", Entity: EntityHost} + out.Spec.Egress = append(out.Spec.Egress, host) + out.Spec.Ingress = append(out.Spec.Ingress, host) + return out +} + +// HasServiceNeighbors reports whether any egress/ingress neighbor declares a +// serviceRef / serviceSelector / entity — i.e. whether this profile's +// projection depends on the live cluster view (Service/EndpointSlice/Node) and +// must be re-projected when that view changes. Keyed on the raw fields, not on +// whether they currently resolve, so a profile projected before the informers +// synced is still marked and re-projects once they do. +func HasServiceNeighbors(cp *v1beta1.ContainerProfile) bool { + if cp == nil { + return false + } + for i := range cp.Spec.Egress { + if hasServiceFields(&cp.Spec.Egress[i]) { + return true + } + } + for i := range cp.Spec.Ingress { + if hasServiceFields(&cp.Spec.Ingress[i]) { + return true + } + } + return false +} + +// Must mirror specFromNeighbor's gate: ServiceRefNamespace alone is not a serviceRef. +func hasServiceFields(n *v1beta1.NetworkNeighbor) bool { + return n.ServiceRefName != "" || n.ServiceSelector != nil || n.Entity != "" +} + +// specFromNeighbor extracts a PeerSpec from a NetworkNeighbor, reporting false +// if the neighbor declares none of the service/entity selectors (a plain +// ipAddresses / dnsNames / podSelector neighbor is left untouched). +func specFromNeighbor(n *v1beta1.NetworkNeighbor, profileNs string) (PeerSpec, bool) { + // Cheap-reject a plain ipAddresses/dnsNames neighbor before allocating a + // []PortProto it would only discard (hot on every projection's non-service + // neighbors). + if n.Entity == "" && n.ServiceRefName == "" && n.ServiceSelector == nil { + return PeerSpec{}, false + } + spec := PeerSpec{Ports: portsFromNeighbor(n.Ports)} + switch { + case n.Entity != "": + spec.Entity = n.Entity + case n.ServiceRefName != "": + spec.ServiceRef = &ServiceRef{Namespace: n.ServiceRefNamespace, Name: n.ServiceRefName} + case n.ServiceSelector != nil: + // Only equality (matchLabels) is honored. A MatchExpressions clause or + // an empty matchLabels would either be silently ignored (broadening the + // match) or resolve to every Service — fail closed instead. + if len(n.ServiceSelector.MatchExpressions) > 0 || len(n.ServiceSelector.MatchLabels) == 0 { + return PeerSpec{}, false + } + spec.ServiceSelector = n.ServiceSelector.MatchLabels + // namespaceSelector, identical tiers to the podSelector path + // (namespaceSelectorMatches): OMITTED (nil) => the profile's own + // namespace; explicit EMPTY {} => cluster-wide (opt-in); explicit + // metadata.name= => that namespace. The lister scopes only on the + // metadata.name equality, so any richer explicit form fails closed. + switch nsl := n.NamespaceSelector; { + case nsl == nil: + spec.NamespaceLabels = map[string]string{"kubernetes.io/metadata.name": profileNs} + case len(nsl.MatchExpressions) == 0 && len(nsl.MatchLabels) == 0: + // explicit {} => cluster-wide: leave NamespaceLabels nil + case len(nsl.MatchExpressions) == 0 && len(nsl.MatchLabels) == 1 && nsl.MatchLabels["kubernetes.io/metadata.name"] != "": + spec.NamespaceLabels = nsl.MatchLabels + default: + return PeerSpec{}, false + } + default: + return PeerSpec{}, false + } + return spec, true +} + +func portsFromNeighbor(ports []v1beta1.NetworkPort) []PortProto { + if len(ports) == 0 { + return nil + } + out := make([]PortProto, 0, len(ports)) + for i := range ports { + p := &ports[i] + var port int32 + if p.Port != nil { + port = *p.Port + } + out = append(out, PortProto{Port: port, Protocol: strings.ToUpper(string(p.Protocol))}) + } + return out +} diff --git a/pkg/networkpeer/expand_test.go b/pkg/networkpeer/expand_test.go new file mode 100644 index 0000000000..f663565884 --- /dev/null +++ b/pkg/networkpeer/expand_test.go @@ -0,0 +1,314 @@ +package networkpeer + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +func port(name string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Name: name, Protocol: "TCP", Port: &p} +} + +// TestExpandServiceNeighbors_Egress: a serviceRef (alertmanager) + a host +// entity neighbor expand into selector-free ipAddresses neighbors carrying the +// resolved IPs and the original ports; a plain ipAddresses neighbor and an +// unresolvable serviceRef contribute nothing. +func TestExpandServiceNeighbors_Egress(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "am", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + {Identifier: "plain", Type: "internal", IPAddresses: []string{"10.43.0.0/16"}, Ports: []v1beta1.NetworkPort{port("TCP-443", 443)}}, + {Identifier: "ghost", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "missing", Ports: []v1beta1.NetworkPort{port("TCP-1", 1)}}, + } + out := ExpandServiceNeighbors(in, "", l) + + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor (alertmanager only), got %d", len(out)) + } + got := out[0] + if got.Identifier != "am-resolved" { + t.Errorf("identifier: got %q", got.Identifier) + } + // ClusterIP + both endpoints, port carried over. + wantIPs := map[string]bool{"10.43.54.190": false, "10.42.0.44": false, "10.42.0.84": false} + for _, ip := range got.IPAddresses { + if _, ok := wantIPs[ip]; !ok { + t.Errorf("unexpected resolved IP %s", ip) + } + wantIPs[ip] = true + } + for ip, seen := range wantIPs { + if !seen { + t.Errorf("missing resolved IP %s", ip) + } + } + if len(got.Ports) != 1 || got.Ports[0].Port == nil || *got.Ports[0].Port != 9093 { + t.Errorf("ports not carried over: %+v", got.Ports) + } + // serviceRef implies the Service cluster FQDN as a dnsName (R0005 suppression). + if len(got.DNSNames) != 1 || got.DNSNames[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN not emitted: %v", got.DNSNames) + } +} + +// TestExpandServiceNeighbors_HostEntity: fromEntity host resolves to node + +// gateway IPs on the health port. +func TestExpandServiceNeighbors_HostEntity(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Type: "internal", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := ExpandServiceNeighbors(in, "", l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized host neighbor, got %d", len(out)) + } + // host entity is not a Service: no FQDN. + if len(out[0].DNSNames) != 0 { + t.Errorf("host entity must not emit a dnsName: %v", out[0].DNSNames) + } + // Feed the synthesized entry through the address matcher the same way the + // projection would, to prove end-to-end intent. + tuples := Resolve(PeerSpec{Entity: "host", Ports: []PortProto{{Port: 9440, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("gateway kubelet probe should match") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("wrong port must not match") + } +} + +// TestExpandServiceNeighbors_Selector: serviceSelector expands across all +// matching Services in the scoped namespace. +func TestExpandServiceNeighbors_Selector(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{{ + Identifier: "guestbooks", + Type: "internal", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + }} + out := ExpandServiceNeighbors(in, "", l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor, got %d", len(out)) + } + if len(out[0].IPAddresses) != 2 { + t.Errorf("expected 2 guestbook ClusterIPs, got %v", out[0].IPAddresses) + } +} + +func TestExpandServiceNeighbors_NoPortsMeansAnyPort(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "st", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "storage"}, + } + out := ExpandServiceNeighbors(in, "", l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor, got %d", len(out)) + } + if len(out[0].Ports) != 0 { + t.Errorf("a portless source neighbor must synthesize a portless (any-port) entry, got %+v", out[0].Ports) + } + if len(out[0].IPAddresses) != 1 || out[0].IPAddresses[0] != "10.43.70.156" { + t.Errorf("storage ClusterIP expected, got %v", out[0].IPAddresses) + } +} + +func TestExpandServiceNeighbors_SelectorFQDNFanout(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{{ + Identifier: "guestbooks", + Type: "internal", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + }} + out := ExpandServiceNeighbors(in, "", l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor, got %d", len(out)) + } + want := map[string]bool{"guestbook-ui.gitops-demo.svc.cluster.local": false, "helm-guestbook.gitops-demo.svc.cluster.local": false} + for _, d := range out[0].DNSNames { + if _, ok := want[d]; !ok { + t.Errorf("unexpected FQDN %s", d) + } + want[d] = true + } + for d, seen := range want { + if !seen { + t.Errorf("selector fanout must imply FQDN %s, got %v", d, out[0].DNSNames) + } + } +} + +// TestExpandServiceNeighbors_NilLister: no cluster view, no expansion. +func TestExpandServiceNeighbors_NilLister(t *testing.T) { + in := []v1beta1.NetworkNeighbor{{Identifier: "am", Entity: "host"}} + if out := ExpandServiceNeighbors(in, "", nil); out != nil { + t.Errorf("nil lister must expand to nil, got %v", out) + } +} + +// TestExpandServiceNeighbors_SelectorFailClosed: a serviceSelector carrying +// MatchExpressions (unsupported) or an empty matchLabels must NOT broaden the +// allowlist — it resolves to nothing. +func TestExpandServiceNeighbors_SelectorFailClosed(t *testing.T) { + l := realFluxTopology() + cases := []v1beta1.NetworkNeighbor{ + {Identifier: "expr", ServiceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "guestbook"}, + MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "tier", Operator: metav1.LabelSelectorOpExists}}, + }, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + {Identifier: "empty", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + } + for _, n := range cases { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{n}, "", l); len(out) != 0 { + t.Errorf("%s: selector must fail closed, got %d", n.Identifier, len(out)) + } + } +} + +// TestWithResolvedServiceNeighbors: the CP-level wrapper appends resolved +// neighbors without mutating the input, and is a no-op when nothing resolves. +func TestWithResolvedServiceNeighbors(t *testing.T) { + l := realFluxTopology() + cp := &v1beta1.ContainerProfile{} + cp.Spec.Egress = []v1beta1.NetworkNeighbor{ + {Identifier: "am", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + } + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := WithResolvedServiceNeighbors(cp, l) + + if len(cp.Spec.Egress) != 1 || len(cp.Spec.Ingress) != 1 { + t.Fatalf("input CP must not be mutated: eg=%d in=%d", len(cp.Spec.Egress), len(cp.Spec.Ingress)) + } + if len(out.Spec.Egress) != 2 { + t.Errorf("egress: want original + 1 resolved, got %d", len(out.Spec.Egress)) + } + if len(out.Spec.Ingress) != 2 { + t.Errorf("ingress: want original + 1 resolved, got %d", len(out.Spec.Ingress)) + } + + // No-op cases. + if got := WithResolvedServiceNeighbors(cp, nil); got != cp { + t.Error("nil lister must return the same CP unchanged") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if got := WithResolvedServiceNeighbors(plain, l); got != plain { + t.Error("a CP with no service/entity neighbors must return unchanged (same pointer)") + } +} + +// TestExpandServiceNeighbors_NamespaceSelectorFailClosed: a namespaceSelector +// is honored only as the single equality kubernetes.io/metadata.name=; any +// other form must fail closed rather than silently broaden cluster-wide. +func TestExpandServiceNeighbors_NamespaceSelectorFailClosed(t *testing.T) { + l := realFluxTopology() + withNS := func(nsSel *metav1.LabelSelector) v1beta1.NetworkNeighbor { + return v1beta1.NetworkNeighbor{ + Identifier: "svc", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: nsSel, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + } + } + bad := []*metav1.LabelSelector{ + {MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpExists}}}, + {MatchLabels: map[string]string{"env": "prod"}}, // wrong key + {MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo", "x": "y"}}, // extra key + } + for i, ns := range bad { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{withNS(ns)}, "", l); len(out) != 0 { + t.Errorf("bad namespaceSelector[%d] must fail closed, got %d", i, len(out)) + } + } + good := withNS(&metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}) + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{good}, "", l); len(out) != 1 { + t.Errorf("metadata.name namespaceSelector should resolve, got %d", len(out)) + } + // An explicit EMPTY {} namespaceSelector is cluster-wide (opt-in), NOT fail-closed. + clusterWide := withNS(&metav1.LabelSelector{}) + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{clusterWide}, "gitops-demo", l); len(out) != 1 { + t.Errorf("empty {} namespaceSelector must resolve cluster-wide, got %d", len(out)) + } +} + +// TestHasServiceNeighbors: a profile with a serviceRef/serviceSelector/entity +// neighbor is flagged as depending on the live cluster view; a plain +// ipAddresses/dnsNames profile is not. +func TestHasServiceNeighbors(t *testing.T) { + if HasServiceNeighbors(nil) { + t.Error("nil CP must be false") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if HasServiceNeighbors(plain) { + t.Error("plain ipAddresses profile must not use service resolution") + } + for _, n := range []v1beta1.NetworkNeighbor{ + {ServiceRefName: "alertmanager", ServiceRefNamespace: "honey"}, + {ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "x"}}}, + {Entity: "host"}, + } { + cp := &v1beta1.ContainerProfile{} + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{n} + if !HasServiceNeighbors(cp) { + t.Errorf("neighbor %+v should be flagged", n) + } + } + nsOnly := &v1beta1.ContainerProfile{} + nsOnly.Spec.Ingress = []v1beta1.NetworkNeighbor{{ServiceRefNamespace: "honey"}} + if HasServiceNeighbors(nsOnly) { + t.Error("ServiceRefNamespace without ServiceRefName must not be flagged (specFromNeighbor rejects it)") + } +} + +// WithHostPeer injects entity:host into both directions; after resolution the +// node's InternalIP and CNI gateway land in the address surface, so node-IP +// traffic (kubelet probes, hostNetwork peers) is covered by the address matcher +// and never alerts — the default host-peer allowlist (alertOnHostPeers=false). +func TestWithHostPeer_ResolvesNodeIPsIntoAddressSurface(t *testing.T) { + l := realFluxTopology() // hostIPs: node 192.168.0.191, gateway 10.42.0.1 + cp := &v1beta1.ContainerProfile{} + + withHost := WithResolvedServiceNeighbors(WithHostPeer(cp), l) + + collect := func(ns []v1beta1.NetworkNeighbor) map[string]bool { + s := map[string]bool{} + for i := range ns { + for _, ip := range ns[i].IPAddresses { + s[ip] = true + } + } + return s + } + for _, dir := range []struct { + name string + ns []v1beta1.NetworkNeighbor + }{{"egress", withHost.Spec.Egress}, {"ingress", withHost.Spec.Ingress}} { + got := collect(dir.ns) + for _, ip := range []string{"192.168.0.191", "10.42.0.1"} { + if !got[ip] { + t.Errorf("%s: node IP %s must resolve into the address surface, got %v", dir.name, ip, got) + } + } + } +} + +// A nil profile is a no-op, and WithHostPeer must not mutate its input. +func TestWithHostPeer_NilAndNoMutation(t *testing.T) { + if WithHostPeer(nil) != nil { + t.Error("WithHostPeer(nil) must be nil") + } + cp := &v1beta1.ContainerProfile{Spec: v1beta1.ContainerProfileSpec{Egress: []v1beta1.NetworkNeighbor{{Identifier: "keep"}}}} + _ = WithHostPeer(cp) + if len(cp.Spec.Egress) != 1 || len(cp.Spec.Ingress) != 0 { + t.Errorf("WithHostPeer must not mutate the input; egress=%d ingress=%d", len(cp.Spec.Egress), len(cp.Spec.Ingress)) + } +} diff --git a/pkg/networkpeer/lister.go b/pkg/networkpeer/lister.go new file mode 100644 index 0000000000..65c27b03ff --- /dev/null +++ b/pkg/networkpeer/lister.go @@ -0,0 +1,192 @@ +package networkpeer + +import ( + "net" + "sync/atomic" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" +) + +// InformerLister is the production Lister, backed by Service / EndpointSlice / +// Node informer listers. It resolves a serviceRef/serviceSelector to the +// Service's ClusterIP(s) ∪ its EndpointSlice addresses, and the "host" entity +// to every node's InternalIP(s) plus the CNI gateway derived from its PodCIDR. +type InformerLister struct { + services corelisters.ServiceLister + slices discoverylisters.EndpointSliceLister + nodes corelisters.NodeLister + // nodeName scopes the "host" entity to the local node. Empty means every + // node (used by tests); production passes the agent's own node so kubelet + // probes from this node's gateway match without broadening "host" to the + // whole cluster. + nodeName string + // generation advances on every observed Service/EndpointSlice/Node change + // (bumped from informer event handlers wired in cmd/main.go). + generation atomic.Int64 +} + +// Generation returns the current cluster-view generation. +func (l *InformerLister) Generation() int64 { return l.generation.Load() } + +// Bump advances the generation; wire it to the informer event handlers. +func (l *InformerLister) Bump() { l.generation.Add(1) } + +func NewInformerLister(services corelisters.ServiceLister, slices discoverylisters.EndpointSliceLister, nodes corelisters.NodeLister, nodeName string) *InformerLister { + return &InformerLister{services: services, slices: slices, nodes: nodes, nodeName: nodeName} +} + +var _ Lister = (*InformerLister)(nil) + +func (l *InformerLister) ServiceByName(namespace, name string) (*ServiceInfo, bool) { + svc, err := l.services.Services(namespace).Get(name) + if err != nil { + return nil, false + } + return l.serviceInfo(svc), true +} + +func (l *InformerLister) ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo { + // Never resolve an empty selector to labels.Everything() — that would + // allowlist every Service in the cluster. Fail closed. + if len(serviceSelector) == 0 { + return nil + } + svcs, err := l.services.List(labels.SelectorFromSet(serviceSelector)) + if err != nil { + return nil + } + // nil namespaceLabels => cluster-wide (all namespaces); a non-nil map scopes + // strictly to its metadata.name (a present-but-empty name matches nothing). + scoped := namespaceLabels != nil + wantNS := namespaceLabels["kubernetes.io/metadata.name"] + var out []*ServiceInfo + for _, svc := range svcs { + if scoped && svc.Namespace != wantNS { + continue + } + out = append(out, l.serviceInfo(svc)) + } + return out +} + +func (l *InformerLister) HostIPs() []string { + nodes, err := l.nodes.List(labels.Everything()) + if err != nil { + return nil + } + var ips []string + for _, n := range nodes { + if l.nodeName != "" && n.Name != l.nodeName { + continue + } + for _, addr := range n.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + ips = append(ips, addr.Address) + } + } + for _, cidr := range podCIDRs(n) { + if gw := gatewayIP(cidr); gw != "" { + ips = append(ips, gw) + } + } + } + return dedupe(ips) +} + +func (l *InformerLister) serviceInfo(svc *corev1.Service) *ServiceInfo { + info := &ServiceInfo{Namespace: svc.Namespace, Name: svc.Name, Labels: svc.Labels} + for _, ip := range svc.Spec.ClusterIPs { + if ip != "" && ip != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, ip) + } + } + if len(info.ClusterIPs) == 0 && svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, svc.Spec.ClusterIP) + } + info.EndpointIPs = l.endpointIPs(svc.Namespace, svc.Name) + return info +} + +func (l *InformerLister) endpointIPs(namespace, service string) []string { + sel := labels.SelectorFromSet(labels.Set{discoveryv1.LabelServiceName: service}) + slices, err := l.slices.EndpointSlices(namespace).List(sel) + if err != nil { + return nil + } + var ips []string + for _, es := range slices { + for i := range es.Endpoints { + ips = append(ips, es.Endpoints[i].Addresses...) + } + } + return dedupe(ips) +} + +// TrimService and TrimEndpointSlice are informer TransformFuncs that drop the +// bulk the resolver never reads — managedFields and annotations (1–4 KiB per +// real object), and for EndpointSlices every per-endpoint field but Addresses — +// before objects enter the cluster-wide cache. Wire via Informer().SetTransform +// so a DaemonSet's per-node Service/EndpointSlice cache stays small. Identity +// and resourceVersion are preserved so listing/indexing is unaffected. +func TrimService(obj interface{}) (interface{}, error) { + if svc, ok := obj.(*corev1.Service); ok { + svc.ManagedFields = nil + svc.Annotations = nil + } + return obj, nil +} + +func TrimEndpointSlice(obj interface{}) (interface{}, error) { + if es, ok := obj.(*discoveryv1.EndpointSlice); ok { + es.ManagedFields = nil + es.Annotations = nil + for i := range es.Endpoints { + es.Endpoints[i] = discoveryv1.Endpoint{Addresses: es.Endpoints[i].Addresses} + } + } + return obj, nil +} + +func podCIDRs(n *corev1.Node) []string { + if len(n.Spec.PodCIDRs) > 0 { + return n.Spec.PodCIDRs + } + if n.Spec.PodCIDR != "" { + return []string{n.Spec.PodCIDR} + } + return nil +} + +// gatewayIP returns the conventional CNI gateway for a pod CIDR: the network +// address + 1 (e.g. 10.42.0.0/24 -> 10.42.0.1). Masqueraded node-sourced +// traffic (kubelet health probes) appears from this address. IPv6 CIDRs yield +// no gateway (the .1 convention is IPv4). +func gatewayIP(cidr string) string { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return "" + } + ip := ipNet.IP.To4() + if ip == nil { + return "" + } + gw := make(net.IP, len(ip)) + copy(gw, ip) + for i := len(gw) - 1; i >= 0; i-- { + gw[i]++ + if gw[i] != 0 { + break + } + } + // A /31 or /32 (or a network address ending in .255 that overflows) yields a + // gateway outside the CIDR — never allowlist an IP the pod network doesn't + // actually contain. + if !ipNet.Contains(gw) { + return "" + } + return gw.String() +} diff --git a/pkg/networkpeer/lister_test.go b/pkg/networkpeer/lister_test.go new file mode 100644 index 0000000000..98d0143b10 --- /dev/null +++ b/pkg/networkpeer/lister_test.go @@ -0,0 +1,147 @@ +package networkpeer + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +// newTestLister builds an InformerLister over a fake cluster seeded from the +// real flux/kubescape topology, exercising the production Service/EndpointSlice/ +// Node -> Lister path (not the hand-written fake used by the resolver tests). +func newTestLister(t *testing.T) *InformerLister { + t.Helper() + client := fake.NewClientset( + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager", Labels: map[string]string{"app": "alertmanager"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.54.190", ClusterIPs: []string{"10.43.54.190"}}, + }, + &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager-x1", Labels: map[string]string{discoveryv1.LabelServiceName: "alertmanager"}}, + Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.42.0.44", "10.42.0.84"}}}, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "gitops-demo", Name: "guestbook-ui", Labels: map[string]string{"app": "guestbook"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.111.192", ClusterIPs: []string{"10.43.111.192"}}, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "tanzee"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }, + &corev1.Node{ // a second node whose IPs must NOT leak into this node's "host" + ObjectMeta: metav1.ObjectMeta{Name: "other"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.9.0/24", PodCIDRs: []string{"10.42.9.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.99"}}}, + }, + ) + factory := informers.NewSharedInformerFactory(client, 0) + l := NewInformerLister( + factory.Core().V1().Services().Lister(), + factory.Discovery().V1().EndpointSlices().Lister(), + factory.Core().V1().Nodes().Lister(), + "tanzee", + ) + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + factory.Start(stop) + factory.WaitForCacheSync(stop) + return l +} + +func TestInformerLister_ServiceByName(t *testing.T) { + l := newTestLister(t) + svc, ok := l.ServiceByName("honey", "alertmanager") + if !ok { + t.Fatal("alertmanager Service should resolve") + } + if len(svc.ClusterIPs) != 1 || svc.ClusterIPs[0] != "10.43.54.190" { + t.Errorf("ClusterIPs: got %v", svc.ClusterIPs) + } + if len(svc.EndpointIPs) != 2 { + t.Errorf("EndpointIPs: got %v (want 2 from the EndpointSlice)", svc.EndpointIPs) + } + // End-to-end through the resolver, like the projection will. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: []PortProto{{Port: 9093, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.43.54.190", 9093, "TCP") || !Matches(tuples, "10.42.0.44", 9093, "TCP") { + t.Errorf("resolver over informer lister should match ClusterIP + endpoints") + } + if _, ok := l.ServiceByName("honey", "nope"); ok { + t.Error("unknown Service must not resolve") + } +} + +func TestInformerLister_HostIPs(t *testing.T) { + l := newTestLister(t) + ips := l.HostIPs() + want := map[string]bool{"192.168.0.191": false, "10.42.0.1": false} + for _, ip := range ips { + if _, ok := want[ip]; ok { + want[ip] = true + } + } + for ip, seen := range want { + if !seen { + t.Errorf("HostIPs missing %s (got %v)", ip, ips) + } + } + for _, ip := range ips { + if ip == "192.168.0.99" || ip == "10.42.9.1" { + t.Errorf("HostIPs must be scoped to the local node; leaked other node's %s", ip) + } + } +} + +// TestInformerLister_EmptySelectorFailsClosed: an empty serviceSelector must +// NOT resolve to every Service in the cluster. +func TestInformerLister_EmptySelectorFailsClosed(t *testing.T) { + l := newTestLister(t) + if got := l.ServicesByLabels(map[string]string{}, nil); len(got) != 0 { + t.Errorf("empty selector must fail closed, got %d services", len(got)) + } + if got := Resolve(PeerSpec{ServiceSelector: map[string]string{}, Ports: tcp(80)}, l); len(got) != 0 { + t.Errorf("Resolve with empty selector must yield nothing, got %v", got) + } +} + +func TestInformerLister_ServicesByLabels(t *testing.T) { + l := newTestLister(t) + svcs := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}) + if len(svcs) != 1 || svcs[0].Name != "guestbook-ui" { + t.Fatalf("expected guestbook-ui, got %v", svcs) + } + // Namespace scoping excludes a same-label service elsewhere (none here) and + // a wrong-namespace filter yields nothing. + if got := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "other"}); len(got) != 0 { + t.Errorf("namespace filter should exclude, got %v", got) + } +} + +func TestGatewayIP(t *testing.T) { + cases := map[string]string{ + "10.42.0.0/24": "10.42.0.1", + "10.244.5.0/24": "10.244.5.1", + "2001:db8::/64": "", + "10.42.0.5/32": "", // /32 host: incremented gateway is outside the CIDR + "255.255.255.255/32": "", // overflow to 0.0.0.0, out of CIDR + } + for cidr, want := range cases { + if got := gatewayIP(cidr); got != want { + t.Errorf("gatewayIP(%s)=%q want %q", cidr, got, want) + } + } +} + +func TestInformerLister_Generation(t *testing.T) { + l := newTestLister(t) + g0 := l.Generation() + l.Bump() + l.Bump() + if l.Generation() != g0+2 { + t.Errorf("Generation must advance on Bump: got %d want %d", l.Generation(), g0+2) + } +} diff --git a/pkg/networkpeer/namespace_consistency_test.go b/pkg/networkpeer/namespace_consistency_test.go new file mode 100644 index 0000000000..0e7f98088a --- /dev/null +++ b/pkg/networkpeer/namespace_consistency_test.go @@ -0,0 +1,109 @@ +package networkpeer + +import ( + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func nsName(name string) *metav1.LabelSelector { + return &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": name}} +} + +// TestExpandServiceNeighbors_NamespaceSelectorConsistency proves the service +// (serviceSelector) resolution path scopes namespaces with the SAME three tiers +// as the podSelector matcher — compare row-for-row with +// containerprofilenetwork.TestNamespaceSelectorMatches_TruthTable and +// TestWasSelectorInPeers_NamespaceDisambiguation: +// +// OMITTED (nil) => the profile's OWN namespace +// explicit EMPTY {} => cluster-wide (opt-in) +// explicit metadata.name => that namespace +// +// An identical Service {app: api} exists in the profile's namespace ("prod") and +// in "attacker"; profileNs is "prod". +func TestExpandServiceNeighbors_NamespaceSelectorConsistency(t *testing.T) { + const profileNs = "prod" + l := &fakeLister{services: map[string]*ServiceInfo{ + "prod/api": {Namespace: "prod", Name: "api", Labels: map[string]string{"app": "api", "__ns__": "prod"}, ClusterIPs: []string{"10.0.0.1"}}, + "attacker/api": {Namespace: "attacker", Name: "api", Labels: map[string]string{"app": "api", "__ns__": "attacker"}, ClusterIPs: []string{"10.0.0.2"}}, + }} + svcSel := &metav1.LabelSelector{MatchLabels: map[string]string{"app": "api"}} + + resolved := func(nsSel *metav1.LabelSelector) map[string]bool { + n := v1beta1.NetworkNeighbor{Identifier: "api", ServiceSelector: svcSel, NamespaceSelector: nsSel, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}} + got := map[string]bool{} + for _, o := range ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{n}, profileNs, l) { + for _, ip := range o.IPAddresses { + got[ip] = true + } + } + return got + } + + cases := []struct { + name string + nsSel *metav1.LabelSelector + wantProd, wantAtk bool + }{ + {"omitted => same ns (prod only)", nil, true, false}, + {"empty {} => cluster-wide (both)", &metav1.LabelSelector{}, true, true}, + {"metadata.name=prod => prod only", nsName("prod"), true, false}, + {"metadata.name=attacker => attacker only", nsName("attacker"), false, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := resolved(tc.nsSel) + if got["10.0.0.1"] != tc.wantProd || got["10.0.0.2"] != tc.wantAtk { + t.Fatalf("prod=%v attacker=%v, want prod=%v attacker=%v", + got["10.0.0.1"], got["10.0.0.2"], tc.wantProd, tc.wantAtk) + } + }) + } +} + +// TestExpandServiceNeighbors_VendorPortableProfile is the service-path twin of +// containerprofilenetwork.TestWasSelectorInPeers_VendorPortableProfile: a vendor +// ships the SAME serviceSelector peers WITHOUT its namespace, installed anywhere. +// DNS is pinned to kube-system by name; Prometheus uses {} (any ns, the vendor +// can't know monitoring's namespace); the app's own backend uses an omitted +// selector (same ns). A same-labelled backend Service in "evil" is NOT resolved. +func TestExpandServiceNeighbors_VendorPortableProfile(t *testing.T) { + topo := func(install string) *fakeLister { + return &fakeLister{services: map[string]*ServiceInfo{ + "kube-system/kube-dns": {Namespace: "kube-system", Name: "kube-dns", Labels: map[string]string{"k8s-app": "kube-dns", "__ns__": "kube-system"}, ClusterIPs: []string{"10.96.0.10"}}, + "monitoring/prometheus": {Namespace: "monitoring", Name: "prometheus", Labels: map[string]string{"app.kubernetes.io/name": "prometheus", "__ns__": "monitoring"}, ClusterIPs: []string{"10.96.1.1"}}, + install + "/backend": {Namespace: install, Name: "backend", Labels: map[string]string{"app.kubernetes.io/name": "acme-backend", "__ns__": install}, ClusterIPs: []string{"10.96.2.2"}}, + "evil/backend": {Namespace: "evil", Name: "backend", Labels: map[string]string{"app.kubernetes.io/name": "acme-backend", "__ns__": "evil"}, ClusterIPs: []string{"10.96.9.9"}}, + }} + } + peers := []v1beta1.NetworkNeighbor{ + {Identifier: "dns", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k8s-app": "kube-dns"}}, NamespaceSelector: nsName("kube-system"), Ports: []v1beta1.NetworkPort{port("UDP-53", 53)}}, + {Identifier: "prometheus", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "prometheus"}}, NamespaceSelector: &metav1.LabelSelector{}, Ports: []v1beta1.NetworkPort{port("TCP-9090", 9090)}}, + {Identifier: "backend", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "acme-backend"}}, Ports: []v1beta1.NetworkPort{port("TCP-8080", 8080)}}, + } + + for _, install := range []string{"acme", "tenant-42"} { + t.Run("install="+install, func(t *testing.T) { + ips := map[string]bool{} + for _, o := range ExpandServiceNeighbors(peers, install, topo(install)) { + for _, ip := range o.IPAddresses { + ips[ip] = true + } + } + if !ips["10.96.0.10"] { + t.Error("DNS in kube-system must resolve (metadata.name pin)") + } + if !ips["10.96.1.1"] { + t.Error("Prometheus must resolve in monitoring ({} cluster-wide)") + } + if !ips["10.96.2.2"] { + t.Errorf("the app's own backend in the install ns %q must resolve (omitted = same ns)", install) + } + if ips["10.96.9.9"] { + t.Error("LABEL-COPY: a same-labelled backend Service in another ns must NOT resolve") + } + }) + } +} diff --git a/pkg/networkpeer/perf_bench_test.go b/pkg/networkpeer/perf_bench_test.go new file mode 100644 index 0000000000..e3343214b8 --- /dev/null +++ b/pkg/networkpeer/perf_bench_test.go @@ -0,0 +1,283 @@ +package networkpeer + +import ( + "fmt" + "runtime" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" + "k8s.io/client-go/tools/cache" +) + +func benchService(i int) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", i%50), + Name: fmt.Sprintf("svc-%d", i), + Labels: map[string]string{"app": fmt.Sprintf("app-%d", i), "team": fmt.Sprintf("team-%d", i%20)}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: fmt.Sprintf("10.43.%d.%d", i/256, i%256), + ClusterIPs: []string{fmt.Sprintf("10.43.%d.%d", i/256, i%256)}, + Ports: []corev1.ServicePort{{Name: "http", Port: 8080, Protocol: corev1.ProtocolTCP}}, + }, + } +} + +func benchSlice(svcIdx, sliceIdx, endpoints int) *discoveryv1.EndpointSlice { + eps := make([]discoveryv1.Endpoint, 0, endpoints) + ready := true + for e := 0; e < endpoints; e++ { + eps = append(eps, discoveryv1.Endpoint{ + Addresses: []string{fmt.Sprintf("10.42.%d.%d", (svcIdx*7+e)%256, (sliceIdx*31+e)%256)}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: fmt.Sprintf("ns-%d", svcIdx%50), Name: fmt.Sprintf("pod-%d-%d-%d", svcIdx, sliceIdx, e)}, + NodeName: ptrTo(fmt.Sprintf("node-%d", e%10)), + }) + } + return &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", svcIdx%50), + Name: fmt.Sprintf("svc-%d-%d", svcIdx, sliceIdx), + Labels: map[string]string{discoveryv1.LabelServiceName: fmt.Sprintf("svc-%d", svcIdx)}, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: eps, + Ports: []discoveryv1.EndpointPort{{Name: ptrTo("http"), Port: ptrTo(int32(8080)), Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0]}}, + } +} + +func ptrTo[T any](v T) *T { return &v } + +// buildBenchLister backs an InformerLister with plain cache indexers (the same +// store type a SharedInformer uses) so benchmarks measure lister/resolution +// cost without fake-clientset watch machinery. +func buildBenchLister(tb testing.TB, nServices, slicesPerSvc, endpointsPerSlice int) *InformerLister { + tb.Helper() + svcIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + sliceIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + nodeIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for i := 0; i < nServices; i++ { + if err := svcIdx.Add(benchService(i)); err != nil { + tb.Fatal(err) + } + for s := 0; s < slicesPerSvc; s++ { + if err := sliceIdx.Add(benchSlice(i, s, endpointsPerSlice)); err != nil { + tb.Fatal(err) + } + } + } + if err := nodeIdx.Add(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "bench-node"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }); err != nil { + tb.Fatal(err) + } + return NewInformerLister( + corelisters.NewServiceLister(svcIdx), + discoverylisters.NewEndpointSliceLister(sliceIdx), + corelisters.NewNodeLister(nodeIdx), + "bench-node", + ) +} + +// benchProfile builds a ContainerProfile with nPlain ordinary ipAddresses +// egress neighbors, nRefs serviceRef neighbors, and some opens/execs bulk so +// DeepCopy cost is realistic. +func benchProfile(nPlain, nRefs, nOpens int) *v1beta1.ContainerProfile { + cp := &v1beta1.ContainerProfile{} + cp.Name = "bench-cp" + port := int32(8080) + for i := 0; i < nPlain; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("plain-%d", i), + Type: "external", + IPAddresses: []string{fmt.Sprintf("52.216.%d.%d", i/256, i%256)}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nRefs; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("ref-%d", i), + Type: "internal", + ServiceRefNamespace: fmt.Sprintf("ns-%d", i%50), + ServiceRefName: fmt.Sprintf("svc-%d", i), + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nOpens; i++ { + cp.Spec.Opens = append(cp.Spec.Opens, v1beta1.OpenCalls{ + Path: fmt.Sprintf("/usr/lib/x86_64-linux-gnu/lib-%d.so.%d", i, i%9), + Flags: []string{"O_RDONLY", "O_CLOEXEC"}, + }) + cp.Spec.Execs = append(cp.Spec.Execs, v1beta1.ExecCalls{ + Path: fmt.Sprintf("/usr/bin/tool-%d", i), + Args: []string{fmt.Sprintf("--flag-%d", i)}, + }) + } + return cp +} + +func BenchmarkServiceByName_1kSvc_5kSlices(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := l.ServiceByName("ns-7", "svc-7"); !ok { + b.Fatal("service must resolve") + } + } +} + +func BenchmarkServicesByLabels_1kSvc(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + sel := map[string]string{"app": "app-7"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if out := l.ServicesByLabels(sel, nil); len(out) != 1 { + b.Fatalf("want 1 service, got %d", len(out)) + } + } +} + +func BenchmarkResolveIPs_ServiceRef(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + spec := PeerSpec{ServiceRef: &ServiceRef{Namespace: "ns-7", Name: "svc-7"}} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +func BenchmarkResolveIPs_Entity_Host(b *testing.B) { + l := buildBenchLister(b, 10, 1, 2) + spec := PeerSpec{Entity: "host"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +// BenchmarkWithResolvedServiceNeighbors_NoServiceFields is the 99% case: a +// profile with only plain ipAddresses neighbors. The function documents itself +// as a no-op then — this measures whether the no-op is actually free. +func BenchmarkWithResolvedServiceNeighbors_NoServiceFields(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, n := range []int{100, 1000} { + cp := benchProfile(n, 0, 0) + b.Run(fmt.Sprintf("plainNeighbors=%d", n), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out != cp { + b.Fatal("no-op path must return the same pointer") + } + } + }) + } +} + +// BenchmarkWithResolvedServiceNeighbors_Resolving measures the full expansion: +// resolution + DeepCopy of the whole profile (opens/execs bulk included). +func BenchmarkWithResolvedServiceNeighbors_Resolving(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, tc := range []struct{ plain, refs, opens int }{ + {plain: 20, refs: 1, opens: 200}, + {plain: 20, refs: 8, opens: 200}, + {plain: 20, refs: 8, opens: 2000}, + } { + cp := benchProfile(tc.plain, tc.refs, tc.opens) + b.Run(fmt.Sprintf("plain=%d/refs=%d/opens=%d", tc.plain, tc.refs, tc.opens), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out == cp { + b.Fatal("resolving path must copy") + } + } + }) + } +} + +func BenchmarkHasServiceNeighbors_1kPlain(b *testing.B) { + cp := benchProfile(1000, 0, 0) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if HasServiceNeighbors(cp) { + b.Fatal("plain profile must not report service neighbors") + } + } +} + +// TestInformerCacheMemoryEstimate approximates the heap retained by a +// cluster-wide Service + EndpointSlice informer cache at 1k Services / 5k +// EndpointSlices (10 endpoints each), vs Services alone. Run with -run +// InformerCacheMemoryEstimate -v. +func TestInformerCacheMemoryEstimate(t *testing.T) { + measure := func(build func() []interface{}) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + objs := build() + runtime.GC() + runtime.ReadMemStats(&after) + runtime.KeepAlive(objs) + if after.HeapAlloc < before.HeapAlloc { + return 0 + } + return after.HeapAlloc - before.HeapAlloc + } + + svcBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + _ = idx.Add(benchService(i)) + } + return []interface{}{idx} + }) + sliceBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + _ = idx.Add(benchSlice(i, s, 10)) + } + } + return []interface{}{idx} + }) + strippedBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + full := benchSlice(i, s, 10) + stripped := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: full.Namespace, Name: full.Name, Labels: full.Labels}, + AddressType: full.AddressType, + } + for _, ep := range full.Endpoints { + stripped.Endpoints = append(stripped.Endpoints, discoveryv1.Endpoint{Addresses: ep.Addresses}) + } + _ = idx.Add(stripped) + } + } + return []interface{}{idx} + }) + t.Logf("1000 Services in indexer: ~%d KiB total, ~%d B/object", svcBytes/1024, svcBytes/1000) + t.Logf("5000 EndpointSlices (10 endpoints each) in indexer: ~%d KiB total, ~%d B/object", sliceBytes/1024, sliceBytes/5000) + t.Logf("5000 STRIPPED EndpointSlices (addresses+labels only, SetTransform mitigation): ~%d KiB total, ~%d B/object", strippedBytes/1024, strippedBytes/5000) +} diff --git a/pkg/networkpeer/resolve.go b/pkg/networkpeer/resolve.go new file mode 100644 index 0000000000..de8c1375d7 --- /dev/null +++ b/pkg/networkpeer/resolve.go @@ -0,0 +1,228 @@ +// Package networkpeer resolves Kubernetes-native network-neighbor selectors +// (a Service reference, a Service label selector, or a reserved entity such as +// "host") into the concrete (IP, port, protocol) tuples an egress/ingress +// allowlist should match. +// +// It exists so a ContainerProfile can express cluster-infrastructure peers — +// Service ClusterIPs (alertmanager, kube-apiserver via default/kubernetes, +// kube-dns, ...) and host/kubelet traffic — portably (by name, resolved +// per-cluster) and narrowly (only the referenced Service/entity), instead of a +// broad ipAddresses CIDR over the whole service network. See +// k8sstormcenter/node-agent#92. +// +// Resolution is intentionally decoupled from live matching: callers resolve a +// PeerSpec to []AllowTuple once (e.g. at projection time) via a Lister backed +// by Service/EndpointSlice/Node informers, then match observed connections +// against the tuples with Matches. The Lister interface keeps the resolver +// unit-testable against a fake cluster view. +package networkpeer + +import "strings" + +// EntityHost is the reserved entity naming the local node: its InternalIP(s) +// and the CNI gateway address. It is the one peer class no Service object can +// represent (kubelet health probes, node-sourced / masqueraded traffic). +const EntityHost = "host" + +// clusterDNSSuffix is the in-cluster Service DNS zone. A serviceRef / +// serviceSelector implies the cluster FQDN ..svc. so a +// client dialling the Service by DNS is allowlisted without a parallel +// dnsNames entry. Kubernetes' default zone; overridden per-cluster is rare. +const clusterDNSSuffix = "svc.cluster.local" + +// PortProto is a single allowed destination port/protocol. Protocol is +// upper-case ("TCP"/"UDP"); an empty Protocol matches any protocol. +type PortProto struct { + Port int32 + Protocol string +} + +// ServiceRef names a single Service by namespace and name. +type ServiceRef struct { + Namespace string + Name string +} + +// PeerSpec is the storage-agnostic form of one serviceRef / serviceSelector / +// entity network-neighbor entry. Exactly one of ServiceRef, ServiceSelector, +// or Entity is expected to be set; Ports scopes the resolved tuples. +type PeerSpec struct { + ServiceRef *ServiceRef + ServiceSelector map[string]string + NamespaceLabels map[string]string + Entity string + Ports []PortProto +} + +// ServiceInfo is the resolver's view of one Service. +type ServiceInfo struct { + Namespace string + Name string + Labels map[string]string + ClusterIPs []string + EndpointIPs []string +} + +// Lister is the read-only cluster view the resolver needs. Production wires it +// to Service/EndpointSlice/Node informer listers; tests use a fake. +type Lister interface { + ServiceByName(namespace, name string) (*ServiceInfo, bool) + ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo + HostIPs() []string + // Generation increments whenever the underlying cluster view changes (any + // Service/EndpointSlice/Node event). Callers store it alongside a projected + // profile and re-project when it advances, so resolved IPs don't go stale on + // endpoint churn or caches that filled after projection. + Generation() int64 +} + +// AllowTuple is one concrete (IP, port, protocol) a resolved PeerSpec permits. +type AllowTuple struct { + IP string + Port int32 + Protocol string +} + +// Resolve expands spec into the concrete tuples it authorises, using l for the +// current cluster view. A spec with no resolvable target (unknown Service, +// selector matching nothing, unknown entity) yields no tuples — never a +// match-all. A spec with no Ports yields one tuple per IP with Port 0 / +// Protocol "" (any-port), so callers that ignore ports still work; callers +// that enforce ports should treat Port 0 as "unspecified". +func Resolve(spec PeerSpec, l Lister) []AllowTuple { + if l == nil { + return nil + } + ips := resolveIPs(spec, l) + if len(ips) == 0 { + return nil + } + return expand(ips, spec.Ports) +} + +// ResolveIPs returns just the IPs a spec resolves to, ignoring ports. Used by +// the projection-time expansion, which pairs them with the neighbor's own +// ports. +func ResolveIPs(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + return resolveIPs(spec, l) +} + +func resolveIPs(spec PeerSpec, l Lister) []string { + if spec.Entity != "" { + if strings.EqualFold(spec.Entity, EntityHost) { + return dedupe(l.HostIPs()) + } + return nil + } + var ips []string + for _, svc := range resolveServices(spec, l) { + ips = append(ips, serviceIPs(svc)...) + } + return dedupe(ips) +} + +// resolveServices returns the Services a serviceRef / serviceSelector spec +// matches. An entity spec matches no Service and returns nil; an empty +// serviceSelector fails closed (never every Service). +func resolveServices(spec PeerSpec, l Lister) []*ServiceInfo { + switch { + case spec.ServiceRef != nil: + svc, ok := l.ServiceByName(spec.ServiceRef.Namespace, spec.ServiceRef.Name) + if !ok || svc == nil { + return nil + } + return []*ServiceInfo{svc} + case spec.ServiceSelector != nil: + if len(spec.ServiceSelector) == 0 { + return nil + } + return l.ServicesByLabels(spec.ServiceSelector, spec.NamespaceLabels) + default: + return nil + } +} + +// ResolveDNSNames returns the cluster FQDN(s) — ..svc. — +// of the Services a serviceRef / serviceSelector spec matches, so a client +// dialling the Service by DNS is allowlisted alongside its IPs. Entity specs +// and unresolvable selectors yield nothing. +func ResolveDNSNames(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + var out []string + for _, svc := range resolveServices(spec, l) { + if svc == nil || svc.Namespace == "" || svc.Name == "" { + continue + } + out = append(out, svc.Name+"."+svc.Namespace+"."+clusterDNSSuffix) + } + return dedupe(out) +} + +func serviceIPs(svc *ServiceInfo) []string { + out := make([]string, 0, len(svc.ClusterIPs)+len(svc.EndpointIPs)) + out = append(out, svc.ClusterIPs...) + out = append(out, svc.EndpointIPs...) + return dedupe(out) +} + +func expand(ips []string, ports []PortProto) []AllowTuple { + if len(ports) == 0 { + out := make([]AllowTuple, 0, len(ips)) + for _, ip := range ips { + out = append(out, AllowTuple{IP: ip}) + } + return out + } + out := make([]AllowTuple, 0, len(ips)*len(ports)) + for _, ip := range ips { + for _, p := range ports { + out = append(out, AllowTuple{IP: ip, Port: p.Port, Protocol: strings.ToUpper(p.Protocol)}) + } + } + return out +} + +// Matches reports whether the observed (ip, port, protocol) connection is +// permitted by any tuple. Matching is port-sensitive: a tuple with Port 0 +// (any-port) matches any observed port; otherwise the port must be equal. An +// empty tuple Protocol matches any protocol. +func Matches(tuples []AllowTuple, ip string, port int32, protocol string) bool { + protocol = strings.ToUpper(protocol) + for _, t := range tuples { + if t.IP != ip { + continue + } + if t.Port != 0 && t.Port != port { + continue + } + if t.Protocol != "" && t.Protocol != protocol { + continue + } + return true + } + return false +} + +func dedupe(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} diff --git a/pkg/networkpeer/resolve_test.go b/pkg/networkpeer/resolve_test.go new file mode 100644 index 0000000000..9a16e0a828 --- /dev/null +++ b/pkg/networkpeer/resolve_test.go @@ -0,0 +1,359 @@ +package networkpeer + +import ( + "sort" + "testing" +) + +// fakeLister is a static cluster view seeded from the real Flux/kubescape +// topology observed on the k3s dev cluster (issue #92). It lets the resolver +// tests assert against genuine ClusterIPs/endpoints without a live cluster. +type fakeLister struct { + services map[string]*ServiceInfo // key "ns/name" + hostIPs []string +} + +func (f *fakeLister) ServiceByName(ns, name string) (*ServiceInfo, bool) { + s, ok := f.services[ns+"/"+name] + return s, ok +} + +func (f *fakeLister) ServicesByLabels(sel, nsLabels map[string]string) []*ServiceInfo { + var out []*ServiceInfo + for _, s := range f.services { + if nsLabels != nil { + if s.Labels["__ns__"] != nsLabels["kubernetes.io/metadata.name"] { + continue + } + } + if labelsSubset(sel, s.Labels) { + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func (f *fakeLister) HostIPs() []string { return f.hostIPs } + +func (f *fakeLister) Generation() int64 { return 0 } + +func labelsSubset(want, have map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// realFluxTopology mirrors what `kubectl get svc/endpoints` returned live. +func realFluxTopology() *fakeLister { + return &fakeLister{ + hostIPs: []string{"192.168.0.191", "10.42.0.1"}, // node InternalIP + CNI gateway + services: map[string]*ServiceInfo{ + "honey/alertmanager": { + Namespace: "honey", Name: "alertmanager", + Labels: map[string]string{"app": "alertmanager", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.54.190"}, + EndpointIPs: []string{"10.42.0.44", "10.42.0.84"}, + }, + "honey/storage": { + Namespace: "honey", Name: "storage", + Labels: map[string]string{"app": "storage", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.70.156"}, + }, + "default/kubernetes": { // k3s: apiserver endpoint is the node IP (Kind: Host) + Namespace: "default", Name: "kubernetes", + Labels: map[string]string{"__ns__": "default"}, + ClusterIPs: []string{"10.43.0.1"}, + EndpointIPs: []string{"192.168.0.191"}, + }, + "argocd/argocd-server": { + Namespace: "argocd", Name: "argocd-server", + Labels: map[string]string{"app.kubernetes.io/name": "argocd-server", "__ns__": "argocd"}, + ClusterIPs: []string{"10.43.173.14"}, + }, + "gitops-demo/guestbook-ui": { + Namespace: "gitops-demo", Name: "guestbook-ui", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.111.192"}, + }, + "gitops-demo/helm-guestbook": { + Namespace: "gitops-demo", Name: "helm-guestbook", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.3.63"}, + }, + }, + } +} + +func tcp(port int32) []PortProto { return []PortProto{{Port: port, Protocol: "TCP"}} } + +// Test A1 — serviceRef egress (alertmanager) is narrow AND port-sensitive: +// matches its ClusterIP and endpoints on 9093 only; a sibling service on the +// same port stays visible to R0011 (the whole point vs a /16). +func TestServiceRef_Alertmanager(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: tcp(9093)}, l) + + cases := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {"10.43.54.190", 9093, "TCP", true, "ClusterIP + port match"}, + {"10.42.0.44", 9093, "TCP", true, "backing endpoint IP"}, + {"10.42.0.84", 9093, "TCP", true, "backing endpoint IP"}, + {"10.43.54.190", 8080, "TCP", false, "wrong port (port-sensitive)"}, + {"10.43.54.190", 9093, "UDP", false, "wrong protocol"}, + {"10.43.173.14", 9093, "TCP", false, "argocd-server — different service, detection preserved"}, + {"10.43.70.156", 9093, "TCP", false, "storage — different service, detection preserved"}, + } + for _, c := range cases { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%s:%d/%s)=%v want %v (%s)", c.ip, c.port, c.proto, got, c.want, c.why) + } + } +} + +// Test A2 — kube-apiserver needs no dedicated entity: toService default/kubernetes +// resolves to the ClusterIP AND the node-IP endpoint (k3s embeds the apiserver). +func TestServiceRef_KubeApiserver(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + for _, ip := range []string{"10.43.0.1", "192.168.0.191"} { + if !Matches(tuples, ip, 443, "TCP") { + t.Errorf("apiserver egress %s:443 should match via default/kubernetes", ip) + } + } + if Matches(tuples, "10.43.0.1", 6443, "TCP") { + t.Errorf("apiserver :6443 must not match (port 443 only)") + } +} + +// Test A3 — host entity: kubelet probe from the node/gateway matches on the +// health port only; a pod-CIDR source or wrong port does not. +func TestEntityHost(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(9440)}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("kubelet probe 10.42.0.1:9440 should match fromEntity host") + } + if !Matches(tuples, "192.168.0.191", 9440, "TCP") { + t.Errorf("node InternalIP :9440 should match fromEntity host") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("host :9090 must not match (port 9440 only)") + } + if Matches(tuples, "10.42.0.55", 9440, "TCP") { + t.Errorf("a pod IP must not match fromEntity host") + } +} + +// Test A4 — serviceSelector fans out across all matching Services (the two +// gitops-demo guestbook services share app=guestbook), scoped by namespace. +func TestServiceSelector_GuestbookFanout(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + Ports: tcp(80), + }, l) + for _, ip := range []string{"10.43.111.192", "10.43.3.63"} { + if !Matches(tuples, ip, 80, "TCP") { + t.Errorf("guestbook service %s:80 should match app=guestbook selector", ip) + } + } + if Matches(tuples, "10.43.173.14", 80, "TCP") { + t.Errorf("argocd-server must not match app=guestbook selector") + } +} + +// Test A5 — no accidental match-all: unknown service, unknown entity, and a +// selector matching nothing all resolve to zero tuples. +func TestResolve_NoMatchAll(t *testing.T) { + l := realFluxTopology() + specs := []PeerSpec{ + {ServiceRef: &ServiceRef{"honey", "does-not-exist"}, Ports: tcp(443)}, + {Entity: "world", Ports: tcp(443)}, + {ServiceSelector: map[string]string{"app": "nope"}, Ports: tcp(443)}, + } + for i, s := range specs { + if tuples := Resolve(s, l); len(tuples) != 0 { + t.Errorf("spec[%d] should resolve to no tuples, got %d", i, len(tuples)) + } + } + if Matches(nil, "10.43.0.1", 443, "TCP") { + t.Errorf("nil tuples must never match") + } +} + +// Test A7 — serviceRef/serviceSelector imply the Service cluster FQDN(s); +// entity and unresolvable specs imply none. +func TestResolveDNSNames(t *testing.T) { + l := realFluxTopology() + one := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}}, l) + if len(one) != 1 || one[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN: got %v", one) + } + fan := ResolveDNSNames(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + }, l) + want := map[string]bool{"guestbook-ui.gitops-demo.svc.cluster.local": true, "helm-guestbook.gitops-demo.svc.cluster.local": true} + if len(fan) != 2 || !want[fan[0]] || !want[fan[1]] { + t.Errorf("selector FQDN fanout: got %v", fan) + } + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity implies no FQDN, got %v", got) + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "nope"}}, l); got != nil { + t.Errorf("unresolvable serviceRef implies no FQDN, got %v", got) + } +} + +// Test A6 — a nil Lister and any-port (no Ports) behave safely. +func TestResolve_Edges(t *testing.T) { + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + l := realFluxTopology() + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}}, l) + if !Matches(anyPort, "10.43.70.156", 443, "TCP") || !Matches(anyPort, "10.43.70.156", 8443, "TCP") { + t.Errorf("a serviceRef with no Ports should match any observed port on its IP") + } +} + +// TestServiceRef_TruthTable_KubeAPIServer is the full matrix for serviceRef +// resolution + matching, using the always-present default/kubernetes Service +// (the API server). Ports are the 443 the apiserver Service exposes; the +// resolved set is its ClusterIP plus its backing endpoint (the node IP on k3s). +func TestServiceRef_TruthTable_KubeAPIServer(t *testing.T) { + l := realFluxTopology() + const ( + clusterIP = "10.43.0.1" // default/kubernetes ClusterIP + endpoint = "192.168.0.191" // apiserver backing endpoint (node IP) + ) + + // Ported serviceRef: 443/TCP only. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {clusterIP, 443, "TCP", true, "ClusterIP on the exposed port"}, + {endpoint, 443, "TCP", true, "backing endpoint (apiserver node) on the exposed port"}, + {clusterIP, 443, "tcp", true, "protocol match is case-insensitive"}, + {clusterIP, 6443, "TCP", false, "wrong port (port-sensitive)"}, + {clusterIP, 443, "UDP", false, "wrong protocol"}, + {"10.43.54.190", 443, "TCP", false, "a different Service's ClusterIP"}, + {"10.42.0.1", 443, "TCP", false, "the node gateway is not this Service"}, + {"10.42.0.55", 443, "TCP", false, "an unrelated pod IP"}, + {"", 443, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- resolution edge cases --- + // No ports → any observed port on the resolved IPs matches. + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if !Matches(anyPort, clusterIP, 443, "TCP") || !Matches(anyPort, clusterIP, 6443, "TCP") || !Matches(anyPort, endpoint, 8443, "UDP") { + t.Error("a serviceRef with no ports must match any observed port/proto on its IPs") + } + // Unknown Service → nothing (never a match-all). + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "does-not-exist"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown Service must resolve to zero tuples, got %d", len(got)) + } + // Namespace only, no name → not a resolvable Service. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", ""}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("serviceRef with no name must resolve to zero tuples, got %d", len(got)) + } + // Wrong namespace for the same name → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kubernetes"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("kubernetes Service exists only in default; other namespace must resolve to zero, got %d", len(got)) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + // Headless Service (no ClusterIP) → resolves to its endpoints only. + l.services["default/headless"] = &ServiceInfo{Namespace: "default", Name: "headless", EndpointIPs: []string{"10.42.9.9"}} + hl := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "headless"}, Ports: tcp(80)}, l) + if !Matches(hl, "10.42.9.9", 80, "TCP") { + t.Error("a headless Service must resolve to its endpoint IPs") + } + // DNS names: the Service FQDN is implied. + dns := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if len(dns) != 1 || dns[0] != "kubernetes.default.svc.cluster.local" { + t.Errorf("serviceRef must imply the cluster FQDN, got %v", dns) + } +} + +// TestEntityHost_TruthTable is the full matrix for the "host" entity: it +// resolves to the node's InternalIP(s) plus the CNI gateway, and nothing else. +func TestEntityHost_TruthTable(t *testing.T) { + l := realFluxTopology() // hostIPs: node 192.168.0.191, gateway 10.42.0.1 + const ( + nodeIP = "192.168.0.191" + gateway = "10.42.0.1" + ) + + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {nodeIP, 10250, "TCP", true, "node InternalIP on the kubelet port"}, + {gateway, 10250, "TCP", true, "CNI gateway (masqueraded node traffic)"}, + {nodeIP, 10250, "tcp", true, "protocol case-insensitive"}, + {nodeIP, 9090, "TCP", false, "wrong port"}, + {nodeIP, 10250, "UDP", false, "wrong protocol"}, + {"10.42.0.55", 10250, "TCP", false, "a pod IP is not a host IP"}, + {"10.43.0.1", 10250, "TCP", false, "a ClusterIP is not a host IP"}, + {"", 10250, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- edge cases --- + // No ports → any observed port on the host IPs. + anyPort := Resolve(PeerSpec{Entity: EntityHost}, l) + if !Matches(anyPort, nodeIP, 22, "TCP") || !Matches(anyPort, gateway, 53, "UDP") { + t.Error("host entity with no ports must match any observed port on its IPs") + } + // "host" is case-insensitive. + if got := Resolve(PeerSpec{Entity: "HOST", Ports: tcp(10250)}, l); !Matches(got, nodeIP, 10250, "TCP") { + t.Error("the host entity name must be case-insensitive") + } + // An unknown entity resolves to nothing (never a match-all). + if got := Resolve(PeerSpec{Entity: "world", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown entity must resolve to zero tuples, got %d", len(got)) + } + // Empty entity string → nothing. + if got := Resolve(PeerSpec{Entity: "", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("empty entity must resolve to zero tuples, got %d", len(got)) + } + // host entity implies no DNS name (it is not a Service). + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity must imply no FQDN, got %v", got) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } +} diff --git a/pkg/objectcache/addr_ports_test.go b/pkg/objectcache/addr_ports_test.go new file mode 100644 index 0000000000..d1dd2ee88d --- /dev/null +++ b/pkg/objectcache/addr_ports_test.go @@ -0,0 +1,43 @@ +package objectcache + +import ( + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func np(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func TestExtractAddrPorts_ZeroPortIsALiteralNotAWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{np("TCP", 0), np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports, "a zero-port entry must not collapse the entry to fully open") + assert.Contains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 2) +} + +func TestExtractAddrPorts_AbsentStanzaIsTheOnlyWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }) + assert.Len(t, groups, 1) + assert.Nil(t, groups[0].Ports) +} + +func TestExtractAddrPorts_NilPortEntryContributesNothing(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.1.2.3"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP", Port: nil}, np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports) + assert.NotContains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 1) +} diff --git a/pkg/objectcache/containerprofilecache/containerprofilecache.go b/pkg/objectcache/containerprofilecache/containerprofilecache.go index 2a5394d18e..a3606ba3b5 100644 --- a/pkg/objectcache/containerprofilecache/containerprofilecache.go +++ b/pkg/objectcache/containerprofilecache/containerprofilecache.go @@ -16,6 +16,7 @@ import ( helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" "github.com/kubescape/node-agent/pkg/config" "github.com/kubescape/node-agent/pkg/metricsmanager" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/resourcelocks" @@ -58,6 +59,15 @@ type CachedContainerProfile struct { State *objectcache.ProfileState CallStackTree *callstackcache.CallStackSearchTree + // UsesServiceResolution is true when the profile declares serviceRef/ + // serviceSelector/entity neighbors, so its projected addresses depend on the + // live cluster view. ListerGen is the service-lister generation captured at + // projection time; the reconciler re-projects such entries when the + // generation advances (endpoint churn, or caches that filled after the + // initial projection). Non-resolving profiles keep the RV/spec fast-skip. + UsesServiceResolution bool + ListerGen int64 + ContainerName string PodName string Namespace string @@ -111,6 +121,10 @@ type ContainerProfileCacheImpl struct { containerLocks *resourcelocks.ResourceLocks storageClient storage.ProfileClient k8sObjectCache objectcache.K8sObjectCache + // serviceLister resolves serviceRef/serviceSelector/entity network + // neighbors to concrete IPs at projection time. nil = feature off (the + // profile projects unchanged), which is what every unit test leaves it as. + serviceLister networkpeer.Lister metricsManager metricsmanager.MetricsManager reconcileEvery time.Duration @@ -174,6 +188,22 @@ func NewContainerProfileCache(cfg config.Config, storageClient storage.ProfileCl return c } +// SetServiceLister installs the cluster view used to resolve +// serviceRef/serviceSelector/entity network neighbors at projection time. It +// is optional: when unset the projection leaves such neighbors unresolved. +func (c *ContainerProfileCacheImpl) SetServiceLister(l networkpeer.Lister) { + c.serviceLister = l +} + +// listerGen returns the current service-lister generation, or 0 when no lister +// is installed (unit tests, or the feature is off). +func (c *ContainerProfileCacheImpl) listerGen() int64 { + if c.serviceLister == nil { + return 0 + } + return c.serviceLister.Generation() +} + // refreshRPC calls fn with a context bounded by c.rpcBudget, enforcing a // per-call SLO so a slow API server cannot stall a full reconciler burst. func (c *ContainerProfileCacheImpl) refreshRPC(ctx context.Context, fn func(context.Context) error) error { @@ -579,9 +609,25 @@ func (c *ContainerProfileCacheImpl) buildEntry( } entry.CallStackTree = tree - // Project under the current spec. + // Project under the current spec, resolving any serviceRef/entity network + // neighbors to concrete IPs first. Record whether this profile depends on + // the live cluster view and the lister generation it was resolved against, + // so the reconciler re-projects it when that view changes. spec := c.snapshotSpec() - projected := Apply(spec, userMerged, tree) + // Resolution-dependent iff the profile has its OWN serviceRef/serviceSelector/ + // entity neighbors OR the synthetic host peer is injected: the host peer + // resolves against the Node lister, which main.go hands over WITHOUT blocking + // on cache sync, so a host-only profile projected before the lister filled in + // must still be re-projected once it does (else HostIPs() stays empty and node + // traffic keeps alerting). Computed before the injection so a profile with no + // own neighbors is keyed on the injection flag, not the synthetic peer. + entry.UsesServiceResolution = networkpeer.HasServiceNeighbors(userMerged) || !c.cfg.AlertOnHostPeers + if !c.cfg.AlertOnHostPeers { + userMerged = networkpeer.WithHostPeer(userMerged) + } + entry.ListerGen = c.listerGen() + projected := Apply(spec, networkpeer.WithResolvedServiceNeighbors(userMerged, c.serviceLister), tree) + projected.ResolvedGen = entry.ListerGen entry.Projected = projected entry.SpecHash = projected.SpecHash diff --git a/pkg/objectcache/containerprofilecache/projection_apply.go b/pkg/objectcache/containerprofilecache/projection_apply.go index f0de872a52..f9e75c6f1c 100644 --- a/pkg/objectcache/containerprofilecache/projection_apply.go +++ b/pkg/objectcache/containerprofilecache/projection_apply.go @@ -51,6 +51,10 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.Execs = projectField(s.Execs, execsPaths, true) pcp.ExecsByPath = extractExecsByPath(cp) + pcp.Namespace = cp.Namespace + pcp.IngressPeers = extractIngressPeers(cp) + pcp.EgressPeers = extractEgressPeers(cp) + endpointPaths := extractEndpointPaths(cp) pcp.Endpoints = projectField(s.Endpoints, endpointPaths, true) @@ -63,6 +67,9 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.IngressDomains = projectField(s.IngressDomains, extractIngressDomains(cp), false) pcp.IngressAddresses = projectField(s.IngressAddresses, extractIngressAddresses(cp), false) + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + return pcp } @@ -260,3 +267,40 @@ func extractIngressAddresses(cp *v1beta1.ContainerProfile) []string { } return addrs } + +// extractIngressPeers / extractEgressPeers carry the label selectors of each +// network-neighbor entry so cp.was_selector_in_{ingress,egress} can match a +// peer by identity. Only entries that actually declare a podSelector are kept. +func extractIngressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Ingress) +} + +func extractEgressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Egress) +} + +func extractPeers(neighbors []v1beta1.NetworkNeighbor) []objectcache.PeerSelector { + var peers []objectcache.PeerSelector + for i := range neighbors { + n := &neighbors[i] + if n.PodSelector == nil { + continue + } + var ports map[string]struct{} + if len(n.Ports) > 0 { + ports = make(map[string]struct{}, len(n.Ports)) + for _, p := range n.Ports { + if p.Port == nil { + continue + } + ports[objectcache.PortKey(string(p.Protocol), *p.Port)] = struct{}{} + } + } + peers = append(peers, objectcache.PeerSelector{ + PodSelector: n.PodSelector, + NamespaceSelector: n.NamespaceSelector, + Ports: ports, + }) + } + return peers +} diff --git a/pkg/objectcache/containerprofilecache/projection_golden_test.go b/pkg/objectcache/containerprofilecache/projection_golden_test.go index 750d120bcb..1450907e29 100644 --- a/pkg/objectcache/containerprofilecache/projection_golden_test.go +++ b/pkg/objectcache/containerprofilecache/projection_golden_test.go @@ -65,6 +65,8 @@ type projectionGolden struct { EgressAddresses objectcache.ProjectedField `json:"egressAddresses"` IngressDomains objectcache.ProjectedField `json:"ingressDomains"` IngressAddresses objectcache.ProjectedField `json:"ingressAddresses"` + IngressPeers []objectcache.PeerSelector `json:"ingressPeers"` + EgressPeers []objectcache.PeerSelector `json:"egressPeers"` ExecsByPath map[string][][]string `json:"execsByPath"` PolicyByRuleId map[string]v1beta1.RulePolicy `json:"policyByRuleId"` CallStacks []callStackSummary `json:"callStacks"` @@ -93,6 +95,8 @@ func toGolden(pcp *objectcache.ProjectedContainerProfile, tree *callstackcache.C EgressAddresses: pcp.EgressAddresses, IngressDomains: pcp.IngressDomains, IngressAddresses: pcp.IngressAddresses, + IngressPeers: pcp.IngressPeers, + EgressPeers: pcp.EgressPeers, ExecsByPath: pcp.ExecsByPath, PolicyByRuleId: pcp.PolicyByRuleId, } @@ -236,9 +240,12 @@ func networkProfile() *v1beta1.ContainerProfile { Ingress: []v1beta1.NetworkNeighbor{ {Identifier: "in-a", DNS: "old.internal", DNSNames: []string{"a.internal", "b.internal"}, IPAddresses: []string{"192.168.1.10", "192.168.0.0/16"}}, {Identifier: "in-b", IPAddresses: []string{wild}}, + {Identifier: "in-c", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "redis-client"}}}, + {Identifier: "in-d", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "probe"}}, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "monitoring"}}}, }, Egress: []v1beta1.NetworkNeighbor{ {Identifier: "eg-a", DNSNames: []string{"c.example.com"}, IPAddress: "203.0.113.7", IPAddresses: []string{"203.0.113.0/24", wild}}, + {Identifier: "eg-b", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "upstream"}}}, }, }, } diff --git a/pkg/objectcache/containerprofilecache/reconciler.go b/pkg/objectcache/containerprofilecache/reconciler.go index bd7517c728..e1002b6d93 100644 --- a/pkg/objectcache/containerprofilecache/reconciler.go +++ b/pkg/objectcache/containerprofilecache/reconciler.go @@ -23,6 +23,7 @@ import ( "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/utils" @@ -418,9 +419,14 @@ func (c *ContainerProfileCacheImpl) refreshOneEntry(ctx context.Context, id stri if spec := c.snapshotSpec(); spec != nil { currentSpecHash = spec.Hash } + // serviceRef/entity profiles must also re-project when the cluster view + // changed since they were resolved (endpoint churn, or caches that filled + // after projection). Non-resolving profiles ignore the lister generation and + // keep the cheap RV/spec fast-skip. if rvsMatchCP(cp, e.RV) && rvsMatchCP(userDefinedCP, e.UserCPRV) && - e.SpecHash == currentSpecHash { + e.SpecHash == currentSpecHash && + (!e.UsesServiceResolution || e.ListerGen == c.listerGen()) { return } @@ -489,28 +495,42 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( // Project under the current spec. spec := c.snapshotSpec() + // Read gen BEFORE resolving so a concurrent Bump() invalidates this projection. + gen := c.listerGen() + // Resolution-dependent iff the profile has its OWN serviceRef/serviceSelector/ + // entity neighbors OR the host peer is injected — the latter resolves against + // the Node lister (handed over without blocking on sync), so a host-only + // profile projected before the lister filled in must be re-projected once it + // does, else HostIPs() stays empty and node traffic keeps alerting. + usesResolution := networkpeer.HasServiceNeighbors(projected) || !c.cfg.AlertOnHostPeers + if !c.cfg.AlertOnHostPeers { + projected = networkpeer.WithHostPeer(projected) + } applyStart := time.Now() - projectedCP := Apply(spec, projected, tree) + projectedCP := Apply(spec, networkpeer.WithResolvedServiceNeighbors(projected, c.serviceLister), tree) + projectedCP.ResolvedGen = gen if c.cfg.ProfileProjection.DetailedMetricsEnabled { c.metricsManager.ObserveProjectionApplyDuration(time.Since(applyStart)) c.observeMemoryMetrics(projected, projectedCP) } newEntry := &CachedContainerProfile{ - Projected: projectedCP, - SpecHash: projectedCP.SpecHash, - State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, - CallStackTree: tree, - ContainerName: prev.ContainerName, - PodName: prev.PodName, - Namespace: prev.Namespace, - PodUID: podUID, - WorkloadID: prev.WorkloadID, - CPName: prev.CPName, - WorkloadName: prev.WorkloadName, - RV: rvOfCP(cp), - UserCPRV: rvOfCP(userDefinedCP), - terminatedSeenAt: prev.terminatedSeenAt, + Projected: projectedCP, + SpecHash: projectedCP.SpecHash, + UsesServiceResolution: usesResolution, + ListerGen: gen, + State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, + CallStackTree: tree, + ContainerName: prev.ContainerName, + PodName: prev.PodName, + Namespace: prev.Namespace, + PodUID: podUID, + WorkloadID: prev.WorkloadID, + CPName: prev.CPName, + WorkloadName: prev.WorkloadName, + RV: rvOfCP(cp), + UserCPRV: rvOfCP(userDefinedCP), + terminatedSeenAt: prev.terminatedSeenAt, } if userDefinedCP != nil { // The user-authored CP is authoritative and complete by definition (no diff --git a/pkg/objectcache/containerprofilecache/resolvedgen_test.go b/pkg/objectcache/containerprofilecache/resolvedgen_test.go new file mode 100644 index 0000000000..7abdc844d1 --- /dev/null +++ b/pkg/objectcache/containerprofilecache/resolvedgen_test.go @@ -0,0 +1,48 @@ +package containerprofilecache + +import ( + "strconv" + "testing" + + "github.com/kubescape/node-agent/pkg/networkpeer" + "github.com/kubescape/node-agent/pkg/objectcache" +) + +// genLister is a cluster view whose generation the test drives directly. +type genLister struct{ gen int64 } + +func (g *genLister) ServiceByName(string, string) (*networkpeer.ServiceInfo, bool) { return nil, false } +func (g *genLister) ServicesByLabels(map[string]string, map[string]string) []*networkpeer.ServiceInfo { + return nil +} +func (g *genLister) HostIPs() []string { return nil } +func (g *genLister) Generation() int64 { return g.gen } + +// TestProjectedResolvedGenFeedsCacheKey: the CEL result cache keys on the +// projected profile's SpecHash+SyncChecksum+ResolvedGen. Re-resolving against a +// moved cluster view changes neither of the first two — an authored profile +// carries no SyncChecksum at all — so without ResolvedGen a result computed +// before the informers filled (e.g. "this address is not in egress") would be +// served from cache forever, and the profile's own re-projection would never +// take effect. +func TestProjectedResolvedGenFeedsCacheKey(t *testing.T) { + l := &genLister{gen: 7} + c := &ContainerProfileCacheImpl{} + c.SetServiceLister(l) + + if got := c.listerGen(); got != 7 { + t.Fatalf("listerGen: got %d want 7", got) + } + + key := func(p *objectcache.ProjectedContainerProfile) string { + return p.SpecHash + "|" + p.SyncChecksum + "|" + strconv.FormatInt(p.ResolvedGen, 10) + } + before := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + l.gen = 8 + after := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + if key(before) == key(after) { + t.Errorf("cache key must change when the profile is re-resolved against a moved cluster view") + } +} diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json index 3b833c5cfd..bda7733814 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json @@ -78,6 +78,41 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "redis-client" + } + }, + "NamespaceSelector": null, + "Ports": null + }, + { + "PodSelector": { + "matchLabels": { + "app": "probe" + } + }, + "NamespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + }, + "Ports": null + } + ], + "egressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "upstream" + } + }, + "NamespaceSelector": null, + "Ports": null + } + ], "execsByPath": null, "policyByRuleId": null, "callStacks": null diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json index dccf5126c1..71fb76abb4 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json @@ -103,6 +103,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json index a343f73b67..13eef20a98 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json @@ -97,6 +97,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index 8d452c7a36..35b5f402cc 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -1,10 +1,27 @@ package objectcache import ( + "strconv" + "strings" + "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// PeerSelector carries a single network-neighbor entry's identity selectors +// (podSelector + namespaceSelector) through the projection so the +// cp.was_selector_in_{ingress,egress} CEL helpers can resolve a runtime peer +// IP to a pod and match it by LABEL rather than by (volatile) IP. The address +// surfaces (Ingress/EgressAddresses) still carry the ipAddress/CIDR form for +// the was_address_in_* helpers; these are complementary. +type PeerSelector struct { + PodSelector *metav1.LabelSelector + NamespaceSelector *metav1.LabelSelector + // Ports mirrors AddrPortGroup.Ports: nil means the neighbor declared no ports stanza and matches any port; a non-empty map matches only its literal (protocol, port) keys. + Ports map[string]struct{} +} + // PathMatcher is implemented by the trie-based matchers in containerprofilecache. type PathMatcher interface { HasMatch(s string) bool @@ -41,9 +58,57 @@ type FieldSpec struct { SuffixMatcher PathMatcher } +// AddrPortGroup pairs one neighbor entry's addresses with its allowed ports. +// Ports == nil means the neighbor declared no ports stanza (indistinguishable +// from an empty one after a storage round-trip) and matches any port; a +// non-empty map matches only its literal (protocol, port) keys. +type AddrPortGroup struct { + Addrs []string + Ports map[string]struct{} +} + +func PortKey(protocol string, port int32) string { + return strings.ToUpper(protocol) + "-" + strconv.Itoa(int(port)) +} + +func ExtractAddrPorts(neighbors []v1beta1.NetworkNeighbor) []AddrPortGroup { + var groups []AddrPortGroup + for i := range neighbors { + n := &neighbors[i] + var addrs []string + if n.IPAddress != "" { + addrs = append(addrs, n.IPAddress) + } + addrs = append(addrs, n.IPAddresses...) + if len(addrs) == 0 { + continue + } + // The only port wildcard is an absent (or empty — protobuf cannot tell + // them apart) ports stanza. A listed entry always restricts: an explicit + // port (0 included) is a literal, a nil port contributes nothing. + ports := make(map[string]struct{}, len(n.Ports)) + for _, p := range n.Ports { + if p.Port == nil { + continue + } + ports[PortKey(string(p.Protocol), *p.Port)] = struct{}{} + } + if len(n.Ports) == 0 { + ports = nil + } + groups = append(groups, AddrPortGroup{Addrs: addrs, Ports: ports}) + } + return groups +} + // ProjectedContainerProfile is the cache-resident compact form. Pure node-agent // internal type; never serialized. Replaces *v1beta1.ContainerProfile in the cache. type ProjectedContainerProfile struct { + // Namespace is the profiled workload's own namespace; a peer entry whose + // NamespaceSelector is nil matches only peers in this namespace (the learned + // encoding and the NetworkPolicyPeer semantic for an absent namespaceSelector). + Namespace string + Opens ProjectedField Execs ProjectedField Endpoints ProjectedField @@ -54,6 +119,18 @@ type ProjectedContainerProfile struct { IngressDomains ProjectedField IngressAddresses ProjectedField + // IngressPeers / EgressPeers carry the podSelector+namespaceSelector of each + // network-neighbor entry (dropped by the address/domain projection) so the + // cp.was_selector_in_{ingress,egress} helpers can match a runtime peer by + // label. Always projected in full (not gated by a rule surface) since they + // are small and only populated when the profile actually declares selectors. + IngressPeers []PeerSelector + EgressPeers []PeerSelector + + // IngressAddrPorts / EgressAddrPorts group each neighbor's addresses with its ports for was_address_port_protocol_in_*. + IngressAddrPorts []AddrPortGroup + EgressAddrPorts []AddrPortGroup + // ExecsByPath carries the per-Path Args slices from cp.Spec.Execs so // downstream consumers (e.g. dynamicpathdetector.CompareExecArgs used // by R0040 in node-agent#807) can run wildcard-aware argv matching @@ -67,7 +144,14 @@ type ProjectedContainerProfile struct { // constraint" (back-compat for pre-projection profiles). ExecsByPath map[string][][]string - SpecHash string + SpecHash string + // ResolvedGen is the cluster-view generation the profile's serviceRef/ + // serviceSelector/entity neighbors were resolved against. It participates in + // the CEL result-cache key: re-projecting against a moved cluster view + // changes the projected addresses without touching SpecHash or SyncChecksum, + // so a result cached before the informers filled would otherwise be served + // forever. Zero for profiles that resolve nothing. + ResolvedGen int64 SyncChecksum string PolicyByRuleId map[string]v1beta1.RulePolicy CallStackTree *callstackcache.CallStackSearchTree diff --git a/pkg/objectcache/v1/mock.go b/pkg/objectcache/v1/mock.go index 789eccb9ec..fa4d324d05 100644 --- a/pkg/objectcache/v1/mock.go +++ b/pkg/objectcache/v1/mock.go @@ -193,9 +193,40 @@ func (r *RuleObjectCacheMock) GetProjectedContainerProfile(containerID string) * } } + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + pcp.EgressPeers = extractMockPeers(cp.Spec.Egress) + pcp.IngressPeers = extractMockPeers(cp.Spec.Ingress) + pcp.Namespace = cp.Namespace + return pcp } +func extractMockPeers(neighbors []v1beta1.NetworkNeighbor) []objectcache.PeerSelector { + var peers []objectcache.PeerSelector + for i := range neighbors { + if neighbors[i].PodSelector == nil { + continue + } + var ports map[string]struct{} + if len(neighbors[i].Ports) > 0 { + ports = make(map[string]struct{}, len(neighbors[i].Ports)) + for _, p := range neighbors[i].Ports { + if p.Port == nil { + continue + } + ports[objectcache.PortKey(string(p.Protocol), *p.Port)] = struct{}{} + } + } + peers = append(peers, objectcache.PeerSelector{ + PodSelector: neighbors[i].PodSelector, + NamespaceSelector: neighbors[i].NamespaceSelector, + Ports: ports, + }) + } + return peers +} + func (r *RuleObjectCacheMock) SetProjectionSpec(spec objectcache.RuleProjectionSpec) { r.projectionSpecMu.Lock() r.projectionSpec = spec diff --git a/pkg/rulemanager/cel/libraries/cache/function_cache.go b/pkg/rulemanager/cel/libraries/cache/function_cache.go index ba07eafcd3..22af6beed6 100644 --- a/pkg/rulemanager/cel/libraries/cache/function_cache.go +++ b/pkg/rulemanager/cel/libraries/cache/function_cache.go @@ -2,6 +2,7 @@ package cache import ( "fmt" + "strconv" "strings" "time" @@ -101,8 +102,10 @@ func HashForContainerProfile(oc objectcache.ObjectCache) func([]ref.Val) string } // Include SyncChecksum so the key changes when profile content is updated // under the same projection spec, preventing stale cached results after - // the profile learns new paths/execs/etc. - return pcp.SpecHash + "|" + pcp.SyncChecksum + // the profile learns new paths/execs/etc. ResolvedGen covers the same + // hazard for serviceRef/entity neighbors, whose projected addresses move + // with the cluster view while both other components stay put. + return pcp.SpecHash + "|" + pcp.SyncChecksum + "|" + strconv.FormatInt(pcp.ResolvedGen, 10) } } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go index 58058c2aed..1a78b2bcef 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go @@ -83,6 +83,9 @@ type containerProfileNetworkFuncSpec struct { arity int // call invokes the shared implementation method on l. call func(l *containerProfileNetworkLibrary, args []ref.Val) ref.Val + // noCache bypasses the functionCache: a map argument has no stable scalar + // cache key, and the selector match is cheap (O(selectors)). + noCache bool } var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ @@ -140,6 +143,26 @@ var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ return l.wasAddressPortProtocolInIngress(a[0], a[1], a[2], a[3]) }, }, + { + name: "was_selector_in_egress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType), cel.IntType, cel.StringType}, + resultType: cel.BoolType, + arity: 5, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInEgress(a[0], a[1], a[2], a[3], a[4]) + }, + noCache: true, + }, + { + name: "was_selector_in_ingress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType), cel.IntType, cel.StringType}, + resultType: cel.BoolType, + arity: 5, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInIngress(a[0], a[1], a[2], a[3], a[4]) + }, + noCache: true, + }, } // declarationsWithPrefix builds the cel.FunctionOpt map for every function in @@ -165,6 +188,9 @@ func (l *containerProfileNetworkLibrary) declarationsWithPrefix(namePrefix, over if l.detailedMetrics && l.metrics != nil { l.metrics.IncHelperCall(fullName) } + if spec.noCache { + return cache.ConvertProfileNotAvailableErrToBool(spec.call(l, values), false) + } wrapperFunc := func(args ...ref.Val) ref.Val { return spec.call(l, args) } @@ -270,6 +296,9 @@ func (e *containerProfileNetworkCostEstimator) EstimateCallCost(function, overlo case "cp.is_domain_in_egress", "cp.is_domain_in_ingress": // Cache lookup + O(n) list iteration + O(m) slice.Contains on DNS names per entry cost = 35 + case "cp.was_selector_in_egress", "cp.was_selector_in_ingress": + // O(selectors) label-set match per peer entry + cost = 30 case "cp.was_address_port_protocol_in_egress", "cp.was_address_port_protocol_in_ingress": // Cache lookup + O(n) address search + O(p) nested port/protocol matching cost = 45 diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/cost_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/cost_test.go index faa409bbec..91665de611 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/cost_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/cost_test.go @@ -13,3 +13,21 @@ func TestNetworkCostEstimator_NilForUnknownFunction(t *testing.T) { assert.NotNil(t, est.EstimateCallCost("cp.was_address_in_egress", "", nil, nil), "known function must return a cost estimate") } + +func TestNetworkCostEstimator_CoversEveryDeclaredFunction(t *testing.T) { + est := &containerProfileNetworkCostEstimator{} + for _, spec := range containerProfileNetworkFuncSpecs { + assert.NotNil(t, est.EstimateCallCost("cp."+spec.name, "", nil, nil), + "declared function cp.%s must have a cost estimate", spec.name) + } +} + +func TestLegacyCostEstimator_TranslatesPrefix(t *testing.T) { + est := &legacyNetworkCostEstimator{inner: &containerProfileNetworkCostEstimator{}, legacyPrefix: "nn.", canonicalPrefix: "cp."} + for _, spec := range containerProfileNetworkFuncSpecs { + got := est.EstimateCallCost("nn."+spec.name, "", nil, nil) + want := (&containerProfileNetworkCostEstimator{}).EstimateCallCost("cp."+spec.name, "", nil, nil) + assert.Equal(t, want, got, "nn.%s must cost the same as cp.%s", spec.name, spec.name) + } + assert.Nil(t, est.EstimateCallCost("nn.does_not_exist", "", nil, nil)) +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go index e515a5fd73..ca07412dc2 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go @@ -12,6 +12,7 @@ import ( "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -32,6 +33,7 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { }) nn := &v1beta1.ContainerProfile{} + nn.Namespace = "prod" nn.Spec = v1beta1.ContainerProfileSpec{ Egress: []v1beta1.NetworkNeighbor{ { @@ -72,6 +74,10 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { }, }, }, + { + Identifier: "db-clients", + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "db-client"}}, + }, }, Ingress: []v1beta1.NetworkNeighbor{ { @@ -101,6 +107,11 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { }, }, }, + { + Identifier: "frontends", + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "frontend"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "web"}}, + }, }, } objCache.SetContainerProfile(nn) @@ -209,28 +220,24 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent egress address with port and protocol", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent ingress address with port and protocol", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing address and port", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "UDP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing ingress address and port", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 8080, "UDP")`, - expectedResult: true, + expectedResult: false, }, { name: "Complex network check with port and protocol - egress", @@ -243,9 +250,43 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: both sides match on address only → true. name: "Mixed valid and invalid port protocol checks", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "TCP") && cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, + expectedResult: false, + }, + { + name: "Check egress selector peer", + expression: `cp.was_selector_in_egress(containerID, "prod", {"app": "db-client"}, 443, "TCP")`, + expectedResult: true, + }, + { + name: "Check egress selector peer wrong labels", + expression: `cp.was_selector_in_egress(containerID, "prod", {"app": "attacker"}, 443, "TCP")`, + expectedResult: false, + }, + { + name: "Check egress selector peer unresolved namespace", + expression: `cp.was_selector_in_egress(containerID, "", {"app": "db-client"}, 443, "TCP")`, + expectedResult: false, + }, + { + name: "Check ingress selector peer with namespace scope", + expression: `cp.was_selector_in_ingress(containerID, "web", {"app": "frontend"}, 443, "TCP")`, + expectedResult: true, + }, + { + name: "Check ingress selector peer wrong namespace", + expression: `cp.was_selector_in_ingress(containerID, "prod", {"app": "frontend"}, 443, "TCP")`, + expectedResult: false, + }, + { + name: "Selector direction isolation - egress peer not in ingress", + expression: `cp.was_selector_in_ingress(containerID, "prod", {"app": "db-client"}, 443, "TCP")`, + expectedResult: false, + }, + { + name: "Combined address and selector check", + expression: `cp.was_address_in_egress(containerID, "8.8.8.8") && cp.was_selector_in_egress(containerID, "prod", {"app": "db-client"}, 443, "TCP")`, expectedResult: true, }, } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/legacy_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/legacy_test.go index 391c512ad2..b8ab9ab4bf 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/legacy_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/legacy_test.go @@ -12,6 +12,7 @@ import ( objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -59,6 +60,7 @@ func TestLegacyNNMatchesCP(t *testing.T) { }) profile := &v1beta1.ContainerProfile{} + profile.Namespace = "redis" profile.Spec = v1beta1.ContainerProfileSpec{ Egress: []v1beta1.NetworkNeighbor{ { @@ -68,6 +70,10 @@ func TestLegacyNNMatchesCP(t *testing.T) { {Name: "tcp-80", Protocol: "TCP", Port: ptr.To(int32(80))}, }, }, + { + Identifier: "redis-clients", + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "redis-client"}}, + }, }, Ingress: []v1beta1.NetworkNeighbor{ { @@ -77,6 +83,10 @@ func TestLegacyNNMatchesCP(t *testing.T) { {Name: "tcp-8080", Protocol: "TCP", Port: ptr.To(int32(8080))}, }, }, + { + Identifier: "lb-clients", + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "lb-client"}}, + }, }, } objCache.SetContainerProfile(profile) @@ -138,6 +148,24 @@ func TestLegacyNNMatchesCP(t *testing.T) { nn: `nn.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 8080, "TCP")`, want: true, }, + { + name: "was_selector_in_egress", + cp: `cp.was_selector_in_egress(containerID, "redis", {"app": "redis-client"}, 443, "TCP")`, + nn: `nn.was_selector_in_egress(containerID, "redis", {"app": "redis-client"}, 443, "TCP")`, + want: true, + }, + { + name: "was_selector_in_ingress", + cp: `cp.was_selector_in_ingress(containerID, "redis", {"app": "lb-client"}, 443, "TCP")`, + nn: `nn.was_selector_in_ingress(containerID, "redis", {"app": "lb-client"}, 443, "TCP")`, + want: true, + }, + { + name: "was_selector_in_egress (miss)", + cp: `cp.was_selector_in_egress(containerID, "redis", {"app": "unknown"}, 443, "TCP")`, + nn: `nn.was_selector_in_egress(containerID, "redis", {"app": "unknown"}, 443, "TCP")`, + want: false, + }, } for _, tc := range testCases { diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 3d97e85f2e..7aae6876f6 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -2,6 +2,7 @@ package containerprofilenetwork import ( "net" + "reflect" "strings" "github.com/google/cel-go/common/types" @@ -10,6 +11,8 @@ import ( "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" "github.com/kubescape/node-agent/pkg/rulemanager/profilehelper" "github.com/kubescape/storage/pkg/registry/file/networkmatch" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) // matchIPField is the wildcard-aware adapter from the projection layer's @@ -59,6 +62,29 @@ func matchIPField(field *objectcache.ProjectedField, observed string) bool { return networkmatch.MatchIP(entries, observed) } +// matchAddrPort reports whether observed (address, protocol, port) falls within +// any single neighbor entry: its addresses match AND the entry allows the port +// (nil Ports = no ports stanza = any port; a populated map matches literal keys only). +func matchAddrPort(groups []objectcache.AddrPortGroup, address, protocol string, port int32) bool { + if address == "" { + return false + } + key := objectcache.PortKey(protocol, port) + for i := range groups { + g := &groups[i] + if !networkmatch.MatchIP(g.Addrs, address) { + continue + } + if g.Ports == nil { + return true + } + if _, ok := g.Ports[key]; ok { + return true + } + } + return false +} + func matchDNSField(field *objectcache.ProjectedField, observed string) bool { if observed == "" || field == nil { return false @@ -171,9 +197,6 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if !ok { return types.MaybeNoSuchOverloadErr(address) } - // port/protocol projection (AddressPortsByAddr) is out of scope for the - // projection-v1 layer upstream landed; matchers degrade to address-only. - // Wildcards remain enforced via matchIPField. portInt, ok := port.Value().(int64) if !ok { return types.MaybeNoSuchOverloadErr(port) @@ -181,14 +204,15 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.EgressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.EgressAddrPorts, addressStr, protocolStr, int32(portInt))) } func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(containerID, address, port, protocol ref.Val) ref.Val { @@ -210,12 +234,132 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(contain if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.IngressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.IngressAddrPorts, addressStr, protocolStr, int32(portInt))) +} + +// namespaceSelectorMatches follows NetworkPolicy semantics. An OMITTED (nil) +// selector means the profile's OWN namespace (peerNs == profileNs): a peer is +// same-namespace unless it says otherwise, so a pod that merely copies an +// allowlisted peer's labels in another namespace does not match. An explicit +// EMPTY selector ({}) is cluster-wide, opt-in (LabelSelectorAsSelector maps {} +// to Everything). An explicit selector with labels matches the peer namespace by +// its kubernetes.io/metadata.name label (e.g. kube-system). Unparseable fails closed. +func namespaceSelectorMatches(sel *metav1.LabelSelector, peerNs, profileNs string) bool { + if sel == nil { + return peerNs == profileNs + } + s, err := metav1.LabelSelectorAsSelector(sel) + if err != nil { + return false + } + return s.Matches(labels.Set{"kubernetes.io/metadata.name": peerNs}) +} + +// wasSelectorInPeers reports whether the peer identified by (podLabels, ns) +// matches any peer entry's podSelector AND its namespaceSelector. An empty +// podSelector matches NOTHING (fail closed → the peer alerts), the opposite of +// NetworkPolicy's match-all: an allowlist entry must name what it permits. +func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs, protocol string, port int32) bool { + key := objectcache.PortKey(protocol, port) + for i := range peers { + peer := &peers[i] + if peer.PodSelector == nil || + (len(peer.PodSelector.MatchLabels) == 0 && len(peer.PodSelector.MatchExpressions) == 0) { + continue + } + ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + continue + } + if !ps.Matches(podLabels) || !namespaceSelectorMatches(peer.NamespaceSelector, ns, profileNs) { + continue + } + if peer.Ports == nil { + return true + } + if _, ok := peer.Ports[key]; ok { + return true + } + } + return false +} + +func (l *containerProfileNetworkLibrary) wasSelectorInIngress(containerID, namespace, podLabels, port, protocol ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, port, protocol, true) +} + +func (l *containerProfileNetworkLibrary) wasSelectorInEgress(containerID, namespace, podLabels, port, protocol ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, port, protocol, false) +} + +// wasSelectorIn reports whether the runtime peer — identified by the namespace +// and pod labels that Inspektor Gadget's kubeipresolver stamps onto the network +// event — matches any of the profile's ingress-or-egress peer selectors. +// +// Matching on the peer's identity (namespace + labels) rather than its IP is the +// whole point: it is stable across pod IP churn AND works across nodes, because +// kubeipresolver resolves the peer against a cluster-wide pod inventory before +// the event ever reaches CEL. There is deliberately no IP→pod lookup here — that +// would reintroduce a dependency on node-agent's node-local pod cache, which is +// exactly what breaks cross-node peers. +func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, podLabels, port, protocol ref.Val, ingress bool) ref.Val { + if l.objectCache == nil { + return types.NewErr("objectCache is nil") + } + containerIDStr, ok := containerID.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(containerID) + } + nsStr, ok := namespace.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(namespace) + } + portInt, ok := port.Value().(int64) + if !ok { + return types.MaybeNoSuchOverloadErr(port) + } + protocolStr, ok := protocol.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(protocol) + } + if nsStr == "" { + // The peer did not resolve to a pod (external IP, or the resolver had no + // inventory entry): a nil peer never satisfies a selector — it alerts. + return types.Bool(false) + } + peerLabels := refValToStringMap(podLabels) + cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) + if err != nil { + return cache.NewProfileNotAvailableErr("%v", err) + } + peers := cp.EgressPeers + if ingress { + peers = cp.IngressPeers + } + if len(peers) == 0 { + return types.Bool(false) + } + return types.Bool(wasSelectorInPeers(peers, labels.Set(peerLabels), nsStr, cp.Namespace, protocolStr, int32(portInt))) +} + +// refValToStringMap converts a CEL map argument to a Go map[string]string. A nil +// or non-map value yields nil (treated as "peer has no labels"). +func refValToStringMap(v ref.Val) map[string]string { + if v == nil { + return nil + } + native, err := v.ConvertToNative(reflect.TypeOf(map[string]string(nil))) + if err != nil { + return nil + } + m, _ := native.(map[string]string) + return m } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go index 10321073cc..e94ac732e7 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go @@ -99,22 +99,20 @@ func TestWasAddressPortProtocolInEgress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "192.168.1.100", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "192.168.1.100", port: 80, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -235,22 +233,20 @@ func TestWasAddressPortProtocolInIngress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "172.16.0.10", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "172.16.0.10", port: 8080, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -405,14 +401,15 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { functionCache: cache.NewFunctionCache(cache.DefaultFunctionCacheConfig()), } - // v1 degradation: address-only matching; nil port in profile no longer checked. + // A listed entry with a nil port contributes nothing: the only port + // wildcard is an ABSENT ports stanza, so these addresses stay restricted. result := lib.wasAddressPortProtocolInEgress( types.String("test-container-id"), types.String("192.168.1.100"), types.Int(80), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) result = lib.wasAddressPortProtocolInIngress( types.String("test-container-id"), @@ -420,5 +417,5 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { types.Int(8080), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go new file mode 100644 index 0000000000..b2a42df60b --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go @@ -0,0 +1,112 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/google/cel-go/common/types" + "github.com/kubescape/node-agent/pkg/objectcache" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func port(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func evalEgressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInEgress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func evalIngressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInIngress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func TestWasAddressPortProtocolInEgress_PortWildcard(t *testing.T) { + noPorts := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }, nil) + assert.Equal(t, types.Bool(true), evalEgressPort(noPorts, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(noPorts, "1.1.1.1", 8080, "TCP")) + + zeroPort := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(zeroPort, "93.184.216.34", 8080, "TCP"), + "an explicit port 0 is a literal, not a wildcard: only an absent ports stanza opens the entry") +} + +func TestWasAddressPortProtocolInIngress_Symmetric(t *testing.T) { + lib := buildLibWithContainer(t, nil, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"172.16.0.0/12"}, Ports: []v1beta1.NetworkPort{port("TCP", 6379)}}, + }) + assert.Equal(t, types.Bool(true), evalIngressPort(lib, "172.16.5.9", 6379, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "172.16.5.9", 5432, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "10.0.0.1", 6379, "TCP")) +} + +func TestWasAddressPortProtocolInEgress_ZeroPortIsNotAWildcard(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 53, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_MixedZeroPortKeepsEveryProtocolRestricted(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{port("TCP", 0), port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 9999, "TCP"), + "an explicit {TCP,0} entry no longer opens TCP: wildcard is expressed only by omitting the ports stanza") + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 54, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_NilPortEntryContributesNothing(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP"}, port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 8080, "TCP")) + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) +} + +func TestMatchAddrPort_TruthTable(t *testing.T) { + groups := objectcache.ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.0.0/8"}, Ports: []v1beta1.NetworkPort{port("TCP", 443)}}, + {IPAddresses: []string{"192.168.1.5"}}, + {IPAddresses: []string{"172.16.0.9"}, Ports: []v1beta1.NetworkPort{port("UDP", 53)}}, + {IPAddresses: []string{"9.9.9.9"}, Ports: []v1beta1.NetworkPort{port("tcp", 8443)}}, + }) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {"10.1.2.3", 443, "TCP", true, "CIDR member on the declared port"}, + {"10.1.2.3", 443, "tcp", true, "observed protocol match is case-insensitive"}, + {"10.1.2.3", 443, "Tcp", true, "mixed-case protocol still matches"}, + {"10.1.2.3", 80, "TCP", false, "wrong port (port-sensitive)"}, + {"10.1.2.3", 443, "UDP", false, "wrong protocol"}, + {"192.168.1.5", 9999, "TCP", true, "absent ports stanza = any port"}, + {"192.168.1.5", 53, "udp", true, "absent ports stanza = any protocol too"}, + {"172.16.0.9", 53, "UDP", true, "literal IP + UDP port"}, + {"172.16.0.9", 53, "TCP", false, "protocol-sensitive even on the right port"}, + {"9.9.9.9", 8443, "TCP", true, "lowercase profile protocol is normalised at build"}, + {"8.8.8.8", 443, "TCP", false, "address absent from every group"}, + {"", 443, "TCP", false, "empty address never matches"}, + } + for _, c := range grid { + if got := matchAddrPort(groups, c.ip, c.proto, c.port); got != c.want { + t.Errorf("matchAddrPort(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + assert.False(t, matchAddrPort(nil, "10.1.2.3", "TCP", 443), "nil groups must never match") + assert.False(t, matchAddrPort([]objectcache.AddrPortGroup{}, "10.1.2.3", "TCP", 443), "empty groups must never match") +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_eval_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_eval_test.go new file mode 100644 index 0000000000..78f862820e --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_eval_test.go @@ -0,0 +1,165 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func buildSelectorLib(t *testing.T) *containerProfileNetworkLibrary { + t.Helper() + return buildLibWithContainer(t, + []v1beta1.NetworkNeighbor{ + {Identifier: "redis-clients", PodSelector: podSel(map[string]string{"app": "redis-client"})}, + {Identifier: "metrics", PodSelector: podSel(map[string]string{"app": "metrics"}), NamespaceSelector: nsSel("monitoring")}, + {Identifier: "plain-ip", IPAddresses: []string{"10.0.0.5"}}, + }, + []v1beta1.NetworkNeighbor{ + {Identifier: "lb", PodSelector: podSel(map[string]string{"app": "ingress-client"})}, + }) +} + +func labelsVal(m map[string]string) ref.Val { + return types.DefaultTypeAdapter.NativeToValue(m) +} + +func TestWasSelectorIn_EvalTruthTable(t *testing.T) { + lib := buildSelectorLib(t) + cases := []struct { + name string + ingress bool + cid string + ns string + labels map[string]string + want bool + why string + }{ + {"egress label match, omitted nsSel, same ns", false, "cid", "redis", map[string]string{"app": "redis-client"}, true, "labels match; omitted namespaceSelector => the profile's own namespace"}, + {"egress label match, omitted nsSel, FOREIGN ns rejected", false, "cid", "attacker", map[string]string{"app": "redis-client"}, false, "omitted namespaceSelector is same-namespace; a label-copy in another ns must not match"}, + {"egress label mismatch", false, "cid", "redis", map[string]string{"app": "other"}, false, "unknown peer identity must alert"}, + {"egress peer only in ingress list", false, "cid", "redis", map[string]string{"app": "ingress-client"}, false, "direction isolation: ingress-only selector must not open egress"}, + {"ingress peer matches", true, "cid", "redis", map[string]string{"app": "ingress-client"}, true, "declared ingress selector matches"}, + {"ingress peer only in egress list", true, "cid", "redis", map[string]string{"app": "redis-client"}, false, "direction isolation: egress-only selector must not open ingress"}, + {"ns-scoped peer, right ns", false, "cid", "monitoring", map[string]string{"app": "metrics"}, true, "explicit namespaceSelector matches metadata.name"}, + {"ns-scoped peer, wrong ns", false, "cid", "prod", map[string]string{"app": "metrics"}, false, "explicit namespaceSelector rejects other namespaces"}, + {"empty namespace fails closed", false, "cid", "", map[string]string{"app": "redis-client"}, false, "an unresolved peer (no ns) never satisfies a selector"}, + {"nil labels fail closed", false, "cid", "redis", nil, false, "a peer with no labels matches no non-empty selector"}, + {"empty labels fail closed", false, "cid", "redis", map[string]string{}, false, "empty label set matches no non-empty selector"}, + {"profile unavailable", false, "missing-cid", "redis", map[string]string{"app": "redis-client"}, false, "no profile means no allowlist entry"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var res ref.Val + if tc.ingress { + res = lib.wasSelectorInIngress(types.String(tc.cid), types.String(tc.ns), labelsVal(tc.labels), types.Int(443), types.String("TCP")) + } else { + res = lib.wasSelectorInEgress(types.String(tc.cid), types.String(tc.ns), labelsVal(tc.labels), types.Int(443), types.String("TCP")) + } + res = cache.ConvertProfileNotAvailableErrToBool(res, false) + assert.Equal(t, types.Bool(tc.want), res, tc.why) + }) + } +} + +func TestWasSelectorIn_NoPeersFailsClosed(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {Identifier: "plain-ip", IPAddresses: []string{"10.0.0.5"}}, + }, nil) + res := lib.wasSelectorInEgress(types.String("cid"), types.String("redis"), labelsVal(map[string]string{"app": "redis-client"}), types.Int(443), types.String("TCP")) + res = cache.ConvertProfileNotAvailableErrToBool(res, false) + assert.Equal(t, types.Bool(false), res, "a profile with no selector peers must match no peer identity") + res = lib.wasSelectorInIngress(types.String("cid"), types.String("redis"), labelsVal(map[string]string{"app": "redis-client"}), types.Int(443), types.String("TCP")) + res = cache.ConvertProfileNotAvailableErrToBool(res, false) + assert.Equal(t, types.Bool(false), res) +} + +func TestWasSelectorIn_ErrorEdges(t *testing.T) { + nilLib := &containerProfileNetworkLibrary{objectCache: nil} + assert.True(t, types.IsError(nilLib.wasSelectorInEgress(types.String("cid"), types.String("ns"), labelsVal(nil), types.Int(443), types.String("TCP")))) + assert.True(t, types.IsError(nilLib.wasSelectorInIngress(types.String("cid"), types.String("ns"), labelsVal(nil), types.Int(443), types.String("TCP")))) + + lib := buildSelectorLib(t) + assert.True(t, types.IsError(lib.wasSelectorInEgress(types.Int(1), types.String("ns"), labelsVal(nil), types.Int(443), types.String("TCP")))) + assert.True(t, types.IsError(lib.wasSelectorInEgress(types.String("cid"), types.Int(1), labelsVal(nil), types.Int(443), types.String("TCP")))) +} + +func TestRefValToStringMap(t *testing.T) { + assert.Nil(t, refValToStringMap(nil)) + assert.Nil(t, refValToStringMap(types.String("not-a-map"))) + assert.Equal(t, map[string]string{"a": "b"}, refValToStringMap(labelsVal(map[string]string{"a": "b"}))) +} + +func TestWasSelectorIn_CELEndToEnd(t *testing.T) { + lib := buildSelectorLib(t) + env, err := cel.NewEnv(cel.Variable("containerID", cel.StringType), cel.Lib(lib)) + assert.NoError(t, err) + + cases := []struct { + expr string + want bool + why string + }{ + {`cp.was_selector_in_egress(containerID, "redis", {"app": "redis-client"}, 443, "TCP")`, true, "declared egress peer matches through the CEL binding"}, + {`cp.was_selector_in_egress(containerID, "", {"app": "redis-client"}, 443, "TCP")`, false, "empty peer namespace fails closed through the binding"}, + {`cp.was_selector_in_egress(containerID, "redis", {}, 443, "TCP")`, false, "empty label map fails closed"}, + {`cp.was_selector_in_ingress(containerID, "redis", {"app": "ingress-client"}, 443, "TCP")`, true, "declared ingress peer matches"}, + {`cp.was_selector_in_ingress(containerID, "redis", {"app": "redis-client"}, 443, "TCP")`, false, "egress-only selector must not open ingress"}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + ast, issues := env.Compile(tc.expr) + assert.NoError(t, issues.Err()) + prg, err := env.Program(ast) + assert.NoError(t, err) + out, _, err := prg.Eval(map[string]interface{}{"containerID": "cid"}) + assert.NoError(t, err) + assert.Equal(t, tc.want, out.Value(), tc.why) + }) + } + + ast, issues := env.Compile(`cp.was_selector_in_egress(containerID, "redis", {"app": "redis-client"}, 443, "TCP")`) + assert.NoError(t, issues.Err()) + prg, err := env.Program(ast) + assert.NoError(t, err) + out, _, err := prg.Eval(map[string]interface{}{"containerID": "unknown-cid"}) + assert.NoError(t, err) + assert.Equal(t, false, out.Value(), "profile-unavailable converts to false at the binding, never an error") +} + +// TestWasSelectorIn_VendorPortableProfileEndToEnd drives the concrete vendor +// example through the real CEL binding: a profile shipped WITHOUT a namespace, +// installed into "acme". DNS is pinned to kube-system by name; Prometheus and +// Alertmanager use {} (any namespace, since the vendor can't know the customer's +// monitoring namespace); the app's own frontend uses an omitted selector (same +// namespace as the install). The label-copy attacker in "evil" is rejected. +func TestWasSelectorIn_VendorPortableProfileEndToEnd(t *testing.T) { + lib := buildLibWithContainerNS(t, "acme", + []v1beta1.NetworkNeighbor{ + {Identifier: "dns", PodSelector: podSel(map[string]string{"k8s-app": "kube-dns"}), NamespaceSelector: nsSel("kube-system")}, + {Identifier: "prometheus", PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "prometheus"}), NamespaceSelector: &metav1.LabelSelector{}}, + {Identifier: "alertmanager", PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "alertmanager"}), NamespaceSelector: &metav1.LabelSelector{}}, + }, + []v1beta1.NetworkNeighbor{ + {Identifier: "frontend", PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "acme-frontend"})}, + }) + + eg := func(ns string, l map[string]string) ref.Val { + return lib.wasSelectorInEgress(types.String("cid"), types.String(ns), labelsVal(l), types.Int(9090), types.String("TCP")) + } + ing := func(ns string, l map[string]string) ref.Val { + return lib.wasSelectorInIngress(types.String("cid"), types.String(ns), labelsVal(l), types.Int(443), types.String("TCP")) + } + + assert.Equal(t, types.Bool(true), eg("kube-system", map[string]string{"k8s-app": "kube-dns"}), "DNS in kube-system matches (metadata.name pin)") + assert.Equal(t, types.Bool(false), eg("acme", map[string]string{"k8s-app": "kube-dns"}), "a kube-dns pod in the install ns is NOT the kube-system peer") + assert.Equal(t, types.Bool(true), eg("monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}), "prometheus matches in any ns ({})") + assert.Equal(t, types.Bool(true), eg("observability", map[string]string{"app.kubernetes.io/name": "alertmanager"}), "alertmanager matches in any ns ({})") + assert.Equal(t, types.Bool(true), ing("acme", map[string]string{"app.kubernetes.io/name": "acme-frontend"}), "frontend in the install ns matches (omitted = same ns)") + assert.Equal(t, types.Bool(false), ing("evil", map[string]string{"app.kubernetes.io/name": "acme-frontend"}), "LABEL-COPY: acme-frontend in another ns must not match") +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go new file mode 100644 index 0000000000..ae04c10457 --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go @@ -0,0 +1,273 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/objectcache" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func podSel(m map[string]string) *metav1.LabelSelector { return &metav1.LabelSelector{MatchLabels: m} } +func nsSel(name string) *metav1.LabelSelector { + return &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": name}} +} + +// TestWasSelectorInPeers_TruthTable is the full matrix for peer-selector +// matching under NetworkPolicy semantics: matching is on pod LABELS; an OMITTED +// (nil) namespaceSelector means the profile's OWN namespace; an explicit empty +// {} is cluster-wide; an explicit metadata.name selector pins a namespace; an +// empty podSelector matches NOTHING (fail closed, opposite of NetworkPolicy). +// profileNs is the namespace the profiled workload runs in ("redis" here). +func TestWasSelectorInPeers_TruthTable(t *testing.T) { + const profileNs = "redis" + client := labels.Set{"app": "redis-client"} + clientPlus := labels.Set{"app": "redis-client", "tier": "cache"} + + exprIn := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpIn, Values: []string{"redis-client"}}}} + exprExists := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpExists}}} + + p := func(pod, ns *metav1.LabelSelector) objectcache.PeerSelector { + return objectcache.PeerSelector{PodSelector: pod, NamespaceSelector: ns} + } + + cases := []struct { + name string + peers []objectcache.PeerSelector + labels labels.Set + peerNs string + want bool + }{ + // --- podSelector shapes --- + {"nil podSelector never matches", []objectcache.PeerSelector{p(nil, nil)}, client, "redis", false}, + {"empty podSelector matches nothing (same ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, client, "redis", false}, + {"empty podSelector matches nothing (empty labels)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, labels.Set{}, "redis", false}, + {"empty podSelector matches nothing (explicit ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nsSel("redis"))}, client, "redis", false}, + + // --- omitted (nil) namespaceSelector => SAME namespace as the profile --- + {"label match, omitted ns, same ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"label match, omitted ns, FOREIGN ns REJECTED (same-ns default)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "attacker", false}, + {"label mismatch, omitted ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{"app": "other"}, "redis", false}, + {"selector is a subset of pod labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, clientPlus, "redis", true}, + {"non-empty selector vs empty labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{}, "redis", false}, + + // --- explicit metadata.name namespaceSelector (named namespace) --- + {"label+ns match", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "redis", true}, + {"explicit ns mismatch rejects (same labels)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "attacker", false}, + {"explicit ns names a third namespace", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("other"))}, client, "redis", false}, + + // --- explicit EMPTY {} namespaceSelector => cluster-wide (opt-in) --- + {"empty {} ns selector matches any ns (same)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), &metav1.LabelSelector{})}, client, "redis", true}, + {"empty {} ns selector matches any ns (foreign)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), &metav1.LabelSelector{})}, client, "attacker", true}, + + // --- matchExpressions (omitted ns => same ns) --- + {"matchExpressions In matches", []objectcache.PeerSelector{p(exprIn, nil)}, client, "redis", true}, + {"matchExpressions Exists matches labelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, client, "redis", true}, + {"matchExpressions Exists rejects unlabelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, labels.Set{}, "redis", false}, + + // --- list semantics --- + {"empty peer list", nil, client, "redis", false}, + {"one of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"none of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "y"}), nil)}, client, "redis", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, tc.labels, tc.peerNs, profileNs, "TCP", 443); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) + } + }) + } +} + +// TestNamespaceSelectorMatches_TruthTable pins the three namespaceSelector tiers: +// omitted (nil) = the profile's own namespace; explicit metadata.name = a named +// namespace (profileNs ignored); explicit empty {} = cluster-wide. +func TestNamespaceSelectorMatches_TruthTable(t *testing.T) { + cases := []struct { + name string + sel *metav1.LabelSelector + peerNs string + profileNs string + want bool + }{ + {"omitted matches same ns", nil, "prod", "prod", true}, + {"omitted rejects foreign ns (same-ns default)", nil, "attacker", "prod", false}, + {"explicit metadata.name matches (profileNs ignored)", nsSel("prod"), "prod", "other", true}, + {"explicit metadata.name rejects other", nsSel("prod"), "attacker", "prod", false}, + {"empty {} is cluster-wide", &metav1.LabelSelector{}, "anywhere", "prod", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := namespaceSelectorMatches(tc.sel, tc.peerNs, tc.profileNs); got != tc.want { + t.Fatalf("namespaceSelectorMatches = %v, want %v", got, tc.want) + } + }) + } +} + +func TestWasSelectorInPeers_InvalidSelectorFailsClosed(t *testing.T) { + const profileNs = "redis" + client := labels.Set{"app": "redis-client"} + badPod := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpIn}}} + badNs := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "kubernetes.io/metadata.name", Operator: "BadOp", Values: []string{"redis"}}}} + good := podSel(map[string]string{"app": "redis-client"}) + + cases := []struct { + name string + peers []objectcache.PeerSelector + want bool + why string + }{ + {"invalid podSelector alone", []objectcache.PeerSelector{{PodSelector: badPod}}, false, "unparseable podSelector must never match"}, + {"invalid podSelector skipped, later valid peer matches", []objectcache.PeerSelector{{PodSelector: badPod}, {PodSelector: good}}, true, "one bad entry must not poison the list"}, + {"valid podSelector, invalid namespaceSelector", []objectcache.PeerSelector{{PodSelector: good, NamespaceSelector: badNs}}, false, "unparseable namespaceSelector fails closed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, client, "redis", profileNs, "TCP", 443); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v — %s", got, tc.want, tc.why) + } + }) + } +} + +func TestNamespaceSelectorMatches_InvalidSelectorFailsClosed(t *testing.T) { + bad := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpIn}}} + if namespaceSelectorMatches(bad, "redis", "redis") { + t.Fatal("an unparseable namespaceSelector must fail closed, not match") + } +} + +// A selector peer with a Ports set is port-aware, mirroring the address matcher: +// nil Ports means any port, a declared (proto,port) matches only itself, and an +// empty-but-non-nil map matches nothing. Peers are same-namespace (omitted ns). +func TestWasSelectorInPeers_PortAware(t *testing.T) { + const profileNs = "redis" + sel := podSel(map[string]string{"app": "redis-client"}) + client := labels.Set{"app": "redis-client"} + withPorts := func(keys ...string) objectcache.PeerSelector { + m := map[string]struct{}{} + for _, k := range keys { + m[k] = struct{}{} + } + return objectcache.PeerSelector{PodSelector: sel, Ports: m} + } + k6379 := objectcache.PortKey("TCP", 6379) + cases := []struct { + name string + peers []objectcache.PeerSelector + proto string + port int32 + want bool + }{ + {"nil ports matches any port", []objectcache.PeerSelector{{PodSelector: sel}}, "TCP", 6379, true}, + {"declared port matches", []objectcache.PeerSelector{withPorts(k6379)}, "TCP", 6379, true}, + {"undeclared port rejected", []objectcache.PeerSelector{withPorts(k6379)}, "TCP", 5432, false}, + {"wrong protocol rejected", []objectcache.PeerSelector{withPorts(k6379)}, "UDP", 6379, false}, + {"empty ports map matches nothing", []objectcache.PeerSelector{{PodSelector: sel, Ports: map[string]struct{}{}}}, "TCP", 6379, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, client, "redis", profileNs, tc.proto, tc.port); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) + } + }) + } +} + +// TestWasSelectorInPeers_NamespaceDisambiguation states the collision case: two +// pods carry IDENTICAL podSelector labels in different namespaces. Omitted ns +// matches only the profile's own namespace (rejecting a label-copy elsewhere); +// explicit {} is cluster-wide; explicit metadata.name pins one namespace. +func TestWasSelectorInPeers_NamespaceDisambiguation(t *testing.T) { + const profileNs = "prod" + sameLabels := labels.Set{"app": "frontend"} + frontend := podSel(map[string]string{"app": "frontend"}) + + omitted := []objectcache.PeerSelector{{PodSelector: frontend}} + if !wasSelectorInPeers(omitted, sameLabels, "prod", profileNs, "TCP", 443) { + t.Fatal("omitted ns: frontend in the profile's own namespace must match") + } + if wasSelectorInPeers(omitted, sameLabels, "attacker", profileNs, "TCP", 443) { + t.Fatal("LABEL-COPY: omitted ns must reject the same-labelled pod in another namespace") + } + + clusterWide := []objectcache.PeerSelector{{PodSelector: frontend, NamespaceSelector: &metav1.LabelSelector{}}} + if !wasSelectorInPeers(clusterWide, sameLabels, "prod", profileNs, "TCP", 443) || + !wasSelectorInPeers(clusterWide, sameLabels, "attacker", profileNs, "TCP", 443) { + t.Fatal("empty {} ns selector must match in ANY namespace (cluster-wide opt-in)") + } + + pinned := []objectcache.PeerSelector{{PodSelector: frontend, NamespaceSelector: nsSel("prod")}} + if !wasSelectorInPeers(pinned, sameLabels, "prod", profileNs, "TCP", 443) { + t.Fatal("pinned ns=prod: frontend in prod must match") + } + if wasSelectorInPeers(pinned, sameLabels, "attacker", profileNs, "TCP", 443) { + t.Fatal("pinned ns=prod: identical-labelled frontend elsewhere must be rejected") + } +} + +// TestWasSelectorInPeers_VendorPortableProfile is the real-world story: a vendor +// ships a signed ContainerProfile WITHOUT its own namespace (the customer installs +// the workload into a namespace the vendor cannot know at signing time). The SAME +// peer bytes must work in any install namespace. The three tiers cover it: +// - DNS -> namespaceSelector metadata.name=kube-system (a universal name) +// - Prometheus/Alertmanager -> namespaceSelector {} (any ns; vendor can't know it) +// - own frontend -> omitted namespaceSelector (same ns as the install) +// +// It also proves the label-copy defense: an attacker's acme-frontend in another +// namespace does NOT inherit the same-namespace peer's trust. +func TestWasSelectorInPeers_VendorPortableProfile(t *testing.T) { + dns := objectcache.PeerSelector{ + PodSelector: podSel(map[string]string{"k8s-app": "kube-dns"}), + NamespaceSelector: nsSel("kube-system"), + } + prometheus := objectcache.PeerSelector{ + PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "prometheus"}), + NamespaceSelector: &metav1.LabelSelector{}, + } + alertmanager := objectcache.PeerSelector{ + PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "alertmanager"}), + NamespaceSelector: &metav1.LabelSelector{}, + } + frontend := objectcache.PeerSelector{ + PodSelector: podSel(map[string]string{"app.kubernetes.io/name": "acme-frontend"}), + } + peers := []objectcache.PeerSelector{dns, prometheus, alertmanager, frontend} + + dnsPod := labels.Set{"k8s-app": "kube-dns"} + promPod := labels.Set{"app.kubernetes.io/name": "prometheus"} + amPod := labels.Set{"app.kubernetes.io/name": "alertmanager"} + frontPod := labels.Set{"app.kubernetes.io/name": "acme-frontend"} + + for _, install := range []string{"acme", "tenant-42"} { + t.Run("install="+install, func(t *testing.T) { + match := func(pod labels.Set, peerNs string) bool { + return wasSelectorInPeers(peers, pod, peerNs, install, "TCP", 9090) + } + if !match(dnsPod, "kube-system") { + t.Fatal("DNS: kube-dns in kube-system must match") + } + if match(dnsPod, install) { + t.Fatal("DNS: a kube-dns pod in the install namespace must NOT match the kube-system-pinned peer") + } + if !match(promPod, "monitoring") || !match(promPod, "observability") { + t.Fatal("Prometheus: {} must match prometheus in ANY namespace") + } + if !match(amPod, "monitoring") { + t.Fatal("Alertmanager: {} must match alertmanager in the monitoring namespace") + } + if !match(frontPod, install) { + t.Fatalf("frontend: acme-frontend in the install ns %q must match (omitted = same ns)", install) + } + if match(frontPod, "evil") { + t.Fatal("LABEL-COPY: acme-frontend in another namespace must NOT match the same-namespace peer") + } + }) + } +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go index e0a16c2299..d105e0de75 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go @@ -15,6 +15,13 @@ import ( // Helper: build a ready-to-use library with a single-container profile. func buildLibWithContainer(t *testing.T, neighbors []v1beta1.NetworkNeighbor, ingressNeighbors []v1beta1.NetworkNeighbor) *containerProfileNetworkLibrary { + return buildLibWithContainerNS(t, "redis", neighbors, ingressNeighbors) +} + +// buildLibWithContainerNS builds the library with the profiled workload in a +// specific namespace, so an omitted (same-namespace) namespaceSelector resolves +// against it. +func buildLibWithContainerNS(t *testing.T, ns string, neighbors []v1beta1.NetworkNeighbor, ingressNeighbors []v1beta1.NetworkNeighbor) *containerProfileNetworkLibrary { t.Helper() objCache := objectcachev1.RuleObjectCacheMock{ ContainerIDToSharedData: maps.NewSafeMap[string, *objectcache.WatchedContainerData](), @@ -26,6 +33,7 @@ func buildLibWithContainer(t *testing.T, neighbors []v1beta1.NetworkNeighbor, in }, }) nn := &v1beta1.ContainerProfile{} + nn.Namespace = ns nn.Spec = v1beta1.ContainerProfileSpec{ Egress: neighbors, Ingress: ingressNeighbors, @@ -272,20 +280,16 @@ func TestWasAddressPortProtocolInEgress_PortWrapRejected(t *testing.T) { }, }, nil) - // See TestWasAddressPortProtocolInEgress_WithCIDR for the - // port/protocol regression note. The port-range guard ([0, 65535]) - // still applies — what's gone is port-specific matching: any in-range - // port matches if the address matches. cases := []struct { name string port int64 want bool }{ {"in-range hit", 443, true}, - {"in-range miss", 444, true}, // was: false (port mismatch). Now matches: address-only after projection-v1. - {"wrap-to-443 rejected", 4294967739, false}, // (1<<32)+443 — range guard fires - {"negative rejected", -1, false}, // range guard fires - {"too-large rejected", 65536, false}, // range guard fires + {"in-range miss", 444, false}, + {"wrap-to-443 rejected", 4294967739, false}, + {"negative rejected", -1, false}, + {"too-large rejected", 65536, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -350,27 +354,17 @@ func TestWasAddressPortProtocolInEgress_WithCIDR(t *testing.T) { }, }, nil) - // NOTE: upstream's projection-v1 (PR #799) explicitly drops port/protocol - // granularity from the address surface — the comment in network.go reads - // "port/protocol projection (AddressPortsByAddr) is out of scope for v1; - // degrade to address-only matching". So the matcher now only checks IP. - // - // Spec §4.7 still says ports[] is per-neighbor; the runtime gap is a - // known limitation flagged in the rebase commit. Test expectations - // updated to match runtime reality. Bringing port/protocol back is a - // follow-up: would need projection_apply to surface a per-address - // (port, protocol) set into ProjectedContainerProfile and the CEL - // helper to consult it. cases := []struct { observed string port int64 proto string want bool }{ - {"10.1.2.3", 443, "TCP", true}, // CIDR match (port/proto not enforced) - {"10.1.2.3", 80, "TCP", true}, // was: wrong port — now matches address-only - {"10.1.2.3", 443, "UDP", true}, // was: wrong protocol — now matches address-only - {"11.0.0.1", 443, "TCP", false}, // outside CIDR — still rejected + {"10.1.2.3", 443, "TCP", true}, + {"10.1.2.3", 80, "TCP", false}, + {"10.1.2.3", 443, "UDP", false}, + {"11.0.0.1", 443, "TCP", false}, + {"10.1.2.3", 443, "tcp", true}, } for _, tc := range cases { t.Run(tc.observed, func(t *testing.T) { diff --git a/pkg/rulemanager/cel/selector_compile_test.go b/pkg/rulemanager/cel/selector_compile_test.go new file mode 100644 index 0000000000..91fca459fa --- /dev/null +++ b/pkg/rulemanager/cel/selector_compile_test.go @@ -0,0 +1,41 @@ +package cel + +import ( + "testing" + "time" + + "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/objectcache" + objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" +) + +// TestCompileSelectorRules pins that the selector rules type-check against the +// real event object type: event.dstPodLabels is declared as a generic CEL map +// and must remain assignable to the was_selector_in_{ingress,egress} map param. +// Regression guard for the R0012 ingress rule that consumes the IG-enriched +// peer namespace + labels. +func TestCompileSelectorRules(t *testing.T) { + objCache := &objectcachev1.RuleObjectCacheMock{ + ContainerIDToSharedData: maps.NewSafeMap[string, *objectcache.WatchedContainerData](), + } + c, err := NewCEL(objCache, config.Config{ + CelConfigCache: cache.FunctionCacheConfig{MaxSize: 1000, TTL: time.Minute}, + }) + if err != nil { + t.Fatalf("NewCEL: %v", err) + } + + exprs := []string{ + `cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels, event.dstPort, event.proto)`, + `cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels, event.dstPort, event.proto)`, + // The full R0012 ingress expression as bound in default-rules.yaml. + `event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && !cp.was_address_in_ingress(event.containerId, event.dstAddr) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels, event.dstPort, event.proto)`, + } + for _, e := range exprs { + if err := c.registerExpression(e); err != nil { + t.Fatalf("expression failed to compile: %q\n%v", e, err) + } + } +} diff --git a/pkg/utils/cel.go b/pkg/utils/cel.go index 39c3ad40f2..b95cc1c115 100644 --- a/pkg/utils/cel.go +++ b/pkg/utils/cel.go @@ -202,6 +202,35 @@ var CelFields = map[string]*celtypes.FieldType{ return celtypes.Int(x.Raw.GetDstPort()), nil }), }, + // dstNamespace / dstPodLabels carry the peer identity that IG's + // kubeipresolver resolves cluster-wide (independent of node-agent's + // node-local pod cache), so selector rules can match a peer on any node. + "dstNamespace": { + Type: celtypes.StringType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + return celtypes.String(x.Raw.GetDstEndpoint().Namespace), nil + }), + }, + "dstPodLabels": { + Type: celtypes.MapType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + pl := x.Raw.GetDstEndpoint().PodLabels + if pl == nil { + pl = map[string]string{} + } + return pl, nil + }), + }, "exepath": { Type: celtypes.StringType, IsSet: isSet, diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index 03d5137555..7d4096a3fe 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -11,6 +11,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["watch", "list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "watch", "list"] diff --git a/tests/chart/templates/node-agent/configmap.yaml b/tests/chart/templates/node-agent/configmap.yaml index 523b5bbac6..b64a63024b 100644 --- a/tests/chart/templates/node-agent/configmap.yaml +++ b/tests/chart/templates/node-agent/configmap.yaml @@ -14,6 +14,7 @@ data: "prometheusExporterEnabled": {{ eq .Values.nodeAgent.config.prometheusExporter "enable" }}, "runtimeDetectionEnabled": {{ eq .Values.capabilities.runtimeDetection "enable" }}, "networkServiceEnabled": {{ eq .Values.capabilities.networkPolicyService "enable" }}, + "networkServiceResolutionEnabled": {{ if hasKey .Values.nodeAgent.config "networkServiceResolution" }}{{ .Values.nodeAgent.config.networkServiceResolution }}{{ else }}true{{ end }}, "malwareDetectionEnabled": {{ eq .Values.capabilities.malwareDetection "enable" }}, "httpDetectionEnabled": {{ eq .Values.capabilities.httpDetection "enable" }}, "initialDelay": "{{ .Values.nodeAgent.config.learningPeriod }}", diff --git a/tests/chart/templates/node-agent/default-rule-binding.yaml b/tests/chart/templates/node-agent/default-rule-binding.yaml index 3d8f7847b4..755bd39055 100644 --- a/tests/chart/templates/node-agent/default-rule-binding.yaml +++ b/tests/chart/templates/node-agent/default-rule-binding.yaml @@ -39,5 +39,6 @@ spec: - ruleName: "Exec to pod" - ruleName: "Port forward to pod" - ruleName: "Unexpected Egress Network Traffic" + - ruleName: "Unexpected Ingress Network Traffic" - ruleName: "Unexpected Ptrace Syscall Usage" - ruleName: "Unexpected io_uring Operation Detected" diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 512b4d9ec8..e39a5f7c0d 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels, event.dstPort, event.proto)" profileDependency: 0 profileDataRequired: egressAddresses: all @@ -329,6 +329,31 @@ spec: - "network" - "anomaly" - "networkprofile" + - name: "Unexpected Ingress Network Traffic" + enabled: true + id: "R0012" + description: "Detecting unexpected ingress network traffic that is not allowlisted by application profile. Symmetric twin of R0011: internal and external peers alike, only loopback excluded; internal peers are allowlisted narrowly via addresses or resolved serviceRef/serviceSelector entries rather than a serviceCIDR that blinds detection to lateral movement." + expressions: + message: "'Unexpected ingress network communication from: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' to: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'HOST' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels, event.dstPort, event.proto)" + profileDependency: 0 + profileDataRequired: + ingressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" - name: "Unexpected process arguments" enabled: true id: "R0040" diff --git a/tests/chart/values.yaml b/tests/chart/values.yaml index 1aea3a150f..3083c99bd9 100644 --- a/tests/chart/values.yaml +++ b/tests/chart/values.yaml @@ -33,7 +33,7 @@ storage: name: "storage" image: repository: quay.io/kubescape/storage - tag: v0.0.156 + tag: v0.0.320 pullPolicy: Always cleanupInterval: "6h" labels: @@ -58,6 +58,7 @@ nodeAgent: maxLearningPeriod: 2m learningPeriod: 1m updatePeriod: 30s + networkServiceResolution: true maxDelaySeconds: 1 prometheusExporter: enable httpExporterConfig: {} diff --git a/tests/component_test.go b/tests/component_test.go index 154e441e8b..52b6637055 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -1102,13 +1102,9 @@ func Test_20_AlertOnPartialThenLearnProcessTest(t *testing.T) { t.Fatalf("timeout after %s waiting for %s", timeout, desc) } - // Give node-agent time to project the authored profile before generating - // events (matches Test_28; evaluating an unloaded profile is unreliable). - time.Sleep(30 * time.Second) - // PHASE 1 — subject NOT in the profile must alert. Doubles as the - // profile-load gate: once the authored CP is loaded, ls (not allowed) - // fires R0001. + // profile-load gate: re-exec ls inside the poll until R0001 fires, which only + // happens once the authored CP is loaded (no fixed projection sleep needed). waitFor(func() bool { wl.ExecIntoPod([]string{"/usr/bin/ls", "-l"}, containerName) return countR0001("ls") > 0 @@ -1126,11 +1122,9 @@ func Test_20_AlertOnPartialThenLearnProcessTest(t *testing.T) { _, err = storageClient.ContainerProfiles(ns.Name).Update(context.Background(), cur, metav1.UpdateOptions{}) require.NoError(t, err, "update CP: add ls, remove id") - // Propagation delay before the reload gate (not an assertion gate). - time.Sleep(20 * time.Second) - - // RELOAD GATE (positive) — the removed canary (id) must now alert, which - // proves node-agent reloaded the new revision (which also contains ls). + // RELOAD GATE (positive) — re-exec the removed canary (id) inside the poll + // until it alerts, which proves node-agent reloaded the new revision (which + // also contains ls). No fixed propagation sleep. waitFor(func() bool { wl.ExecIntoPod([]string{"/usr/bin/id"}, containerName) return countR0001("id") > 0 @@ -1138,20 +1132,21 @@ func Test_20_AlertOnPartialThenLearnProcessTest(t *testing.T) { t.Logf("reload confirmed: R0001(id)=%d", countR0001("id")) // PHASE 2 — the SAME subject, now in the profile, must NOT produce a NEW - // R0001. Cooldown headroom (per-container/per-rule, count 10) is untouched - // by the id-based gate, so a failed reload here would still let ls alert - // and be caught — this is a real enforcement check, not a vacuous pass. + // R0001. Snapshot ls before, exec ls, then fire a fresh forbidden SENTINEL + // (whoami, never in the profile) AFTER it. node-agent processes events in + // order, so once the sentinel alerts, the ls execs have been processed too — + // making "no new ls alert" deterministic, not a fixed-sleep guess. before := countR0001("ls") - // Guard against phase-1 self-exhaustion: if the per-container/per-rule R0001 - // cooldown budget (cap 10) were already spent, ls could not alert in phase 2 - // regardless of enforcement, making the "no NEW R0001" check below vacuous. require.Less(t, before, 10, "phase 1 exhausted the R0001 ls cooldown budget (before=%d, cap=10); phase 2 would pass vacuously", before) _, _, err = wl.ExecIntoPod([]string{"/usr/bin/ls", "-l"}, containerName) require.NoError(t, err, "exec ls after profile update") _, _, err = wl.ExecIntoPod([]string{"/usr/bin/ls", "-l"}, containerName) require.NoError(t, err, "exec ls after profile update") - time.Sleep(20 * time.Second) // settle so any alert would have surfaced + waitFor(func() bool { + wl.ExecIntoPod([]string{"/usr/bin/whoami"}, containerName) + return countR0001("whoami") > 0 + }, 3*time.Minute, "sentinel whoami (forbidden) must fire R0001 — proves the pipeline drained past the ls execs") after := countR0001("ls") if after != before { logCPs() @@ -1192,6 +1187,19 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { fusioncoreIP = "162.0.217.171" ) port80 := int32(80) + port53 := int32(53) + // R0011 excludes only loopback (maximally noisy by design), so authored + // profiles must allow the pod's own DNS egress to cluster DNS or every + // nslookup mints an R0011 that skews the before/after counts. + clusterDNS := v1beta1.NetworkNeighbor{ + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + IPAddresses: []string{"10.96.0.0/12"}, + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + } ns := testutils.NewRandomNamespace() k8sClient := k8sinterface.NewKubernetesApi() @@ -1220,6 +1228,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: fusioncoreIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, }, }, } @@ -1291,6 +1300,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: subjectIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, } _, err = storageClient.ContainerProfiles(ns.Name).Update(context.Background(), cur, metav1.UpdateOptions{}) require.NoError(t, err, "update CP: add subject IP, remove canary domain") @@ -2718,6 +2728,52 @@ func Test_28_UserDefinedNetworkNeighborhood(t *testing.T) { "fusioncore.ai IP is in NN — should NOT fire R0011") }) + // 162.0.217.171 is allowed on TCP/80 only; :443 is a port violation → R0011. + t.Run("port_violation_different_port_R0011", func(t *testing.T) { + wl := setup(t) + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://162.0.217.171"}, "curl") + t.Logf("curl https://162.0.217.171 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to allowed IP 162.0.217.171 on non-allowed port 443 must fire R0011") + }) + + // 9.9.9.9 is listed with an explicit port 0 — a literal, NOT a wildcard: + // the only port wildcard is an absent ports stanza, so :80/:443 violate. + t.Run("port_zero_is_literal_not_wildcard", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://9.9.9.9"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://9.9.9.9"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "an explicit port-0 entry must not open 9.9.9.9 on other ports") + }) + + // 208.67.222.222 is allowlisted with no ports stanza (ANY); no port fires R0011. + t.Run("port_wildcard_empty_stanza_allows_any", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://208.67.222.222"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://208.67.222.222"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.Equal(t, 0, countByRule(alerts, "R0011"), + "208.67.222.222 allowlisted with empty ports stanza (any) must not fire R0011 on any port") + }) + + // Internal peer 10.96.0.1 (kube-api) is allowlisted on TCP/443 only; :80 is a port violation → R0011. + t.Run("internal_port_violation_R0011", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://10.96.0.1"}, "curl") + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "http://10.96.0.1"}, "curl") + t.Logf("curl http://10.96.0.1:80 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to internal IP 10.96.0.1 on non-allowed port 80 must fire R0011") + }) + // --------------------------------------------------------------- // 28b. Unknown domains — domains NOT in the NN → R0005. // Uses both nslookup (pure DNS) and curl (DNS + TCP). @@ -3413,60 +3469,44 @@ func Test_36_MultiContainerPerContainerBinding(t *testing.T) { path.Join(utils.CurrentDir(), "resources/percontainer-deployment.yaml")) require.NoError(t, err) require.NoError(t, wl.WaitForReady(80)) - // Cache-load latency on the ContainerProfileCache is bursty; 30s covers the - // observed worst case on a loaded runner (matches Test_28). - time.Sleep(30 * time.Second) - - // Exercise each container with BOTH binaries. Expected R0001 (unexpected - // process) per the inverse allow-lists: + // Exercise both binaries in both containers. Re-exec inside the poll so + // authored-CP load latency is absorbed deterministically (no fixed learning + // sleep). The two FORBIDDEN execs are the sentinels: once both have fired + // R0001, the pipeline has processed the whole burst, so the two allowed-exec + // negatives below are deterministic — not "hasn't surfaced yet". // app : whoami -> R0001 (not allowed) ; id -> allowed (no alert) // sidecar : id -> R0001 (not allowed) ; whoami -> allowed (no alert) - wl.ExecIntoPod([]string{"/usr/bin/whoami"}, "app") - wl.ExecIntoPod([]string{"/usr/bin/id"}, "app") - wl.ExecIntoPod([]string{"/usr/bin/id"}, "sidecar") - wl.ExecIntoPod([]string{"/usr/bin/whoami"}, "sidecar") - - var alerts []testutils.Alert - require.Eventually(t, func() bool { - var e error - alerts, e = testutils.GetAlerts(wl.Namespace) - return e == nil - }, 60*time.Second, 5*time.Second, "must be able to fetch alerts") - // Extra settle time for remaining alerts. - time.Sleep(10 * time.Second) - alerts, _ = testutils.GetAlerts(wl.Namespace) - - for i, a := range alerts { - t.Logf(" [%d] %s(%s) comm=%s container=%s", i, - a.Labels["rule_name"], a.Labels["rule_id"], a.Labels["comm"], a.Labels["container_name"]) + exerciseAll := func() { + wl.ExecIntoPod([]string{"/usr/bin/whoami"}, "app") + wl.ExecIntoPod([]string{"/usr/bin/id"}, "app") + wl.ExecIntoPod([]string{"/usr/bin/id"}, "sidecar") + wl.ExecIntoPod([]string{"/usr/bin/whoami"}, "sidecar") } - - countR0001 := func(container, comm string) int { - n := 0 - for _, a := range alerts { - if a.Labels["rule_id"] == "R0001" && - a.Labels["container_name"] == container && - a.Labels["comm"] == comm { - n++ - } + pollUntil(t, exerciseAll, func() bool { + return countRuleAlerts(t, ns.Name, "R0001", "app", "whoami") > 0 && + countRuleAlerts(t, ns.Name, "R0001", "sidecar", "id") > 0 + }, 4*time.Minute, "both forbidden execs (app/whoami, sidecar/id) must fire R0001") + + // Truth table — per-container binding, no cross-inheritance. Evaluated after + // both sentinels fired, so the wantAlert==false rows are deterministic. + for _, r := range []struct { + container, comm string + wantAlert bool + why string + }{ + {"app", "whoami", true, "whoami not in percontainer-app -> must fire R0001 in app"}, + {"sidecar", "id", true, "id not in percontainer-sidecar -> must fire R0001 in sidecar"}, + {"app", "id", false, "id IS in percontainer-app -> must NOT fire (else sidecar's CP leaked in)"}, + {"sidecar", "whoami", false, "whoami IS in percontainer-sidecar -> must NOT fire (else app's CP leaked in)"}, + } { + got := countRuleAlerts(t, ns.Name, "R0001", r.container, r.comm) + if r.wantAlert { + assert.Greater(t, got, 0, r.why) + } else { + assert.Equal(t, 0, got, r.why) } - return n } - // The forbidden process in each container MUST alert. - assert.Greater(t, countR0001("app", "whoami"), 0, - "whoami is NOT in percontainer-app (only sidecar's CP allows it) — must fire R0001 in app") - assert.Greater(t, countR0001("sidecar", "id"), 0, - "id is NOT in percontainer-sidecar (only app's CP allows it) — must fire R0001 in sidecar") - - // The allowed process in each container MUST NOT alert — the - // no-cross-inheritance assertion. If both containers shared one CP, one of - // these would be non-zero. - assert.Equal(t, 0, countR0001("app", "id"), - "id IS in percontainer-app — must NOT fire R0001 in app (non-zero => sidecar's CP leaked in)") - assert.Equal(t, 0, countR0001("sidecar", "whoami"), - "whoami IS in percontainer-sidecar — must NOT fire R0001 in sidecar (non-zero => app's CP leaked in)") - t.Run("refresh_reprojects_authored_CP_update", func(t *testing.T) { k8sClient := k8sinterface.NewKubernetesApi() storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) @@ -3482,22 +3522,11 @@ func Test_36_MultiContainerPerContainerBinding(t *testing.T) { _, err = storageClient.ContainerProfiles(ns.Name).Update(context.Background(), cp, v1.UpdateOptions{}) require.NoError(t, err, "update percontainer-app to forbid id") - time.Sleep(45 * time.Second) - wl.ExecIntoPod([]string{"/usr/bin/id"}, "app") - - require.Eventually(t, func() bool { - a2, e := testutils.GetAlerts(wl.Namespace) - if e != nil { - return false - } - for _, a := range a2 { - if a.Labels["rule_id"] == "R0001" && a.Labels["container_name"] == "app" && a.Labels["comm"] == "id" { - return true - } - } - return false - }, 90*time.Second, 5*time.Second, - "after percontainer-app is updated to forbid id, the reconciler refresh must re-fetch and re-project it so id now fires R0001 in app") + // Re-exec id inside the poll: once the reconciler re-fetches and re-projects + // the updated CP, id (now forbidden) fires R0001 in app. No fixed sleep. + pollUntil(t, func() { wl.ExecIntoPod([]string{"/usr/bin/id"}, "app") }, func() bool { + return countRuleAlerts(t, ns.Name, "R0001", "app", "id") > 0 + }, 3*time.Minute, "reconciler must re-project the updated CP so id now fires R0001 in app") }) } @@ -3689,6 +3718,27 @@ func countRuleAlerts(t *testing.T, ns, ruleID, container, comm string) int { return n } +// pollUntil re-runs act each interval (to absorb profile load / re-projection +// latency) and returns once cond holds; it fails the test deterministically on +// timeout instead of relying on a single fixed sleep. act may be nil for a +// pure wait. Used to gate a NEGATIVE assertion on a POSITIVE sentinel: once the +// sentinel event (which was generated after the action under test) is observed, +// the pipeline has demonstrably drained past that action. +func pollUntil(t *testing.T, act func(), cond func() bool, timeout time.Duration, desc string) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if act != nil { + act() + } + if cond() { + return + } + time.Sleep(3 * time.Second) + } + t.Fatalf("timeout after %s waiting for %s", timeout, desc) +} + // withNodeAgentConfig mutates config.json + restarts the DaemonSet; the returned func reverts + restarts. func withNodeAgentConfig(t *testing.T, mutate func(cfg map[string]any)) func() { t.Helper() @@ -3710,8 +3760,11 @@ func withNodeAgentConfig(t *testing.T, mutate func(cfg map[string]any)) func() { c.Data["config.json"] = body _, e = k8s.KubernetesClient.CoreV1().ConfigMaps(nsKS).Update(context.Background(), c, metav1.UpdateOptions{}) require.NoError(t, e, "update node-agent ConfigMap") + // RestartDaemonSet now blocks until the new-config pod is actually rolled + // out and ready; a short settle lets node-agent re-attach to running + // containers (IG fanotify marks) before the test generates events. require.NoError(t, testutils.RestartDaemonSet(nsKS, cmName), "restart node-agent") - time.Sleep(45 * time.Second) + time.Sleep(20 * time.Second) } apply(string(updated)) return func() { apply(original) } @@ -3840,26 +3893,38 @@ func Test_30_IgnoreExcludeAndLearningDuration(t *testing.T) { require.NoError(t, err, "control workload") require.NoError(t, ctl.WaitForReady(80)) - time.Sleep(90 * time.Second) + // Positive sentinel: the identical, co-deployed CONTROL gets profiled. Once + // it has a ContainerProfile, the excluded one would too if it were not + // excluded — so the excluded's ABSENCE is deterministic, not "not yet". + pollUntil(t, nil, func() bool { + cps, _ := ctl.GetContainerProfiles() + return len(cps) > 0 + }, 3*time.Minute, "the non-excluded control workload must be profiled") exCPs, _ := exNS.GetContainerProfiles() - require.Empty(t, exCPs, "an excluded-namespace workload must produce NO ContainerProfile") - ctlCPs, _ := ctl.GetContainerProfiles() - require.NotEmpty(t, ctlCPs, "a non-excluded workload must still be profiled (exclusion must be selective)") + require.Empty(t, exCPs, "an excluded-namespace workload must produce NO ContainerProfile (the control already has one)") - for i := 0; i < 5; i++ { + // Excluded containers are dropped at IgnoreContainer — no profile AND no + // event collection — so id execs cannot alert. The control-CP sentinel + // above establishes the timing floor, making 0 deterministic. + for i := 0; i < 3; i++ { _, _, _ = exNS.ExecIntoPod([]string{"/usr/bin/id"}, "app") - time.Sleep(2 * time.Second) } - time.Sleep(20 * time.Second) require.Equal(t, 0, countRuleAlerts(t, excluded.Name, "R0001", "app", "id"), "an excluded container must generate no alerts") }) t.Run("LearningDurationOverride", func(t *testing.T) { + // Drive a short AND frequently-checkpointed learning window. maxSniffing + // must be comfortably larger than updateDataPeriod, otherwise the "sniff + // window elapsed -> finalize" check can fall between update ticks and the + // profile never completes (the observed flake with 30s==30s). 40s sniff + + // 10s checkpoints finalizes reliably at ~40-50s, still far under the + // multi-minute default so the elapsed bound below stays discriminating. restore := withNodeAgentConfig(t, func(cfg map[string]any) { - cfg["maxSniffingTimePerContainer"] = "45s" - cfg["initialDelay"] = "10s" + cfg["initialDelay"] = "5s" + cfg["maxSniffingTimePerContainer"] = "40s" + cfg["updateDataPeriod"] = "10s" }) defer restore() @@ -3868,10 +3933,20 @@ func Test_30_IgnoreExcludeAndLearningDuration(t *testing.T) { require.NoError(t, err, "workload") require.NoError(t, wl.WaitForReady(80)) - deadline := time.Now().Add(90 * time.Second) + // Gate on node-agent actually tracking the workload (its CP exists) before + // timing completion, so a slow attach isn't charged against the window. + pollUntil(t, nil, func() bool { + cps, _ := wl.GetContainerProfiles() + return len(cps) > 0 + }, 2*time.Minute, "node-agent must create the learner ContainerProfile") + + // Completion is the deterministic signal; the elapsed bound is the + // discriminator — a sub-2m completion cannot happen under the default + // multi-minute learning period. + startLearn := time.Now() require.NoError(t, wl.WaitForContainerProfileCompletion(90), "profile must complete within the shortened window") - require.True(t, time.Now().Before(deadline), - "completion must track the configured maxSniffingTimePerContainer, not a longer default") + require.Less(t, time.Since(startLearn), 2*time.Minute, + "completion must track the configured ~40s window, not the multi-minute default") }) } @@ -3906,3 +3981,361 @@ func Test_49_EphemeralContainerFullTreatment(t *testing.T) { return countRuleAlerts(t, ns.Name, "R0001", "ephcon", "id") > 0 }, 2*time.Minute, 10*time.Second, "id was not in the ephemeral container's learned profile — it must fire R0001 (detected + alerted like any other container)") } + +// Test_50_ServiceRefNetworkNeighbor validates serviceRef/serviceSelector end to +// end (k8sstormcenter/node-agent#92) against REAL GitOps traffic: a Flux +// source-controller reconciling HelmRepository CRs. Its egress is authored +// purely as Kubernetes-native selectors — serviceRef default/kubernetes for the +// apiserver, serviceRef kube-system/kube-dns for name resolution, and a +// serviceSelector role=helm-repo fanning across the two repo Services it is +// allowed to fetch. Nothing is exec'd and no address is hardcoded; the +// controller's own reconcile loop generates every connection. +// +// The negative is the lateral move a broad serviceCIDR entry would hide: the +// HelmRepository URL is repointed at decoy-repo — a sibling Service on the same +// port, backed by its own pod, carrying none of the selector's labels — and the +// controller itself fetches it. R0011 MUST fire for that and MUST NOT fire for +// the allowlisted Services. +func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + const ( + cpName = "serviceref-flux-cp" + containerName = "manager" + ) + port80, port443, port53 := int32(80), int32(443), int32(53) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + // Authored profile: NETWORK ONLY. No syscalls/execs — they are irrelevant to + // a network test and only add false-positive surface. Every peer is named, + // never addressed. + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "source-controller"}}, + Egress: []v1beta1.NetworkNeighbor{ + { + Identifier: "apiserver", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "default", + ServiceRefName: "kubernetes", + Ports: []v1beta1.NetworkPort{{Name: "TCP-443", Protocol: v1beta1.ProtocolTCP, Port: &port443}}, + }, + { + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "kube-system", + ServiceRefName: "kube-dns", + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + }, + { + Identifier: "helm-repos", + Type: v1beta1.CommunicationTypeEgress, + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"role": "helm-repo"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns.Name}}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + require.Eventually(t, func() bool { + _, e := storageClient.ContainerProfiles(ns.Name).Get(context.Background(), cpName, v1.GetOptions{}) + return e == nil + }, 30*time.Second, time.Second, "authored CP must be in storage before pod deploy") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + waitDeploy := func(name string) { + t.Helper() + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) + } + waitDeploy("helm-repo") + waitDeploy("decoy-repo") + waitDeploy("source-controller") + + countRule := func(ruleID string) int { + alerts, _ := testutils.GetAlerts(ns.Name) + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == ruleID && a.Labels["container_name"] == containerName { + n++ + } + } + return n + } + + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + newRepo := func(name, svc string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": name, "namespace": ns.Name}, + // Trailing-dot FQDN: absolute, so the resolver does not walk the + // search list and emit extra lookups. + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://%s.%s.svc.cluster.local./", svc, ns.Name), + }, + }} + } + repoReady := func(name string) bool { + obj, e := repoClient.Get(context.TODO(), name, metav1.GetOptions{}) + if e != nil { + return false + } + conds, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, c := range conds { + m, ok := c.(map[string]interface{}) + if ok && m["type"] == "Ready" && m["status"] == "True" { + return true + } + } + return false + } + + // Let node-agent bind the profile and fill its Service/EndpointSlice caches + // before any reconcile traffic is judged. + time.Sleep(40 * time.Second) + + // Phase 1 — the controller reconciles both allowlisted repo Services while + // continuously talking to the apiserver and cluster DNS. Every one of those + // peers is named by the profile, so no egress alert may fire. + t.Run("selector_allowed_no_alert", func(t *testing.T) { + for _, r := range []struct{ name, svc string }{{"primary", "helm-primary"}, {"mirror", "helm-mirror"}} { + _, e := repoClient.Create(context.TODO(), newRepo(r.name, r.svc), metav1.CreateOptions{}) + require.NoError(t, e, "create HelmRepository %s", r.name) + } + for _, n := range []string{"primary", "mirror"} { + require.Eventually(t, func() bool { return repoReady(n) }, 3*time.Minute, 10*time.Second, + "HelmRepository %s must reconcile (real fetch through an allowlisted Service)", n) + } + // Two further reconcile intervals of steady-state traffic. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countRule("R0011"), + "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — R0011 may not fire") + }) + + // Phase 2 — the GitOps source of truth is tampered with: primary is + // repointed at decoy-repo, a sibling Service on the same port that the + // role=helm-repo selector does not cover. source-controller fetches it on + // its own next reconcile. This is the lateral move a serviceCIDR entry hides. + t.Run("sibling_service_pivot_fires_alert", func(t *testing.T) { + before := countRule("R0011") + patch := []byte(fmt.Sprintf(`{"spec":{"url":"http://decoy-repo.%s.svc.cluster.local./"}}`, ns.Name)) + _, e := repoClient.Patch(context.TODO(), "primary", types.MergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, e, "repoint HelmRepository at the decoy Service") + require.Eventually(t, func() bool { + return countRule("R0011") > before + }, 4*time.Minute, 15*time.Second, + "egress to an unlisted sibling Service MUST fire R0011 — the selector is narrow, not a blanket") + }) +} + +// Test_51_ServiceRefIngressR0012 is the ingress twin of Test_50: nginx serves +// two real clients; the profile names one of them (ingress serviceRef, whose +// resolution covers the client Service's ClusterIP and pod endpoint IPs), and +// R0012 must stay silent for it while firing for the unlisted one (k6). +func Test_51_ServiceRefIngressR0012(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + const cpName = "serviceref-ingress-cp" + port80 := int32(80) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "helm-repo"}}, + Ingress: []v1beta1.NetworkNeighbor{ + { + Identifier: "gitops-clients", + Type: v1beta1.CommunicationTypeIngress, + ServiceRefNamespace: ns.Name, + ServiceRefName: "source-controller", + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + patch := []byte(fmt.Sprintf(`{"spec":{"template":{"metadata":{"labels":{"kubescape.io/user-defined-profile":%q}}}}}`, cpName)) + _, err = k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Patch(context.TODO(), "helm-repo", types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err, "bind profile to helm-repo pods") + + waitDeploy := func(name string) { + t.Helper() + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 && d.Status.UpdatedReplicas == d.Status.ReadyReplicas + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) + } + waitDeploy("helm-repo") + waitDeploy("source-controller") + + countR0012 := func() int { + alerts, _ := testutils.GetAlerts(ns.Name) + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == "R0012" && a.Labels["container_name"] == "nginx" { + n++ + } + } + return n + } + + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + repo := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": "primary", "namespace": ns.Name}, + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://helm-primary.%s.svc.cluster.local./", ns.Name), + }, + }} + _, err = repoClient.Create(context.TODO(), repo, metav1.CreateOptions{}) + require.NoError(t, err, "create HelmRepository") + + time.Sleep(40 * time.Second) + + t.Run("listed_client_no_r0012", func(t *testing.T) { + // Two reconcile intervals of real source-controller fetches into nginx. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countR0012(), + "ingress from the serviceRef-listed client must not fire R0012") + }) + + t.Run("unlisted_client_fires_r0012", func(t *testing.T) { + before := countR0012() + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-k6.yaml")), + "deploy unlisted k6 client") + require.Eventually(t, func() bool { + return countR0012() > before + }, 4*time.Minute, 15*time.Second, + "ingress from a client no serviceRef names MUST fire R0012") + }) +} + +// Test_53: default posture (learned CP only, no user-defined profile) — replaying the exact learn-window traffic after completion must yield zero R0011/R0012, and the learn window itself must be alert-free (profile_incomplete suppression), cf Test_01/Test_51. +func Test_53_DefaultLearnedNetworkFalsePositives(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + ns := testutils.NewRandomNamespace() + + server, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/nginx-deployment.yaml")) + require.NoError(t, err, "create nginx server workload") + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/nginx-service.yaml")), "create nginx service") + client, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/network-default-client.yaml")) + require.NoError(t, err, "create curl client workload") + require.NoError(t, server.WaitForReady(80), "server ready") + require.NoError(t, client.WaitForReady(80), "client ready") + + time.Sleep(10 * time.Second) + + svcURL := fmt.Sprintf("http://nginx-service.%s.svc.cluster.local./", ns.Name) + sendTraffic := func() { + for i := 0; i < 3; i++ { + _, _, _ = client.ExecIntoPod([]string{"curl", "-sS", "-m", "5", svcURL}, "curl") + time.Sleep(2 * time.Second) + } + } + sendTraffic() + + require.NoError(t, client.WaitForContainerProfileCompletion(100), "client CP completed") + require.NoError(t, server.WaitForContainerProfileCompletion(100), "server CP completed") + + time.Sleep(30 * time.Second) + + learnR0011 := countRuleAlerts(t, ns.Name, "R0011", "curl", "") + learnR0012 := countRuleAlerts(t, ns.Name, "R0012", "nginx", "") + assert.Equal(t, 0, learnR0011, "R0011 must be suppressed (profile_incomplete) during the learn window") + assert.Equal(t, 0, learnR0012, "R0012 must be suppressed (profile_incomplete) during the learn window") + + sendTraffic() + + time.Sleep(30 * time.Second) + + assert.Equal(t, learnR0011, countRuleAlerts(t, ns.Name, "R0011", "curl", ""), + "replayed learn-window egress (kube-dns + nginx service) must not fire R0011; a diff here is a default-posture false positive — the learned svc entry stores the service SELECTOR while kubeipresolver stamps the service metadata LABELS on the event, and nginx-service has none") + assert.Equal(t, learnR0012, countRuleAlerts(t, ns.Name, "R0012", "nginx", ""), + "replayed learn-window ingress (same client pod) must not fire R0012") + + // Node-IP ingress: a hostNetwork client reaches nginx with source = the node + // InternalIP (172.19.0.2 on Kind), which was never in nginx's learn window. + // The default host-peer allowlist (alertOnHostPeers=false) resolves entity:host + // to the node IPs, so this must add zero R0012 — the (b) leg. The (a) leg + // (without the allowlist the node IP is uncovered and would alert) is pinned + // at unit level in networkpeer.TestWithHostPeer_ResolvesNodeIPsIntoAddressSurface. + t.Run("node_ip_ingress_silenced_by_default", func(t *testing.T) { + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/network-hostnet-client.yaml")), "deploy hostNetwork client") + hostClient, err := testutils.NewTestWorkloadFromK8sIdentifiers(ns.Name, "Deployment", "hostnet-client") + require.NoError(t, err, "resolve hostnet-client") + require.NoError(t, hostClient.WaitForReady(80), "hostnet-client ready") + before := countRuleAlerts(t, ns.Name, "R0012", "nginx", "") + for i := 0; i < 5; i++ { + _, _, _ = hostClient.ExecIntoPod([]string{"curl", "-sS", "-m", "5", svcURL}, "curl") + time.Sleep(2 * time.Second) + } + time.Sleep(30 * time.Second) + after := countRuleAlerts(t, ns.Name, "R0012", "nginx", "") + t.Logf("node-IP ingress: R0012(nginx) before=%d after=%d", before, after) + assert.Equal(t, before, after, + "ingress from the node IP must be silenced by the default host-peer allowlist (alertOnHostPeers=false)") + }) + + // Loopback measurement: with the 127.*/::1 rule guards AND the learn-time + // loopback drop removed, localhost traffic is subject to R0011/R0012 and is + // learnable. A curl sidecar hits loopserver on 127.0.0.1 (shared pod netns), + // so both containers baseline the loopback peer; the replay then measures the + // residual FP tax. Unique container names (loopclient/loopserver) keep the + // counts separate from the main nginx/curl workloads. + t.Run("loopback_learn_enforce_measurement", func(t *testing.T) { + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/network-loopback-pod.yaml")), "deploy loopback pod") + lb, err := testutils.NewTestWorkloadFromK8sIdentifiers(ns.Name, "Deployment", "loopback") + require.NoError(t, err, "resolve loopback") + require.NoError(t, lb.WaitForReady(80), "loopback ready") + loop := func() { + for i := 0; i < 5; i++ { + _, _, _ = lb.ExecIntoPod([]string{"curl", "-sS", "-m", "5", "http://127.0.0.1:80/"}, "loopclient") + time.Sleep(2 * time.Second) + } + } + loop() + require.NoError(t, lb.WaitForContainerProfileCompletion(100), "loopback CP completed") + time.Sleep(30 * time.Second) + learn11 := countRuleAlerts(t, ns.Name, "R0011", "loopclient", "") + learn12 := countRuleAlerts(t, ns.Name, "R0012", "loopserver", "") + loop() + time.Sleep(30 * time.Second) + replay11 := countRuleAlerts(t, ns.Name, "R0011", "loopclient", "") + replay12 := countRuleAlerts(t, ns.Name, "R0012", "loopserver", "") + t.Logf("LOOPBACK FP measurement (guards removed): R0011(loopclient) learn=%d replay=%d | R0012(loopserver) learn=%d replay=%d", learn11, replay11, learn12, replay12) + assert.Equal(t, learn11, replay11, "replayed loopback egress must not add R0011 once learned") + assert.Equal(t, learn12, replay12, "replayed loopback ingress must not add R0012 once learned") + }) +} diff --git a/tests/resources/containerprofile-user-defined-network.yaml b/tests/resources/containerprofile-user-defined-network.yaml index f2f6edda1c..1a075941de 100644 --- a/tests/resources/containerprofile-user-defined-network.yaml +++ b/tests/resources/containerprofile-user-defined-network.yaml @@ -58,3 +58,36 @@ spec: - name: TCP-80 protocol: TCP port: 80 + # R0011 excludes only loopback (maximally noisy by design): allow the pod's + # own DNS egress to cluster DNS or every nslookup/curl resolution mints R0011. + - identifier: cluster-dns + type: internal + ipAddresses: + - 10.96.0.0/12 + ports: + - name: UDP-53 + protocol: UDP + port: 53 + - name: TCP-53 + protocol: TCP + port: 53 + - identifier: wildcard-zero-port + type: external + ipAddress: 9.9.9.9 + ports: + - name: TCP-any + protocol: TCP + port: 0 + - identifier: wildcard-empty-ports + type: external + ipAddress: 208.67.222.222 + - identifier: cluster-dns-clusterip + type: internal + ipAddress: 10.96.0.10 + - identifier: kube-api + type: internal + ipAddress: 10.96.0.1 + ports: + - name: TCP-443 + protocol: TCP + port: 443 diff --git a/tests/resources/network-default-client.yaml b/tests/resources/network-default-client.yaml new file mode 100644 index 0000000000..e61aad5948 --- /dev/null +++ b/tests/resources/network-default-client.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: default-net-client + name: default-net-client +spec: + selector: + matchLabels: + app: default-net-client + replicas: 1 + template: + metadata: + labels: + app: default-net-client + spec: + containers: + - name: curl + image: docker.io/curlimages/curl@sha256:08e466006f0860e54fc299378de998935333e0e130a15f6f98482e9f8dab3058 + command: ["sleep", "infinity"] diff --git a/tests/resources/network-hostnet-client.yaml b/tests/resources/network-hostnet-client.yaml new file mode 100644 index 0000000000..1dfe1d9e76 --- /dev/null +++ b/tests/resources/network-hostnet-client.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hostnet-client + name: hostnet-client +spec: + selector: + matchLabels: + app: hostnet-client + replicas: 1 + template: + metadata: + labels: + app: hostnet-client + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: curl + image: docker.io/curlimages/curl@sha256:08e466006f0860e54fc299378de998935333e0e130a15f6f98482e9f8dab3058 + command: ["sleep", "infinity"] diff --git a/tests/resources/network-loopback-pod.yaml b/tests/resources/network-loopback-pod.yaml new file mode 100644 index 0000000000..4898b5d334 --- /dev/null +++ b/tests/resources/network-loopback-pod.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: loopback + name: loopback +spec: + selector: + matchLabels: + app: loopback + replicas: 1 + template: + metadata: + labels: + app: loopback + spec: + containers: + - name: loopserver + image: nginx:1.14.2 + ports: + - containerPort: 80 + - name: loopclient + image: docker.io/curlimages/curl@sha256:08e466006f0860e54fc299378de998935333e0e130a15f6f98482e9f8dab3058 + command: ["sleep", "infinity"] diff --git a/tests/resources/network_fixture_lint_test.go b/tests/resources/network_fixture_lint_test.go index a7ec6cd0e2..3251d320d0 100644 --- a/tests/resources/network_fixture_lint_test.go +++ b/tests/resources/network_fixture_lint_test.go @@ -74,6 +74,11 @@ type netEndpoint struct { // declared target for R-NN-12. PodSelector json.RawMessage `json:"podSelector"` NamespaceSelector json.RawMessage `json:"namespaceSelector"` + // Service/entity targets resolved to concrete IPs at projection time. + // Presence of any counts as a declared target for R-NN-12. + ServiceRefName string `json:"serviceRefName"` + ServiceSelector json.RawMessage `json:"serviceSelector"` + Entity string `json:"entity"` } // hasSelector reports whether a raw selector field was set to a real object @@ -114,7 +119,7 @@ func (v NetViolation) String() string { // R-NN-02 — at least one endpoint (egress or ingress) declared // R-NN-10 — endpoint identifier non-empty // R-NN-11 — endpoint type in {internal, external} (or unset) -// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress) +// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity) // R-NN-13 — dnsNames wildcard tokens are whole-label; no recursive "**", no ascii "..." // R-NN-14 — an entry MUST NOT set both singular ipAddress and plural ipAddresses // R-NN-15 — ipAddresses entries are a literal IP, a CIDR, or the "*" sentinel @@ -176,8 +181,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { add("R-NN-11", where(fmt.Sprintf("type %q is not internal|external", e.Type))) } if len(e.DNSNames) == 0 && len(e.IPAddresses) == 0 && e.DNS == "" && e.IPAddress == "" && - !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) { - add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector)")) + !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) && + e.ServiceRefName == "" && !hasSelector(e.ServiceSelector) && e.Entity == "" { + add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity)")) } if e.IPAddress != "" && len(e.IPAddresses) > 0 { add("R-NN-14", where("sets both singular ipAddress and plural ipAddresses — pick one")) @@ -205,8 +211,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { if p.Protocol != "TCP" && p.Protocol != "UDP" { add("R-NN-20", where(fmt.Sprintf("port %q protocol %q is not TCP|UDP", p.Name, p.Protocol))) } - if p.Port < 1 || p.Port > 65535 { - add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 1..65535", p.Name, p.Port))) + // Port 0 is a literal (an explicit 0 entry); the any-port wildcard is an absent ports stanza, not port 0 (matches R0011/R0012 port semantics). + if p.Port < 0 || p.Port > 65535 { + add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 0..65535", p.Name, p.Port))) } } } diff --git a/tests/resources/serviceref-k6.yaml b/tests/resources/serviceref-k6.yaml new file mode 100644 index 0000000000..6cd3ae0ae8 --- /dev/null +++ b/tests/resources/serviceref-k6.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k6-script +data: + load.js: | + import http from 'k6/http'; + import { sleep } from 'k6'; + export const options = { vus: 2, duration: '5m' }; + export default function () { + http.get('http://helm-primary/index.yaml', { timeout: '5s' }); + sleep(1); + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k6-load +spec: + replicas: 1 + selector: + matchLabels: + app: k6-load + template: + metadata: + labels: + app: k6-load + spec: + containers: + - name: k6 + image: grafana/k6:0.49.0 + args: ["run", "/scripts/load.js", "--quiet", "--no-usage-report"] + volumeMounts: + - name: scripts + mountPath: /scripts + resources: + limits: + cpu: 200m + memory: 192Mi + volumes: + - name: scripts + configMap: + name: k6-script diff --git a/tests/resources/serviceref-suite/00-flux-source-crds.yaml b/tests/resources/serviceref-suite/00-flux-source-crds.yaml new file mode 100644 index 0000000000..bfc14c9c8d --- /dev/null +++ b/tests/resources/serviceref-suite/00-flux-source-crds.yaml @@ -0,0 +1,132 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmRepository + listKind: HelmRepositoryList + plural: helmrepositories + singular: helmrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmcharts.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmChart + listKind: HelmChartList + plural: helmcharts + singular: helmchart + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: GitRepository + listKind: GitRepositoryList + plural: gitrepositories + singular: gitrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: buckets.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: Bucket + listKind: BucketList + plural: buckets + singular: bucket + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ocirepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: OCIRepository + listKind: OCIRepositoryList + plural: ocirepositories + singular: ocirepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/tests/resources/serviceref-suite/10-helm-repo.yaml b/tests/resources/serviceref-suite/10-helm-repo.yaml new file mode 100644 index 0000000000..830be5de98 --- /dev/null +++ b/tests/resources/serviceref-suite/10-helm-repo.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: helm-repo-index +data: + index.yaml: | + apiVersion: v1 + entries: {} + generated: "2020-01-01T00:00:00Z" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helm-repo +spec: + replicas: 1 + selector: + matchLabels: + app: helm-repo + template: + metadata: + labels: + app: helm-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-primary + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-mirror + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: decoy-repo +spec: + replicas: 1 + selector: + matchLabels: + app: decoy-repo + template: + metadata: + labels: + app: decoy-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: decoy-repo +spec: + selector: + app: decoy-repo + ports: + - port: 80 + targetPort: 80 diff --git a/tests/resources/serviceref-suite/20-source-controller.yaml b/tests/resources/serviceref-suite/20-source-controller.yaml new file mode 100644 index 0000000000..613fcaae50 --- /dev/null +++ b/tests/resources/serviceref-suite/20-source-controller.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: source-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: source-controller +rules: + - apiGroups: ["source.toolkit.fluxcd.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: [""] + resources: ["configmaps", "secrets", "events", "serviceaccounts"] + verbs: ["*"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: source-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: source-controller +subjects: + - kind: ServiceAccount + name: source-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: source-controller +spec: + selector: + app: source-controller + ports: + - name: http + port: 80 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: source-controller +spec: + replicas: 1 + selector: + matchLabels: + app: source-controller + strategy: + type: Recreate + template: + metadata: + labels: + app: source-controller + kubescape.io/user-defined-profile: serviceref-flux-cp + spec: + serviceAccountName: source-controller + terminationGracePeriodSeconds: 10 + containers: + - name: manager + image: ghcr.io/fluxcd/source-controller:v1.3.0 + imagePullPolicy: IfNotPresent + args: + - --log-level=info + - --log-encoding=json + - --storage-path=/data + - --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. + - --watch-all-namespaces=false + - --enable-leader-election=false + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TUF_ROOT + value: /tmp/.sigstore + ports: + - containerPort: 9090 + name: http + - containerPort: 9440 + name: healthz + readinessProbe: + httpGet: + path: / + port: http + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /data + name: data + - mountPath: /tmp + name: tmp + securityContext: + fsGroup: 1337 + volumes: + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/tests/testutils/k8s.go b/tests/testutils/k8s.go index b2792d04b9..e515a8f8b6 100644 --- a/tests/testutils/k8s.go +++ b/tests/testutils/k8s.go @@ -1,6 +1,7 @@ package testutils import ( + "bufio" "bytes" "context" "encoding/json" @@ -10,6 +11,7 @@ import ( "math/rand" "os" "path/filepath" + "sort" "strings" "testing" "time" @@ -28,10 +30,16 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + apimachineryyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/scheme" ) @@ -84,6 +92,83 @@ func NewTestWorkload(namespace, resourcePath string) (*TestWorkload, error) { }, nil } +// ApplyMultiDocYAML creates every document in a multi-document YAML file, +// mapping each object's apiVersion/kind to its resource via server discovery so +// custom resources work without a compiled-in table. Namespaced objects land in +// namespace; cluster-scoped ones (CRDs) ignore it. Already-existing objects are +// not an error, so a fixture may be applied by more than one test. +func ApplyMultiDocYAML(namespace, resourcePath string) error { + k8sClient := k8sinterface.NewKubernetesApi() + raw, err := os.ReadFile(resourcePath) + if err != nil { + return err + } + dc, err := discovery.NewDiscoveryClientForConfig(k8sClient.K8SConfig) + if err != nil { + return err + } + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + reader := apimachineryyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(raw))) + for { + doc, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + if strings.TrimSpace(string(doc)) == "" { + continue + } + jsonData, err := yaml.YAMLToJSON(doc) + if err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + obj := &unstructured.Unstructured{} + if err := obj.UnmarshalJSON(jsonData); err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + if obj.GetKind() == "" { + continue + } + gvk := obj.GroupVersionKind() + m, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("%s %s: %w", resourcePath, gvk.String(), err) + } + var ri dynamic.ResourceInterface = k8sClient.DynamicClient.Resource(m.Resource) + if m.Scope.Name() == meta.RESTScopeNameNamespace { + ri = k8sClient.DynamicClient.Resource(m.Resource).Namespace(namespace) + } + if _, err := ri.Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("%s %s/%s: %w", resourcePath, obj.GetKind(), obj.GetName(), err) + } + } + return nil +} + +// ApplyMultiDocDir applies every YAML file in dir in lexical order (filenames +// are numbered so CRDs land before the resources that use them). +func ApplyMultiDocDir(namespace, dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() { + names = append(names, e.Name()) + } + } + sort.Strings(names) + for _, n := range names { + if err := ApplyMultiDocYAML(namespace, filepath.Join(dir, n)); err != nil { + return err + } + } + return nil +} + func (w *TestWorkload) ExecIntoPod(command []string, container string) (string, string, error) { pods, err := w.GetPods() if err != nil { @@ -690,21 +775,27 @@ func RestartDaemonSet(namespace, name string) error { daemonset.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) // Update the daemonset - _, err = k8sClient.KubernetesClient.AppsV1().DaemonSets(namespace).Update(ctx, daemonset, metav1.UpdateOptions{}) + applied, err := k8sClient.KubernetesClient.AppsV1().DaemonSets(namespace).Update(ctx, daemonset, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update daemonset %s/%s: %w", namespace, name, err) } + newGen := applied.Generation - // Wait for the daemonset to be ready + // Wait for the rollout to ACTUALLY complete. The ObservedGeneration gate is + // essential: immediately after Update the old pod is still ready and counted + // as updated, so NumberReady/UpdatedNumberScheduled both equal Desired and the + // checks pass on the pre-restart status — the pod never actually cycles. Only + // once the controller has observed the new generation do the ready/updated + // counts reflect the new pod template. err = backoff.RetryNotify(func() error { updatedDS, err := k8sClient.KubernetesClient.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } - if updatedDS.Status.NumberReady != updatedDS.Status.DesiredNumberScheduled { - return fmt.Errorf("daemonset %s/%s not ready: %d/%d pods ready", - namespace, name, updatedDS.Status.NumberReady, updatedDS.Status.DesiredNumberScheduled) + if updatedDS.Status.ObservedGeneration < newGen { + return fmt.Errorf("daemonset %s/%s rollout not observed yet: observedGeneration %d < %d", + namespace, name, updatedDS.Status.ObservedGeneration, newGen) } if updatedDS.Status.UpdatedNumberScheduled != updatedDS.Status.DesiredNumberScheduled { @@ -712,8 +803,13 @@ func RestartDaemonSet(namespace, name string) error { namespace, name, updatedDS.Status.UpdatedNumberScheduled, updatedDS.Status.DesiredNumberScheduled) } + if updatedDS.Status.NumberReady != updatedDS.Status.DesiredNumberScheduled { + return fmt.Errorf("daemonset %s/%s not ready: %d/%d pods ready", + namespace, name, updatedDS.Status.NumberReady, updatedDS.Status.DesiredNumberScheduled) + } + return nil - }, backoff.WithMaxRetries(backoff.NewConstantBackOff(5*time.Second), 30), func(err error, d time.Duration) { + }, backoff.WithMaxRetries(backoff.NewConstantBackOff(5*time.Second), 60), func(err error, d time.Duration) { logger.L().Info("waiting for daemonset to be ready", helpers.String("daemonset", name), helpers.String("namespace", namespace),