diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4584d725f9b..c8bda4fbeac 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -456,7 +456,10 @@ REPOSITORY_LOCATIONS = dict( org_libc_musl = dict( sha256 = "7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4", strip_prefix = "musl-1.2.3", - urls = ["http://musl.libc.org/releases/musl-1.2.3.tar.gz"], + urls = [ + "https://sources.openwrt.org/musl-1.2.3.tar.gz", + "http://musl.libc.org/releases/musl-1.2.3.tar.gz", + ], manual_license_name = "libc/musl", ), rules_cc = dict( diff --git a/ci/artifact_utils.sh b/ci/artifact_utils.sh index a1eec1a7760..e6d7c0dca26 100644 --- a/ci/artifact_utils.sh +++ b/ci/artifact_utils.sh @@ -107,7 +107,15 @@ create_manifest_update() { tag_name="release/${component}/v${version}" # actions/checkout doesn't get the tag annotation properly. git fetch origin tag "${tag_name}" -f - timestamp="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}' | jq '. | todate')" + # taggerdate is empty for a LIGHTWEIGHT tag → produces `timestamp: ,` → jq syntax + # error → release-metadata step fails even though the image built fine. Fall back to + # the tagged commit's committer date so the manifest is well-formed regardless of how + # the release tag was cut (annotated vs lightweight). + raw_ts="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}')" + if [ -z "${raw_ts}" ]; then + raw_ts="$(git log -1 --format="%ct" "${tag_name}")" + fi + timestamp="$(printf '%s' "${raw_ts}" | jq '. | todate')" jq -s \ "[{name: \"${component}\", artifact: [{timestamp: ${timestamp}, commitHash: \"${commit_hash}\", versionStr: \"${version}\", availableArtifactMirrors: .}]}]" \ diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml new file mode 100644 index 00000000000..b9efc921773 --- /dev/null +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pl +resources: + - ../bootstrap/adaptive_export_role.yaml + - ../bootstrap/adaptive_export_deployment.yaml +images: + - name: vizier-adaptive_export_image + newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image + newTag: 0.14.19-aeprod90 diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index 2db195ff408..251ceed69ce 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -1,115 +1,97 @@ --- +# adaptive-export: node-local forensic capture operator. DaemonSet so each pod +# queries its own node's vizier-pem (pem-direct). Secret seeded per-cluster. apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: adaptive-export + labels: { name: adaptive-export, plane: control } spec: - replicas: 0 selector: - matchLabels: - name: adaptive-export + matchLabels: { name: adaptive-export } template: metadata: - labels: - name: adaptive-export - plane: control + labels: { name: adaptive-export, plane: control } spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - # The beta.kubernetes.io/os label has been deprecated since - # k8s v1.14; every modern kubelet sets kubernetes.io/os. The - # single term below is enough — kept both ORed terms in the - # past for pre-1.14 compatibility. - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux + - { key: kubernetes.io/os, operator: In, values: [linux] } serviceAccountName: pl-adaptive-export-service-account containers: - name: adaptive-export image: vizier-adaptive_export_image:latest - # Bounded so AE can never memory-pressure a node (measured: AE uses - # only ~16-38Mi steady; passthrough with the raised 1M-row cap can - # spike, so 1Gi caps the worst case). CPU was pinned at the old 300m - # limit under concurrent passthrough → raised to 1 core. + ports: + - { name: control, containerPort: 9100, hostPort: 9100 } resources: - requests: - cpu: 200m - memory: 128Mi - limits: - cpu: "1" - memory: 1Gi + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: "1", memory: 1Gi } env: + - name: HOST_IP + valueFrom: { fieldRef: { fieldPath: status.hostIP } } + - name: ADAPTIVE_VIZIER_DIRECT_ADDR + value: "$(HOST_IP):50305" + - name: PL_JWT_SIGNING_KEY + valueFrom: { secretKeyRef: { name: pl-cluster-secrets, key: jwt-signing-key } } + - name: PX_DISABLE_TLS + value: "1" - name: PL_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } - name: PIXIE_API_KEY - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: pixie-api-key + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: pixie-api-key } } - name: CLICKHOUSE_DSN - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: clickhouse-dsn - - name: VERBOSE - value: "true" - - name: DETECTION_INTERVAL_SEC - value: "10" - - name: DETECTION_LOOKBACK_SEC - value: "30" - # EXPORT_MODE controls the reconcile behaviour: - # auto - detection drives on/off (default) - # always - plugin always enabled (bypass detection) - # never - plugin always disabled and ch-* scripts purged - - name: EXPORT_MODE - value: "auto" - # Number of consecutive empty detection ticks before auto-disable fires. - - name: EXPORT_QUIET_TICKS - value: "6" - # Optional overrides for the ClickHouse PxL scripts. When unset they are - # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN. - # Defaults below match soc/tree/clickhouse-lab (forensic-soc-db CHI, - # ingest_writer user, forensic_db database). + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: clickhouse-dsn } } - name: KUBESCAPE_TABLE value: "kubescape_logs" - # - name: CLICKHOUSE_HOST - # value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local" - # - name: CLICKHOUSE_PORT - # value: "9000" - # - name: CLICKHOUSE_USER - # value: "ingest_writer" - # - name: CLICKHOUSE_PASSWORD - # value: "changeme-ingest" - # - name: CLICKHOUSE_DATABASE - # value: "forensic_db" - # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the - # same service-tls-certs secret the broker/PEM use; without this the dx - # bearer JWT crosses the CNI in cleartext. Harmless when control is off. + - name: EXPORT_MODE + value: "never" + # Control surface is secure-by-default (#96): TLS + bearer-JWT auth are ON + # out of the box. The service-tls-certs keypair mounted at /certs below is + # used for TLS (else AE self-generates an ephemeral in-memory cert), and + # PL_JWT_SIGNING_KEY above turns on auth. CONTROL_TLS / CONTROL_REQUIRE_AUTH + # are deprecated no-ops; set CONTROL_INSECURE=true only to opt out (dev). + - name: CONTROL_ADDR + value: ":9100" + - name: ADAPTIVE_PUSH_PIXIE_ROWS + value: "true" + - name: ADAPTIVE_RECONCILE + value: "true" + - name: DEPLOY_TRACEPOINTS + value: "true" + - name: INSTALL_PRESET_SCRIPTS + value: "false" + - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL + value: "4" + - name: ADAPTIVE_ORDER_CHUNK_SEC + value: "600" + - name: VERBOSE + value: "true" volumeMounts: - - name: certs - mountPath: /certs - readOnly: true + - { name: certs, mountPath: /certs, readOnly: true } securityContext: allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault + capabilities: { drop: [ALL] } + seccompProfile: { type: RuntimeDefault } volumes: - name: certs - secret: - secretName: service-tls-certs + secret: { secretName: service-tls-certs } securityContext: runAsUser: 10100 runAsGroup: 10100 fsGroup: 10100 runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + seccompProfile: { type: RuntimeDefault } +--- +apiVersion: v1 +kind: Service +metadata: + name: adaptive-export-control +spec: + selector: { name: adaptive-export } + internalTrafficPolicy: Local # dx reaches its co-located (same-node) AE + ports: + - { name: control, port: 9100, targetPort: 9100 } diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml new file mode 100644 index 00000000000..29d3a15b43a --- /dev/null +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: { name: dx-daemon, namespace: honey } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dx-daemon + namespace: honey + labels: { app: dx-daemon } +spec: + selector: { matchLabels: { app: dx-daemon } } + template: + metadata: + labels: { app: dx-daemon } + spec: + serviceAccountName: dx-daemon + tolerations: [{ operator: Exists }] + terminationGracePeriodSeconds: 35 + containers: + - name: dx-daemon + # OBFUSCATED release rc13 (entlein/dx#138 fault 2 RESOLVED): garble -literals + # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the + # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the + # evidence-manifest + DX_FOREST_PUSHDOWN code. + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc23 + ports: + - { name: findings, containerPort: 9099, hostPort: 9099 } + env: + - { name: NODE_NAME, valueFrom: { fieldRef: { fieldPath: spec.nodeName } } } + - { name: HOST_IP, valueFrom: { fieldRef: { fieldPath: status.hostIP } } } + - { name: DX_RECEIVER_TLS, value: "1" } + # AE control surface is TLS-by-default (#96); the dx client TLS-skip-verifies + # the in-cluster (self-signed/shared) cert and attaches its bearer JWT. + - { name: AE_CONTROL_ADDR, value: "https://adaptive-export-control.pl.svc.cluster.local:9100" } + - { name: PX_API_KEY, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: api-key, optional: true } } } + - { name: PX_CLUSTER_ID, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cluster-id, optional: true } } } + - { name: PX_CLOUD_ADDR, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cloud-addr, optional: true } } } + - { name: DX_BENCH, value: "pemdirect" } + - { name: PL_JWT_SIGNING_KEY, valueFrom: { secretKeyRef: { name: dx-vizier-direct, key: jwt-signing-key, optional: true } } } + - { name: DX_VIZIER_DIRECT_ADDR, value: "vizier-query-broker-svc.pl.svc.cluster.local:50300" } + - { name: PX_DISABLE_TLS, value: "1" } + - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } } + - { name: DX_PX_TIMEOUT_S, value: "90" } + - { name: DX_TELEMETRY_CACHE, value: "1" } + - { name: DX_WORKERS, value: "4" } + # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, + # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). + - { name: DX_FOREST_SCOPE, value: "1" } + # FOREST_PUSHDOWN (entlein/dx#138 fault 3): push the dc_snoop ppid-lineage filter + # INTO the PxL so dx pulls only the alert pod's subtree, not the whole node — + # frees the node-local PEM so AE can export dc_snoop under load (validated: 0→1777). + - { name: DX_FOREST_PUSHDOWN, value: "1" } + - { name: DX_FOREST_PUSHDOWN_DEPTH, value: "4" } + - { name: DX_PRECORRELATE_GRAPH, value: "1" } + - { name: DX_NARROW_WINDOW_BEFORE_MS, value: "150" } + - { name: DX_NARROW_WINDOW_AFTER_MS, value: "150" } # server-side protocol capture lands just after the anomaly + - { name: DX_EVIDENCE_GRAPH_CH, value: "http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db" } + readinessProbe: + httpGet: { path: /healthz, port: 9099, scheme: HTTPS } + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) + pemdirect + # gRPC result streams pull the per-anomaly evidence set into memory; at 1Gi dx is + # OOM-killed mid-workup (exit 137) BEFORE it writes the graph → crash-loop, empty + # graph. Measured peak ~1.3GB/round under the redis kill-chain (entlein/dx#138 + # fault 1); 3Gi clears it reliably on an 8GiB node (validated: restarts=0 over 6+ rounds). + requests: { cpu: 50m, memory: 1Gi } + limits: { cpu: "2", memory: 3Gi } +--- +apiVersion: v1 +kind: Service +metadata: + name: dx-daemon + namespace: honey +spec: + selector: { app: dx-daemon } + internalTrafficPolicy: Local + ports: + - { name: findings, port: 9099, targetPort: 9099 } diff --git a/k8s/vizier/dx/kustomization.yaml b/k8s/vizier/dx/kustomization.yaml new file mode 100644 index 00000000000..7e7edebde60 --- /dev/null +++ b/k8s/vizier/dx/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: honey +resources: + - dx-daemon.yaml diff --git a/private/cockpit/cloud_ingress.yaml b/private/cockpit/cloud_ingress.yaml index 2a89929844c..14651d50680 100644 --- a/private/cockpit/cloud_ingress.yaml +++ b/private/cockpit/cloud_ingress.yaml @@ -6,13 +6,15 @@ metadata: namespace: plc annotations: external-dns.alpha.kubernetes.io/hostname: >- - test.austrianopencloudcommunity.org,work.test.austrianopencloudcommunity.org + test.austrianopencloudcommunity.org,work.test.austrianopencloudcommunity.org,soc.k8sstormcenter.com,work.soc.k8sstormcenter.com cert-manager.io/cluster-issuer: "letsencrypt-prod" spec: tls: - hosts: - test.austrianopencloudcommunity.org - work.test.austrianopencloudcommunity.org + - soc.k8sstormcenter.com + - work.soc.k8sstormcenter.com secretName: cloud-proxy-tls-certs rules: - host: test.austrianopencloudcommunity.org @@ -77,3 +79,65 @@ spec: name: cloud-proxy-service port: number: 443 + - host: soc.k8sstormcenter.com + http: + paths: + - path: /px.services + pathType: Prefix + backend: + service: + name: vzconn-service + port: + number: 51600 + - path: /px.cloudapi + pathType: Prefix + backend: + service: + name: api-service + port: + number: 51200 + - path: /px.api + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 4444 + - path: / + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 443 + - host: work.soc.k8sstormcenter.com + http: + paths: + - path: /px.services + pathType: Prefix + backend: + service: + name: vzconn-service + port: + number: 51600 + - path: /px.cloudapi + pathType: Prefix + backend: + service: + name: api-service + port: + number: 51200 + - path: /px.api + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 4444 + - path: / + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 443 diff --git a/skaffold/skaffold_adaptive_export.yaml b/skaffold/skaffold_adaptive_export.yaml new file mode 100644 index 00000000000..b04d8550990 --- /dev/null +++ b/skaffold/skaffold_adaptive_export.yaml @@ -0,0 +1,44 @@ +--- +# Deploy-only Skaffold for the adaptive_export DaemonSet using a prebuilt image +# (lab / review), overlaying an already-running vizier. Run from the repo root: +# skaffold deploy -f skaffold/skaffold_adaptive_export.yaml +# Bump the image via newTag in k8s/vizier/adaptive_export/kustomization.yaml. +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: adaptive-export +manifests: + kustomize: + paths: + - k8s/vizier/adaptive_export + buildArgs: + - --load-restrictor=LoadRestrictionsNone +deploy: + kubectl: + defaultNamespace: pl + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client crashloops. + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true) + case "$CA" in ""|*:*) ;; *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}";; esac + # Seed pl-adaptive-export-secrets ONLY when a key is supplied; never clobber a good secret with an empty one. + API="${PIXIE_API_KEY:-${PX_API_KEY:-}}" + CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db}" + if [ -n "$API" ]; then + kubectl -n pl create secret generic pl-adaptive-export-secrets \ + --from-literal=pixie-api-key="$API" \ + --from-literal=clickhouse-dsn="$CH_DSN" \ + --dry-run=client -o yaml | kubectl apply -f - + elif ! kubectl -n pl get secret pl-adaptive-export-secrets >/dev/null 2>&1; then + echo "ERROR: set PIXIE_API_KEY (or source keys.env) to seed pl-adaptive-export-secrets" >&2 + exit 1 + else + echo "pl-adaptive-export-secrets exists; PIXIE_API_KEY unset -> leaving it untouched" + fi + os: [linux, darwin] diff --git a/skaffold/skaffold_dx.yaml b/skaffold/skaffold_dx.yaml new file mode 100644 index 00000000000..ea3ee0c7e14 --- /dev/null +++ b/skaffold/skaffold_dx.yaml @@ -0,0 +1,42 @@ +--- +# Deploy-only Skaffold for the dx-daemon DaemonSet (prebuilt image), overlaying an +# already-running vizier + soc stack. Run from the repo root, AFTER adaptive_export +# (the hook mirrors pl-adaptive-export-secrets into honey): +# skaffold deploy -f skaffold/skaffold_dx.yaml +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: dx-daemon +manifests: + kustomize: + paths: + - k8s/vizier/dx +deploy: + kubectl: + defaultNamespace: honey + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + kubectl create namespace honey --dry-run=client -o yaml | kubectl apply -f - + JWT=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.jwt-signing-key}' | base64 -d) + CID=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.cluster-id}' | base64 -d) + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}') + API=$(kubectl -n pl get secret pl-adaptive-export-secrets -o jsonpath='{.data.pixie-api-key}' 2>/dev/null | base64 -d) + CH_URL="${DX_CH_HTTP_URL:-http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/?database=forensic_db}" + kubectl -n honey create secret generic dx-vizier-direct \ + --from-literal=jwt-signing-key="$JWT" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-pixie-auth \ + --from-literal=api-key="$API" \ + --from-literal=cluster-id="$CID" \ + --from-literal=cloud-addr="$CA" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-metastasis-ch \ + --from-literal=http-url="$CH_URL" \ + --dry-run=client -o yaml | kubectl apply -f - + os: [linux, darwin] diff --git a/src/pxl_scripts/Makefile b/src/pxl_scripts/Makefile index 1cca03f4dc5..4e8a3562658 100644 --- a/src/pxl_scripts/Makefile +++ b/src/pxl_scripts/Makefile @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # Update dir name here if you want to add a new directory. -dirs := bpftrace px pxbeta sotw +dirs := bpftrace dx px pxbeta sotw script_files := $(foreach dir,$(dirs),$(wildcard $(dir)/**/*)) EXECUTABLES ?= px diff --git a/src/pxl_scripts/dx/breakout/breakout.pxl b/src/pxl_scripts/dx/breakout/breakout.pxl new file mode 100644 index 00000000000..4400adaf583 --- /dev/null +++ b/src/pxl_scripts/dx/breakout/breakout.pxl @@ -0,0 +1,67 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Runtime-breakout story — workload-agnostic; ClickHouse only. +# +# The DB view (dx_alerts) is deliberately GENERIC: a thin flatten of +# kubescape_logs (pod/namespace/rule/message/sev), reusable by any narrative. +# ALL the story logic — how a message becomes a target node and a kind — lives +# HERE in PxL, so you can fumble with it locally without touching the DB/AE. +# Edit _alerts() to reshape the story; the DDL never has to change. + +import px + + +def _alerts(start_time: str, clickhouse_dsn: str, namespace: str): + a = px.DataFrame('dx_alerts', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + a = a[px.contains(a.namespace, namespace)] + # target = the tail after the last ": " or "to "/"to: " in the alert message. + a.target = px.replace('^.*(: |to:? )', a.message, '') + # collapse the timestamped serviceaccount-token path to one node. + a.target = px.select(px.contains(a.target, 'serviceaccount'), 'serviceaccount/token', a.target) + # kind = coarse rule map, refined for R0002 file sub-types. Extend freely. + a.kind = px.select(a.rule == 'R0001', 'process', + px.select(a.rule == 'R0011', 'egress', + px.select(a.rule == 'R0012', 'ingress', + px.select(a.rule == 'R0005', 'dns', + px.select(a.rule == 'R0004', 'capability', + px.select(px.contains(a.message, 'serviceaccount'), 'token-read', + px.select(px.contains(a.message, '.so'), 'libload', + px.select(px.contains(a.message, '/proc/'), 'proc-read', + px.select(px.contains(a.message, '/tmp'), 'tmp-write', + 'file-access'))))))))) + return a + + +def story(start_time: str, clickhouse_dsn: str, namespace: str): + a = _alerts(start_time, clickhouse_dsn, namespace) + g = a.groupby(['namespace', 'pod', 'target', 'kind', 'rule']).agg( + events=('sev', px.count), sev=('sev', px.max)) + g.from_entity = g.pod + g.to_entity = g.target + return g[['from_entity', 'to_entity', 'kind', 'rule', 'events', 'sev', 'namespace']] + + +def beats(start_time: str, clickhouse_dsn: str, namespace: str): + a = _alerts(start_time, clickhouse_dsn, namespace) + s = a.groupby(['namespace', 'kind', 'rule']).agg(events=('sev', px.count), sev=('sev', px.max)) + return s[['namespace', 'kind', 'rule', 'sev', 'events']] + + +def targets(start_time: str, clickhouse_dsn: str, namespace: str): + a = _alerts(start_time, clickhouse_dsn, namespace) + s = a.groupby(['namespace', 'kind', 'target']).agg(events=('sev', px.count)) + return s[['namespace', 'kind', 'target', 'events']] diff --git a/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl b/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl new file mode 100644 index 00000000000..cc6956aa694 --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl @@ -0,0 +1,53 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# SOC DNS resolution reconstruction — works only with ClickHouse enabled. + +import px + + +def dns_graph(start_time: str, clickhouse_dsn: str, order_id: str): + # DNS events with a non-empty resp_body in the order's ±window, exploded into + # resolution edges by dx_dns_resolve (querier->resolver, then the answer tree + # name->CNAME and name->A). Time-windowed, NOT edge-linked: resolution runs on + # coredns / cluster-DNS, not the attacked pod. k=1 cross join pairs the single + # window row with every DNS edge, then filters by event_time (ns). + win = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + win = win[win.order_id == order_id] + win = win[['lo', 'hi']] + win.k = 1 + d = px.DataFrame('dx_dns_resolve', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + d.k = 1 + d = d.merge(win, how='inner', left_on=['k'], right_on=['k'], suffixes=['', '_w']) + d = d[d.event_time >= d.lo] + d = d[d.event_time <= d.hi] + + # Resolve to a k8s identity so the hops CONNECT: the client's target + # (remote_addr, e.g. 10.43.0.10) and the coredns pod that forwards upstream + # both collapse to the same service node (kube-system/kube-dns). Only query + # edges carry an "ip:53" target / a pod querier; answer edges pass through. + d.resolver_ip = px.replace(':53', d.to_node, '') + d.r_svc = px.service_id_to_service_name(px.ip_to_service_id(d.resolver_ip)) + d.r_nsl = px.nslookup(d.resolver_ip) + d.resolver = px.select(d.r_svc != '', d.r_svc, d.r_nsl) + d.q_svc = px.pod_name_to_service_name(d.from_node) + d.querier = px.select(d.q_svc != '', d.q_svc, d.from_node) + + d.from_entity = px.select(d.kind == 'query', d.querier, d.from_node) + d.to_entity = px.select(d.kind == 'query', d.resolver, d.to_node) + # ts: human-readable UTC datetime with ns precision (toString of the DateTime64 + # in the dx_dns_resolve view); event_time stays int64-ns for the window filter. + return d[['from_entity', 'to_entity', 'edge_label', 'kind', 'ts']] diff --git a/src/pxl_scripts/dx/dns_resolve/manifest.yaml b/src/pxl_scripts/dx/dns_resolve/manifest.yaml new file mode 100644 index 00000000000..72fb0384e68 --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/manifest.yaml @@ -0,0 +1,4 @@ +--- +short: SOC DNS Resolution +long: > + SOC pixie, DNS resolution reconstruction per order; works only with clickhouse enabled. diff --git a/src/pxl_scripts/dx/dns_resolve/vis.json b/src/pxl_scripts/dx/dns_resolve/vis.json new file mode 100644 index 00000000000..7d6d5f80a9d --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/vis.json @@ -0,0 +1,14 @@ +{ + "variables": [ + {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, + {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, + {"name": "order_id", "type": "PX_STRING", "description": "Order to reconstruct DNS for; deep-linked from the evidence graph or set here.", "defaultValue": ""} + ], + "globalFuncs": [ + {"outputName": "g_dns", "func": {"name": "dns_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}} + ], + "widgets": [ + {"name": "DNS resolution (querier -> resolver:53, then name -> CNAME -> A; edges in the alert window)", "position": {"x": 0, "y": 0, "w": 12, "h": 5}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeLabelColumn": "edge_label", "edgeHoverInfo": ["edge_label", "kind", "ts"], "edgeLength": 400}}, + {"name": "DNS edges (exact resolution rows consulted)", "position": {"x": 0, "y": 5, "w": 12, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} + ] +} diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl new file mode 100644 index 00000000000..c1275bb6211 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -0,0 +1,165 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# SOC pixie evidence graph — works only with ClickHouse enabled. + +import px + + +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['order_id', 'kubescape_uid', 'rule_id', 'pod']] + anom = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = orders.merge(anom, how='inner', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_a']) + df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + }) + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target + return df[['from_entity', 'to_entity', 'order_link', 'rule_mitre', 'mitre_tactic', + 'mitre_technique', 'order_id', 'rule', 'process', 'target', 'target_kind', + 'severity', 'alert', 'subject_pod']] + + +def cases(start_time: str, clickhouse_dsn: str): + # The meta-level: one culprit (ns/pod/RootPID) fans out to each step it took. + # dx stamps culprit_key on every order; orders sharing it are one actor's + # campaign (read + exfil + spawns). from=culprit, to=the step (pod:rule). + df = px.DataFrame('dx_cases', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df.from_entity = df.culprit_key + df.to_entity = df.subject_pod + ':' + df.rule_id + df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': 'dx_kubescape_mitre', + 'order_id': df.order_id, + }) + return df[['from_entity', 'to_entity', 'culprit_key', 'rule_id', 'mitre_tactic', + 'mitre_technique', 'severity', 'alert', 'order_id', 'order_link']] + + +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique', 'ts']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.order = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + df.Alert = df.disc + return df[['order', 'ts', 'rule_id', 'Alert', 'mitre_tactic', 'mitre_technique', 'pod']] + + +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid', 'rule_id']] + k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id', 'uniqueID']) + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + # namespace is constant/uninformative for these protocol panels — drop it. + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id).drop(['namespace']) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id).drop(['namespace']) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id).drop(['namespace']) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id).drop(['namespace']) + + +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + # dx_ord__dc_snoop carries a real hostname (from the order/edge join); the raw + # dc_snoop table ships an empty hostname, so px hostname-sharding reads it as 0. + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id).drop(['namespace']) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + # Native profiler (never empty); the ClickHouse stack_trace export is not running. + w = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + w = w[w.order_id == order_id] + w = w[['pod']] + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st = st.merge(w, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_w']) + st = st.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + return st[['pod', 'stack_trace', 'count']] + + +def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): + # ATTACK [event_time-30s, +30s] vs MATCHED 60s BASELINE before it; Int64 offsets + # from dx_orders_win.lo (no float division -> compares against time_to_int64). + orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['pod', 'lo', 'hi']] + orders.alo = orders.lo + 270000000000 + orders.ahi = orders.lo + 330000000000 + orders.blo = orders.lo + 210000000000 + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st.row_time = px.time_to_int64(st.time_) + st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) + + base = st[st.row_time >= st.blo] + base = base[base.row_time < base.alo] + base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + atk = st[st.row_time >= st.alo] + atk = atk[atk.row_time <= atk.ahi] + atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], + suffixes=['_base', '_atk']) + diff.pod = diff.pod_atk + diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') + diff.count = diff.count_atk + diff.delta = diff.count_atk - diff.count_base + + total = atk.groupby(['pod']).agg(total=('count', px.sum)) + merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) + merged.percent = 100 * merged.count / merged.total + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml new file mode 100644 index 00000000000..59572ec64e3 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/manifest.yaml @@ -0,0 +1,4 @@ +--- +short: SOC Evidence Graph +long: > + SOC pixie, works only with clickhouse enabled. diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json new file mode 100644 index 00000000000..339ec607b88 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -0,0 +1,507 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "Window start.", + "defaultValue": "-6h" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "forensic_db DSN: user:pass@host:port/db.", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "graph_table", + "type": "PX_STRING", + "description": "L1 kill-chain graph source (MITRE-enriched).", + "defaultValue": "dx_kubescape_mitre" + }, + { + "name": "order_id", + "type": "PX_STRING", + "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every panel snaps to that order.", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "g_graph", + "func": { + "name": "evidence_graph", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "table", + "variable": "graph_table" + } + ] + } + }, + { + "outputName": "g_cases", + "func": { + "name": "cases", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + } + ] + } + }, + { + "outputName": "g_orders", + "func": { + "name": "orders", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "graph_table", + "variable": "graph_table" + } + ] + } + }, + { + "outputName": "g_kube", + "func": { + "name": "kubescape", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_conn", + "func": { + "name": "conn", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_redis", + "func": { + "name": "redis", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_http", + "func": { + "name": "http", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_dns", + "func": { + "name": "dns", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_pgsql", + "func": { + "name": "pgsql", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_mysql", + "func": { + "name": "mysql", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_dcsnoop", + "func": { + "name": "dc_snoop", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_stack", + "func": { + "name": "stack_trace", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_stackdiff", + "func": { + "name": "stack_diff", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + } + ], + "widgets": [ + { + "name": "CASES (culprit ns/pod/RootPID -> each step; one actor's campaign: read + exfil + spawns grouped)", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_cases", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "mitre_technique", + "edgeThresholds": { + "mediumThreshold": 5, + "highThreshold": 8 + }, + "edgeHoverInfo": [ + "order_link", + "culprit_key", + "rule_id", + "mitre_tactic", + "mitre_technique", + "alert", + "severity", + "order_id" + ], + "edgeLength": 500 + } + }, + { + "name": "Evidence graph (subject pod -> target; edge = ruleID + MITRE technique; click for details + order link)", + "position": { + "x": 0, + "y": 8, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_graph", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "rule_mitre", + "edgeThresholds": { + "mediumThreshold": 5, + "highThreshold": 8 + }, + "edgeHoverInfo": [ + "order_link", + "rule", + "mitre_tactic", + "mitre_technique", + "alert", + "process", + "target", + "severity", + "order_id" + ], + "edgeLength": 500 + } + }, + { + "name": "ORDERS (click an order to filter all panels)", + "position": { + "x": 0, + "y": 8, + "w": 12, + "h": 3 + }, + "globalFuncOutputName": "g_orders", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "kubescape_logs for order (MITRE tactic + technique)", + "position": { + "x": 0, + "y": 11, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_kube", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "conn_stats", + "position": { + "x": 0, + "y": 15, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_conn", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "redis_events", + "position": { + "x": 6, + "y": 15, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_redis", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "http_events", + "position": { + "x": 0, + "y": 19, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_http", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "dns_events", + "position": { + "x": 6, + "y": 19, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_dns", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "pgsql_events", + "position": { + "x": 0, + "y": 23, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_pgsql", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "mysql_events", + "position": { + "x": 6, + "y": 23, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_mysql", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "dc_snoop (file access)", + "position": { + "x": 0, + "y": 27, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_dcsnoop", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "stack_trace (profiler)", + "position": { + "x": 6, + "y": 27, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_stack", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Differential stack trace (attack window vs baseline; red = spiked during attack)", + "position": { + "x": 0, + "y": 31, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_stackdiff", + "displaySpec": { + "@type": "types.px.dev/px.vispb.StackTraceFlameGraph", + "stacktraceColumn": "stack_trace", + "countColumn": "count", + "percentageColumn": "percent", + "podColumn": "pod", + "differenceColumn": "delta" + } + } + ] +} \ No newline at end of file diff --git a/src/pxl_scripts/dx/fullchain/fullchain.pxl b/src/pxl_scripts/dx/fullchain/fullchain.pxl new file mode 100644 index 00000000000..72125155c85 --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/fullchain.pxl @@ -0,0 +1,53 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# SOC full-chain exfil narrative — works only with ClickHouse enabled. +# +# Cross-order/cross-sensor incident graph. Seeded from pods carrying an alert of +# severity >= min_sev (severity is data-driven; the exfil-relevant rules can be +# low-numeric, so min_sev is the editable knob), then overlays every sensor's +# edges for those pods in the query window (a UNION built in the dx_fullchain_edges +# view). Deliberately NOT keyed on a single order_id: the read (pgsql) and the send +# (http/egress) land in different ±50ms anomaly orders, so we correlate by +# pod + window + shared IP instead. Sparse by construction: a missing sensor just +# drops its edges, never the graph. + +import px + + +def _seed_pods(start_time: str, clickhouse_dsn: str, min_sev: int): + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + s = e[e.sev >= min_sev] + s = s.groupby(['pod']).agg(n=('sev', px.count)) + return s[['pod']] + + +def fullchain(start_time: str, clickhouse_dsn: str, min_sev: int): + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + seeds = _seed_pods(start_time, clickhouse_dsn, min_sev) + g = e.merge(seeds, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_s']) + g.from_entity = g.from_node + g.to_entity = g.to_node + return g[['from_entity', 'to_entity', 'edge_label', 'kind', 'pod']] + + +def group(start_time: str, clickhouse_dsn: str, min_sev: int): + # The editable preselection: which pods (and their seed alerts) are in the + # incident. Lower min_sev to pull in more pods, raise it to focus. + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + s = e[e.sev >= min_sev] + s = s.groupby(['pod', 'from_node', 'edge_label']).agg(alerts=('sev', px.count), sev=('sev', px.max)) + return s[['pod', 'edge_label', 'sev', 'alerts']] diff --git a/src/pxl_scripts/dx/fullchain/manifest.yaml b/src/pxl_scripts/dx/fullchain/manifest.yaml new file mode 100644 index 00000000000..d1af395f1ba --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/manifest.yaml @@ -0,0 +1,4 @@ +--- +short: SOC Full-Chain +long: > + SOC pixie, cross-sensor exfil narrative graph per incident; works only with clickhouse enabled. diff --git a/src/pxl_scripts/dx/fullchain/vis.json b/src/pxl_scripts/dx/fullchain/vis.json new file mode 100644 index 00000000000..cd5749bda67 --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/vis.json @@ -0,0 +1,15 @@ +{ + "variables": [ + {"name": "start_time", "type": "PX_STRING", "description": "Incident window (wide enough to re-join split read/send).", "defaultValue": "-90m"}, + {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, + {"name": "min_sev", "type": "PX_INT64", "description": "Seed threshold: include pods with an alert of at least this severity (10 signatures, 5 egress/network, 1 anomalies). Lower to widen the group.", "defaultValue": "5"} + ], + "globalFuncs": [ + {"outputName": "g_fc", "func": {"name": "fullchain", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "min_sev", "variable": "min_sev"}]}}, + {"outputName": "g_grp", "func": {"name": "group", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "min_sev", "variable": "min_sev"}]}} + ], + "widgets": [ + {"name": "Full-chain incident graph (seed alert -> pod -> egress/DNS/SQL; shared IP links the hops)", "position": {"x": 0, "y": 0, "w": 12, "h": 6}, "globalFuncOutputName": "g_fc", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeLabelColumn": "edge_label", "edgeHoverInfo": ["kind", "edge_label", "pod"], "edgeLength": 500}}, + {"name": "Incident group (seed pods + alerts; lower min_sev to widen)", "position": {"x": 0, "y": 6, "w": 12, "h": 3}, "globalFuncOutputName": "g_grp", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} + ] +} diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl deleted file mode 100644 index c5860a94a2c..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018- The Pixie Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -import px - - -def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str): - df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df.requestor = px.select(df.requestor_pod == '', - px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), - df.requestor_pod) - df.responder = px.select(df.responder_pod == '', - px.select(df.responder_service == '', df.responder_ip, df.responder_service), - df.responder_pod) - return df[['requestor', 'responder', - 'requestor_pod', 'responder_pod', - 'requestor_service', 'responder_service', - 'requestor_ip', 'responder_ip', - 'weight', 'max_severity', 'confidence', - 'edge_kind', 'condition', 'criteria', 'num_findings', - 'investigation_id']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml b/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml deleted file mode 100644 index 35ec4c613e4..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -short: DX Evidence Graph -long: > - Severity-weighted, all-protocol pod-to-pod graph for one investigation. Edge records emitted by - dx with weight (sum of CRS evidence severity) on the edges and - max_severity colouring the heat. diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json deleted file mode 100644 index 90befd97383..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "variables": [ - { - "name": "start_time", - "type": "PX_STRING", - "description": "Start time of the window.", - "defaultValue": "-15m" - }, - { - "name": "clickhouse_dsn", - "type": "PX_STRING", - "description": "ClickHouse DSN: user:pass@host:port/db.", - "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" - }, - { - "name": "table", - "type": "PX_STRING", - "description": "dx_evidence_graph", - "defaultValue": "dx_evidence_graph_malignant" - } - ], - "globalFuncs": [ - { - "outputName": "dx_graph", - "func": { - "name": "dx_evidence_graph", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"} - ] - } - } - ], - "widgets": [ - { - "name": "DX Evidence Graph", - "position": {"x": 0, "y": 0, "w": 12, "h": 5}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "requestor", - "toColumn": "responder" - }, - "edgeWeightColumn": "weight", - "edgeColorColumn": "max_severity", - "edgeLabelColumn": "edge_kind", - "edgeThresholds": { - "mediumThreshold": 3, - "highThreshold": 4 - }, - "edgeHoverInfo": [ - "edge_kind", - "condition", - "criteria", - "weight", - "max_severity", - "confidence", - "num_findings", - "investigation_id" - ], - "edgeLength": 500 - } - }, - { - "name": "Edges", - "position": {"x": 0, "y": 5, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } - ] -} diff --git a/src/ui/src/containers/live-widgets/graph/graph.tsx b/src/ui/src/containers/live-widgets/graph/graph.tsx index a8a642eec84..685931bf571 100644 --- a/src/ui/src/containers/live-widgets/graph/graph.tsx +++ b/src/ui/src/containers/live-widgets/graph/graph.tsx @@ -46,6 +46,7 @@ import { } from './graph-utils'; import { formatByDataType, formatBySemType } from '../../format-data/format-data'; import { deepLinkURLFromSemanticType } from '../utils/live-view-params'; +import { ScriptReference } from '../utils/script-reference'; interface AdjacencyList { toColumn: string; @@ -157,8 +158,11 @@ export const Graph = React.memo(({ const [graph, setGraph] = React.useState(null); const [pinned, setPinned] = React.useState>([]); const pinSeq = React.useRef(0); + const [edgeScriptRefs, setEdgeScriptRefs] = React.useState< + Map>(() => new Map()); const [edgeLabels, setEdgeLabels] = React.useState>(() => new Map()); const [labelOffsets, setLabelOffsets] = React.useState>(() => new Map()); @@ -198,6 +202,7 @@ export const Graph = React.memo(({ const nodes = new visData.DataSet(); const idToSemType = {}; const labelMap = new Map(); + const scriptRefMap = new Map(); const selfLoopCounts = new Map(); const selfLoopRank = new Map(); @@ -257,8 +262,17 @@ export const Graph = React.memo(({ if (edgeHoverInfo && edgeHoverInfo.length > 0) { let edgeInfo = ''; - edgeHoverInfo.forEach((info, i) => { + edgeHoverInfo.forEach((info) => { if (info != null) { + // Script-reference columns become the deep link in the pinned popup, + // not a line in the hover text (a hover tooltip can't be clicked). + if (info.semType === SemanticType.ST_SCRIPT_REFERENCE) { + const ref = d[info.name]; + if (ref && ref.script) { + scriptRefMap.set(edgeId, { label: ref.label, script: ref.script, args: ref.args }); + } + return; + } let val: string; if (info.semType === SemanticType.ST_NONE || info.semType === SemanticType.ST_UNSPECIFIED) { val = formatByDataType(info.type, d[info.name]); @@ -266,7 +280,7 @@ export const Graph = React.memo(({ const valWithUnits = formatBySemType(info.semType, d[info.name]); val = `${valWithUnits.val} ${valWithUnits.units}`; } - edgeInfo = `${edgeInfo}${i === 0 ? '' : '
'} ${info.name}: ${val}`; + edgeInfo = `${edgeInfo}${edgeInfo === '' ? '' : '
'} ${info.name}: ${val}`; } }); edge.title = edgeInfo; @@ -279,6 +293,7 @@ export const Graph = React.memo(({ nodes, edges, idToSemType, }); setEdgeLabels(labelMap); + setEdgeScriptRefs(scriptRefMap); setLabelOffsets((prev) => { const next = new Map(); selfLoopRank.forEach((rank, edgeId) => { @@ -331,6 +346,7 @@ export const Graph = React.memo(({ title: String(edgeData?.title ?? ''), x: rect.left + params.pointer.DOM.x, y: rect.top + params.pointer.DOM.y, + scriptRef: edgeScriptRefs.get(edgeId), }]); } }); @@ -401,7 +417,7 @@ export const Graph = React.memo(({ const onPinPointerDown = React.useCallback((key: number, initialX: number, initialY: number) => (e: React.PointerEvent) => { - if ((e.target as HTMLElement).closest('[data-pin-close]')) return; + if ((e.target as HTMLElement).closest('a, [data-pin-close]')) return; e.stopPropagation(); const startX = e.clientX; const startY = e.clientY; @@ -490,6 +506,17 @@ export const Graph = React.memo(({ touchAction: 'none', }} > + {p.scriptRef && ( +
+ +
+ )}
= 0 { break } @@ -140,12 +173,6 @@ func PixieTables() []string { "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", - "dx_unlink", - "dx_dlookup", - "dx_mprotect", - "dx_bpf", - "dx_ptrace", } } diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go index 0da8c706d3d..576bc1c6721 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go @@ -32,7 +32,7 @@ func TestDDL_ReturnsCanonicalForKnownTables(t *testing.T) { t.Fatalf("DDL(%q): %v", name, err) } if !strings.HasPrefix(ddl, "CREATE TABLE IF NOT EXISTS forensic_db.") && - !strings.HasPrefix(ddl, "CREATE VIEW IF NOT EXISTS forensic_db.") { + !strings.HasPrefix(ddl, "CREATE OR REPLACE VIEW forensic_db.") { t.Fatalf("DDL(%q) wrong prefix: %q", name, ddl[:minInt(70, len(ddl))]) } if !strings.HasSuffix(ddl, ";") { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go index 33ba3d5f106..b922fc60286 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go @@ -25,7 +25,6 @@ import "testing" // guards that this list stays a subset of the operator-owned pixie tables. var darkVectorTables = []string{ "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", "dx_unlink", "dx_dlookup", "dx_mprotect", "dx_bpf", "dx_ptrace", } // TestDarkVectorTablesHaveFullMetadata enforces attribution CONSISTENCY: every @@ -33,8 +32,8 @@ var darkVectorTables = []string{ // (namespace, pod, container, hostname), so any dark-vector event is attributable // to its workload uniformly. A table missing one of these is an inconsistency // that dx projection + forensic joins would silently drop — this was the concrete -// gap in dx_vfs_events/dx_unlink (no comm) and all six dx_ tables (no container) -// before they were reconciled to the canonical dc_snoop/creds_change shape. +// gap in the dx_* tracepoint tables before they were removed (no tracepoint was +// ever deployed for them, so they could only ever return zero rows). func TestDarkVectorTablesHaveFullMetadata(t *testing.T) { required := []string{"namespace", "pod", "container", "hostname"} for _, tbl := range darkVectorTables { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 6fa8ed7f002..33a57ccb46b 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -107,10 +107,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.http_events ( resp_body_size Int64, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_method, req_path); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_method, req_path, req_body, resp_status, resp_body); -- http2_messages.beta — http2_messages_table.h CREATE TABLE IF NOT EXISTS forensic_db.`http2_messages.beta` ( @@ -152,10 +153,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.dns_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_body); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_body, resp_body); -- redis_events — redis_table.h CREATE TABLE IF NOT EXISTS forensic_db.redis_events ( @@ -174,10 +176,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.redis_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_cmd); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_cmd, req_args, resp); -- mysql_events — mysql_table.h CREATE TABLE IF NOT EXISTS forensic_db.mysql_events ( @@ -197,10 +200,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.mysql_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_cmd, req_body, resp_status, resp_body); -- pgsql_events — pgsql_table.h CREATE TABLE IF NOT EXISTS forensic_db.pgsql_events ( @@ -218,10 +222,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.pgsql_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req, resp); -- cql_events — cass_table.h CREATE TABLE IF NOT EXISTS forensic_db.cql_events ( @@ -241,10 +246,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.cql_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_op, req_body, resp_op, resp_body); -- mongodb_events — mongodb_table.h CREATE TABLE IF NOT EXISTS forensic_db.mongodb_events ( @@ -264,10 +270,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.mongodb_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_cmd, req_body, resp_status, resp_body); -- kafka_events.beta — kafka_table.h CREATE TABLE IF NOT EXISTS forensic_db.`kafka_events.beta` ( @@ -383,7 +390,9 @@ CREATE TABLE IF NOT EXISTS forensic_db.conn_stats ( bytes_sent Int64, bytes_recv Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + remote_pod String DEFAULT '', + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, trace_role); @@ -531,11 +540,6 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_graph ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; --- dx_evidence_graph_malignant — rule-ins-only view (condition != '') the --- dx_evidence_graph UI reads by default so benign rows stay in ClickHouse. -CREATE VIEW IF NOT EXISTS forensic_db.dx_evidence_graph_malignant AS - SELECT * FROM forensic_db.dx_evidence_graph WHERE `condition` != ''; - -- dx_evidence_manifest — the §9 completeness contract: one row per verdict -- (ruled_in | metastasis), naming the evidence rows dx consulted so the -- validator can join them against what AE persisted (write⊇read, checkable). @@ -567,86 +571,120 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_manifest ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; +-- dx_order_seeds — one row per ORDER dx opens (entlein/dx#136 evidence-loss fix). +-- dx owns the order identity: it computes order_id and decides the dedup +-- granularity (1:1 with uniqueID today; finer — per rule/target/event — later). +-- AE only stores and surfaces exactly what dx emits, so the key is order_id and +-- NOTHING here assumes how many orders map to a uniqueID. dx INSERTs (POST-less, +-- direct CH); AE owns the DDL. ReplacingMergeTree ORDER BY (order_id) dedups +-- re-fires of the same order. NOT a pixie table. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( + order_id String, + unique_id String, + rule_id String, + pod String, + event_time UInt64, + hostname String, + case_key String +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- dx_order_records — the STAMPED consulted set (entlein/dx#136 stamping model). dx +-- writes one row per (order_id, finding): each record it consulted during the workup +-- for a primary kubescape log, stamped with that log's order_id. The panels read THIS +-- (the exact consulted set) instead of a ±300s time window. event_time is derived from +-- time_ so px can read it (UInt64 + hostname, no Bool cols). AE owns the DDL; dx +-- INSERTs. ReplacingMergeTree collapses re-stamps of the same (order_id,row). +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_records ( + order_id String, + unique_id String, + src_table String, + vector String, + source String, + time_ Int64, + pod String, + remote_addr String, + path String, + comm String, + dns_name String, + hostname String, + event_time UInt64 DEFAULT toUInt64(time_) +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, time_, pod, remote_addr, path, comm, dns_name) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- ── NEW identity model (added ALONGSIDE dx_order_seeds/records, which stay) ─── +-- dx_orders — one row per kubescape detection INSTANT. order_id is TRULY unique = +-- hash(uniqueID|Disc|event_time_ns). kubescape_uid/disc are provenance only, NEVER +-- keys. dx INSERTs; AE owns the DDL. +CREATE TABLE IF NOT EXISTS forensic_db.dx_orders ( + order_id String, + kubescape_uid String, + rule_id String, + disc String, + pod String, + event_time UInt64, + hostname String, + culprit_key String DEFAULT '' +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- dx_order_edges — the identity bridge. One row per (order, consulted pixie row): +-- links order_id to a base-table row via unique_id = the dx-computed content hash +-- of the row's fields (FNV-1a 64, lowercase hex String), the SAME value dx stamps +-- onto that base row's unique_id column — so they match by construction, no +-- CH-side hashing. String (not UInt64): a 64-bit integer does not survive a JSON +-- decode through float64. Many-to-many: a row consulted by N orders → N edges; +-- re-stamps collapse. dx INSERTs. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_edges ( + order_id String, + src_table String, + unique_id String, + hostname String, + event_time UInt64 DEFAULT 0 +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, unique_id) + SETTINGS index_granularity = 8192; + +-- dx_ord__conn_stats — join view: conn_stats rows consulted for an order, via the +-- bridge (edge.unique_id = conn_stats.unique_id). Panel filters by order_id. +CREATE OR REPLACE VIEW forensic_db.dx_ord__conn_stats AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.protocol AS protocol, + c.conn_open AS conn_open, + c.conn_close AS conn_close, + c.conn_active AS conn_active, + c.bytes_sent AS bytes_sent, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.conn_stats AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'conn_stats'; + -- ── dx dark-vector tracepoint tables (entlein/dx#126) ──────────────────────── -- Fed by AE-owned bpftrace UpsertTracepoint probes (constantly enabled, no TTL). -- Emit raw kernel pid+comm (NOT upid); namespace/pod enriched at pull time via a -- process_stats join on pid. One column per line (schema-verify parser is line-oriented). -- (dx_dcsnoop superseded by forensic_db.dc_snoop — canonical DateTime64(9)/Int64 -- schema with full k8s metadata; see the dark-vector section above.) -CREATE TABLE IF NOT EXISTS forensic_db.dx_vfs_events ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - op String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_unlink ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - op String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_dlookup ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_mprotect ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - prot UInt64, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -- (dx_creds superseded by forensic_db.creds_change — canonical schema with -- old_uid/new_uid + full k8s metadata.) -CREATE TABLE IF NOT EXISTS forensic_db.dx_bpf ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_ptrace ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -- dc_snoop (dentry cache, V1/V2 process+file) — exported via the OTel/ClickHouse -- retention plugin (px.export). pid-keyed; t = R (reference) / M (miss). -- One column per line (schema-verify parser is line-oriented). @@ -660,7 +698,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.dc_snoop ( pod String, container String, hostname String, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, t, file, pod); -- stack_trace (native continuous profiler stack_traces.beta, V9) — OTel export. @@ -674,7 +713,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.stack_trace ( stack_trace_id Int64, stack_trace String, count Int64, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, upid, stack_trace_id, pod); -- creds_change (commit_creds privilege-escalation to root, V7) — OTel export. @@ -688,5 +728,448 @@ CREATE TABLE IF NOT EXISTS forensic_db.creds_change ( pod String, container String, hostname String, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, old_uid, new_uid, pod); + +-- ── Order-UUID pre-correlation views (entlein/dx#136) ──────────────────────── +-- The px/dx_evidence_graph multi-panel dashboard reads these. Each is created on +-- boot AFTER its base table (Apply is fatal on a missing base): all bases are +-- OperatorOwned, and kubescape_logs is ensured in OperatorOwnedTables just before +-- these views. px read contract: expose event_time UInt64 + hostname + NO Bool cols; +-- ts=toString(time_) readable, row_time Int64 ns for the PxL interval-join. Views +-- are not pixie socket_tracer tables → absent from PixieTables(). + +-- dx_anomaly_orders: ONE row per order dx opened. order_id is dx-assigned and +-- dx owns its granularity (hash(uniqueID) = 1:1 with the log today; finer later), +-- so the view dedups on order_id and makes NO assumption about orders-per-uniqueID. +-- lo/hi are kept for reference (the ±300s span); the CONSULTED records for the +-- order live in dx_order_records, stamped with this order_id. +CREATE OR REPLACE VIEW forensic_db.dx_anomaly_orders AS +SELECT unique_id AS uniqueID, rule_id AS rule, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + order_id, + hostname, event_time +FROM forensic_db.dx_order_seeds +LIMIT 1 BY order_id; + +-- dx_kubescape_anomalies: L1 kill-chain graph (subject_pod -> target), deduped by uniqueID. +CREATE OR REPLACE VIEW forensic_db.dx_kubescape_anomalies AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', (JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID; + +-- dx_src__kubescape_logs: anomaly detail (process tree comm/cmdline/pcomm) per panel. +CREATE OR REPLACE VIEW forensic_db.dx_src__kubescape_logs AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_src__stack_trace: original schema + ts/row_time/event_time. +CREATE OR REPLACE VIEW forensic_db.dx_src__stack_trace AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, container, stack_trace_id, stack_trace, count, hostname +FROM forensic_db.stack_trace; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__redis_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.req_cmd AS req_cmd, + c.req_args AS req_args, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.redis_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'redis_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__http_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_method AS req_method, + c.req_path AS req_path, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.http_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'http_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__dns_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_body AS req_body, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dns_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dns_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__pgsql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req AS req, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.pgsql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'pgsql_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__mysql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mysql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mysql_events'; + +-- dx_ord__cql_events / dx_ord__mongodb_events / dx_ord__creds_change — bridge +-- views: exactly the rows dx consulted for an order, joined on the dx-stamped +-- unique_id. Same shape as the other dx_ord__ views; the panel filters order_id. +CREATE OR REPLACE VIEW forensic_db.dx_ord__cql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_op AS req_op, + c.req_body AS req_body, + c.resp_op AS resp_op, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.cql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'cql_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__mongodb_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mongodb_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mongodb_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__creds_change AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.pid AS pid, + c.comm AS comm, + c.old_uid AS old_uid, + c.new_uid AS new_uid, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.creds_change AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'creds_change'; + +-- dx_cases — the meta-grouping above orders: every order carries culprit_key +-- (ns/pod/RootPID). Orders sharing a culprit_key are one actor's steps (read + +-- exfil + spawns). Joins the kubescape mitre/severity so a case node can be +-- coloured, and counts distinct evidence protocols the culprit touched. +CREATE OR REPLACE VIEW forensic_db.dx_cases AS +SELECT + o.culprit_key AS culprit_key, + o.order_id AS order_id, + o.rule_id AS rule_id, + o.pod AS subject_pod, + o.disc AS alert, + m.mitre_tactic AS mitre_tactic, + m.mitre_technique AS mitre_technique, + m.severity AS severity, + toInt64(toUnixTimestamp64Nano(fromUnixTimestamp64Nano(o.event_time))) AS event_time, + o.hostname AS hostname +FROM forensic_db.dx_orders AS o +LEFT JOIN forensic_db.dx_kubescape_mitre AS m + ON m.uniqueID = o.kubescape_uid AND m.rule = o.rule_id +WHERE o.culprit_key != ''; + +-- dx_case_links — the cross-pod bridge over cases: an order's conn_stats +-- evidence resolves remote_pod (the peer pod), and a culprit lives on that pod. +-- Links the sink's exfil-receipt culprit back to the attacker's culprit that +-- opened the connection. Directed: from = this order's culprit, to = peer culprit. +CREATE OR REPLACE VIEW forensic_db.dx_case_links AS +SELECT DISTINCT + a.culprit_key AS from_culprit, + c.remote_pod AS peer_pod, + b.culprit_key AS to_culprit, + a.hostname AS hostname +FROM forensic_db.dx_orders AS a +INNER JOIN forensic_db.dx_order_edges AS e + ON e.order_id = a.order_id AND e.src_table = 'conn_stats' +INNER JOIN forensic_db.conn_stats AS c + ON c.unique_id = e.unique_id AND c.remote_pod != '' +INNER JOIN forensic_db.dx_orders AS b + ON b.pod = c.remote_pod +WHERE a.culprit_key != '' AND b.culprit_key != '' AND a.culprit_key != b.culprit_key; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__dc_snoop AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.pid AS pid, + c.comm AS comm, + c.t AS t, + c.file AS file, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dc_snoop AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dc_snoop'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__stack_trace AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + c.stack_trace_id AS stack_trace_id, + c.stack_trace AS stack_trace, + c.count AS count, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.stack_trace AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'stack_trace'; + +-- ── MITRE ATT&CK enrichment over kubescape_logs (px/dx_evidence_graph) ──────── +-- dx_kubescape_mitre: L1 graph source — one row per (uniqueID, rule) so orders +-- keyed on (uniqueID, RuleID) all join; MITRE + resolved target from BaseRuntimeMetadata. +CREATE OR REPLACE VIEW forensic_db.dx_kubescape_mitre AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + concat(RuleID, ' · ', JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique')) AS rule_mitre, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, +'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', +concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', +(JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, +'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time, + toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID, rule; + +-- dx_src__kubescape_mitre: kubescape detail panel — MITRE cols after RuleID, plus +-- process tree (comm/pcomm/cmdline). ts/row_time/event_time px-connector convention. +CREATE OR REPLACE VIEW forensic_db.dx_src__kubescape_mitre AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pid') AS pid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'ppid') AS ppid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_orders_win: per-order ±300s baseline/attack window for the differential +-- flamegraph (stack_diff). hostname carried so the px node-shard resolves. +CREATE OR REPLACE VIEW forensic_db.dx_orders_win AS +SELECT order_id, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + event_time, hostname +FROM forensic_db.dx_orders; + +-- dx_dns_resolve: DNS resolution edges exploded from dns_events resp_body +-- (querier->resolver, then the answer tree name->CNAME / name->A). Read by the +-- dx/dns_resolve UI, time-windowed via dx_orders_win. Column named event_time +-- (int64 ns) because the px ClickHouse connector defaults its cursor there. +CREATE OR REPLACE VIEW forensic_db.dx_dns_resolve AS +SELECT toInt64(toUnixTimestamp64Nano(dns_events.event_time)) AS event_time, + toString(dns_events.event_time) AS ts, hostname, + if(pod != '', pod, if(local_addr != '', concat('client:', local_addr), 'client')) AS from_node, + concat(remote_addr, ':', toString(remote_port)) AS to_node, + JSONExtractString(JSONExtractArrayRaw(req_body, 'queries')[1], 'name') AS edge_label, + 'query' AS kind +FROM forensic_db.dns_events +WHERE resp_body != '' AND resp_body != '{}' +UNION ALL +SELECT toInt64(toUnixTimestamp64Nano(dns_events.event_time)) AS event_time, + toString(dns_events.event_time) AS ts, hostname, + JSONExtractString(ans, 'name') AS from_node, + concat(JSONExtractString(ans, 'cname'), JSONExtractString(ans, 'addr')) AS to_node, + JSONExtractString(ans, 'type') AS edge_label, + lower(JSONExtractString(ans, 'type')) AS kind +FROM forensic_db.dns_events +ARRAY JOIN JSONExtractArrayRaw(resp_body, 'answers') AS ans +WHERE resp_body != '' AND JSONExtractString(ans, 'type') != '' + AND concat(JSONExtractString(ans, 'cname'), JSONExtractString(ans, 'addr')) NOT IN ('', '-'); + +-- dx_alerts: GENERIC thin flatten of kubescape_logs (pod/namespace/rule/message/ +-- sev). Reusable seed for any narrative; story logic (target/kind) is derived in +-- the PxL (dx/breakout), never here — so the DDL is portable and stays stable. +CREATE OR REPLACE VIEW forensic_db.dx_alerts AS +SELECT + fromUnixTimestamp64Nano(toInt64(event_time)) AS event_time, + hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'namespace') AS namespace, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, + RuleID AS rule, + message AS message, + JSONExtractInt(BaseRuntimeMetadata,'severity') AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager',''); + +-- dx_breakout_story: runtime-breakout edges (pod -> off-profile target, kind). +-- Story-specific SQL kept for the shipped dashboard; newer dx/breakout PxL derives +-- target/kind from dx_alerts in PxL instead. +CREATE OR REPLACE VIEW forensic_db.dx_breakout_story AS +SELECT + fromUnixTimestamp64Nano(toInt64(event_time)) AS event_time, + hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'namespace') AS namespace, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, + JSONExtractString(RuntimeK8sDetails,'podName') AS from_node, + multiIf(RuleID='R0002' AND position(message,'serviceaccount')>0 AND position(message,'token')>0,'serviceaccount/token (SA cred)', + RuleID='R0002', extractGroups(message,' to (.+)$')[1], + RuleID='R0001', concat('proc:',coalesce(nullIf(extractGroups(message,'([^ /]+)$')[1],''),'?')), + RuleID='R0012', concat('ingress<-',coalesce(nullIf(extractGroups(message,'from: ([^ ]+)')[1],''),'peer')), + RuleID='R0011', concat('egress->',coalesce(nullIf(extractGroups(message,'to: ([^ ]+)')[1],''),'peer')), + RuleID='R0005', concat('dns:',coalesce(nullIf(extractGroups(message,'([^ ]+)$')[1],''),'?')), + RuleID) AS to_node, + RuleID AS rule, + multiIf(RuleID='R0001','process', + RuleID='R0002' AND position(message,'serviceaccount')>0 AND position(message,'token')>0,'token-read', + RuleID='R0002' AND match(message,'\\.so'),'libload', + RuleID='R0002' AND position(message,'/proc/')>0,'proc-read', + RuleID='R0002' AND (position(message,'/etc/')>0 OR position(message,'/runc')>0 OR position(message,'/root')>0),'sensitive', + RuleID='R0002' AND position(message,'/tmp')>0,'tmp-write', + RuleID='R0002','file-access', + RuleID='R0012','ingress', RuleID='R0011','egress', RuleID='R0005','dns', + RuleID='R0004','capability', RuleID='R0003','syscall', + RuleID IN ('R0006','R0007','R0008'),'cred-access','alert') AS kind, + JSONExtractInt(BaseRuntimeMetadata,'severity') AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager','gmp-system', + 'gmp-public','storm','lightening','chain-loadgen',''); + +-- dx_fullchain_edges: cross-sensor exfil edges (kubescape seed + conn egress + +-- dns + pgsql) normalized to one edge shape. Feeds dx/fullchain. The UNION must +-- live in SQL (PxL cannot union DataFrames); labels stay source-based, not parsed. +CREATE OR REPLACE VIEW forensic_db.dx_fullchain_edges AS +SELECT toDateTime64(fromUnixTimestamp64Nano(toInt64(event_time)),9) AS event_time, hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, concat('alert:',RuleID) AS from_node, + JSONExtractString(RuntimeK8sDetails,'podName') AS to_node, RuleID AS edge_label, 'flagged' AS kind, + toInt32(JSONExtractInt(BaseRuntimeMetadata,'severity')) AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager','') +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, remote_addr, + concat('conn/',toString(protocol)), 'connects', toInt32(5) +FROM forensic_db.conn_stats WHERE trace_role=1 AND remote_addr!='' +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, req_body, 'dns', 'resolves', toInt32(5) +FROM forensic_db.dns_events WHERE req_body!='' +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, substring(req,1,60), 'sql', 'sql', toInt32(5) +FROM forensic_db.pgsql_events WHERE req!=''; diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go new file mode 100644 index 00000000000..b2e1051c0fb --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go @@ -0,0 +1,104 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package clickhouse + +import ( + "regexp" + "testing" +) + +// Drift guard between schema.sql and the two registration lists. +// +// schema.sql is the source of truth for DDL, but nothing is created unless the +// object is also listed: Apply iterates OperatorOwnedTables, and DDL/Columns +// resolve through KnownTables. An object that exists in schema.sql and in +// neither list is inert — and inert *silently*: no error at boot, no log line, +// just a view that is never created and panels that return nothing. That has +// happened twice (dx_base__dc_snoop, then dx_ord__{cql,mongodb,creds_change}), +// both caught on a rig rather than in CI. +// +// The reverse drift is cheaper to hit but also worth pinning: a name listed +// with no DDL behind it fails only when DDL() is called for it. + +// createStmt matches the object name in `CREATE TABLE IF NOT EXISTS forensic_db. +// ` (tables) or `CREATE OR REPLACE VIEW forensic_db.` (views — replaced +// on every Apply so a changed definition never needs a manual DROP), with or +// without the backticks used for dotted names (e.g. `http2_messages.beta`). +var createStmt = regexp.MustCompile("(?i)CREATE\\s+(?:TABLE\\s+IF\\s+NOT\\s+EXISTS|OR\\s+REPLACE\\s+VIEW)\\s+forensic_db\\.`?([A-Za-z0-9_.]+)`?") + +// socOwnedTables are declared in schema.sql so the operator can verify and read +// them, but are created by the soc/clickhouse-lab installer — AE must never +// issue their CREATE TABLE. See TestOperatorOwnedTables_DoesNotIncludeKubescape. +var socOwnedTables = map[string]bool{ + "alerts": true, + "kubescape_logs": true, +} + +func schemaObjects(t *testing.T) map[string]bool { + t.Helper() + objs := map[string]bool{} + for _, m := range createStmt.FindAllStringSubmatch(canonicalSchema, -1) { + objs[m[1]] = true + } + if len(objs) == 0 { + t.Fatal("parsed no CREATE statements out of schema.sql — the regex or the file shape changed") + } + return objs +} + +func TestEverySchemaObjectIsRegistered(t *testing.T) { + known := map[string]bool{} + for _, n := range KnownTables { + known[n] = true + } + owned := map[string]bool{} + for _, n := range OperatorOwnedTables { + owned[n] = true + } + + for name := range schemaObjects(t) { + if !known[name] { + t.Errorf("%q is in schema.sql but missing from KnownTables — DDL(%q) will not resolve it", name, name) + } + if socOwnedTables[name] { + if owned[name] { + t.Errorf("%q is soc-owned; AE must not create it", name) + } + continue + } + if !owned[name] { + t.Errorf("%q is in schema.sql but missing from OperatorOwnedTables — Apply will never create it, "+ + "and nothing will report that at boot", name) + } + } +} + +func TestEveryRegisteredNameHasDDL(t *testing.T) { + objs := schemaObjects(t) + for _, name := range KnownTables { + if !objs[name] { + t.Errorf("%q is in KnownTables but has no CREATE statement in schema.sql", name) + } + } + for _, name := range OperatorOwnedTables { + if !objs[name] { + t.Errorf("%q is in OperatorOwnedTables but has no CREATE statement in schema.sql", name) + } + } +} diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index c22b1b8ba71..6253c6a54d5 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -19,19 +19,26 @@ load("//bazel:pl_build_system.bzl", "pl_go_test") go_library( name = "control", - srcs = ["server.go"], + srcs = [ + "server.go", + "tls.go", + ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/control", visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", + "@com_github_sirupsen_logrus//:logrus", ], ) pl_go_test( name = "control_test", - srcs = ["server_test.go"], + srcs = [ + "server_test.go", + "tls_test.go", + ], embed = [":control"], deps = [ "//src/shared/services/utils", diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index 96292715fcb..a98b000136b 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -31,9 +31,13 @@ import ( "context" "encoding/json" "net/http" + "os" + "strconv" "strings" "time" + log "github.com/sirupsen/logrus" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" @@ -67,6 +71,26 @@ type exportAller interface { // anomaly is comfortably inside the pulled slice. const controlExportLookback = 600 * time.Second +// A DEGENERATE /query window (hi <= lo) is widened to controlExportLookback: a +// zero-width window keyed on one finding's timestamp matches no pixie rows. +// +// A deliberately TIGHT window is NOT degenerate and must be honoured. kubescape's +// alert timestamp is the kernel event time (bpf_ktime_get_boot_ns, converted to +// wall clock once, never re-stamped) and it reaches AE as nanos end-to-end, so a +// ±50ms span around an anomaly is meaningful — it is how a chatty protocol +// (pgsql/mysql) stays readable instead of returning tens of thousands of rows. +// This floor previously widened ANY sub-5s window to 600s, silently inflating an +// intentional ±50ms request by 6000× and defeating caller-side narrowing. +// The floor drops to 1ms: still wide enough to catch a sub-microsecond point +// window (which matches no rows), far below any intentional millisecond span. +// ADAPTIVE_MIN_QUERY_WINDOW_MS overrides it. +func minControlQueryWindow() time.Duration { + if v, err := strconv.Atoi(os.Getenv("ADAPTIVE_MIN_QUERY_WINDOW_MS")); err == nil && v > 0 { + return time.Duration(v) * time.Millisecond + } + return time.Millisecond +} + // The control API carries timestamps in the pipeline's ONE unit: unix // NANOSECONDS — the same unit as forensic_db.*.event_time and dx's referral // windows. Read them with time.Unix(0, ns). (This spot previously did @@ -86,12 +110,21 @@ type manifestWriter interface { WriteEvidenceManifest(ctx context.Context, jsonEachRow []byte) error } +// rowsWriter persists dx-handed pixie base rows (loop 1: conn_stats with a +// pre-stamped unique_id) into forensic_db. through the SAME sink the +// controller capture path uses (sink.ClickHouseHTTP.WritePixieRows). +// nil → /dx/rows 501s. +type rowsWriter interface { + WritePixieRows(ctx context.Context, table string, rows []map[string]any) error +} + // Server is the control HTTP surface. type Server struct { set exporter runner queryRunner // may be nil; /query then returns 501 graph graphWriter // may be nil; /dx/evidence_graph then returns 501 manifest manifestWriter // may be nil; /dx/evidence_manifest then returns 501 + rows rowsWriter // may be nil; /dx/rows then returns 501 mux *http.ServeMux verify func(bearer string) error // nil → auth disabled; set via SetAuth } @@ -106,6 +139,7 @@ func New(set exporter, runner queryRunner) *Server { s.mux.HandleFunc("/query", s.handleQuery) s.mux.HandleFunc("/dx/evidence_graph", s.handleDXEvidenceGraph) s.mux.HandleFunc("/dx/evidence_manifest", s.handleDXEvidenceManifest) + s.mux.HandleFunc("/dx/rows", s.handleDXRows) return s } @@ -115,6 +149,9 @@ func (s *Server) SetGraphWriter(g graphWriter) { s.graph = g } // SetManifestWriter wires the dx_evidence_manifest sink. func (s *Server) SetManifestWriter(m manifestWriter) { s.manifest = m } +// SetRowsWriter wires the /dx/rows base-row sink (loop 1). +func (s *Server) SetRowsWriter(rw rowsWriter) { s.rows = rw } + // SetAuth turns on bearer-JWT auth for the control surface, verified with the // SAME shared lib + signing key the vizier broker/PEM use (px.dev/pixie/src/ // shared/services/utils). dx already mints a service JWT (GenerateJWTForService, @@ -179,6 +216,63 @@ func (s *Server) handleDXEvidenceGraph(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } +// dxRowsAllowedTables guards /dx/rows against arbitrary-table writes: only the +// bridged tables dx hands base rows for (each carries a pre-stamped unique_id and +// a dx_ord__ view) are accepted. Mirrors evidencegraph.UIDColsByTable on the dx side. +var dxRowsAllowedTables = map[string]bool{ + "conn_stats": true, + "redis_events": true, + "http_events": true, + "dns_events": true, + "pgsql_events": true, + "mysql_events": true, + "cql_events": true, + "mongodb_events": true, + "dc_snoop": true, + "stack_trace": true, + // creds_change is dx-bridged too (dark tracepoint, keyed on its own columns). + "creds_change": true, +} + +// dxRowsReq is the /dx/rows wire body: dx-handed base rows for one table. +type dxRowsReq struct { + Table string `json:"table"` + Rows []map[string]any `json:"rows"` +} + +// handleDXRows ingests dx-handed base rows (loop 1: conn_stats carrying a +// pre-stamped content-hash unique_id, a hex String) and writes them to +// forensic_db.
via the same sink path the controller capture uses. +// decodeNumber (UseNumber) keeps large integer columns as json.Number so the +// fast encoder emits exact decimal text; the shared decode() would cast them to +// float64, and the sink's appendFloat renders large values in scientific +// notation, which ClickHouse rejects for Int64/UInt64 columns (whole batch 502). +func (s *Server) handleDXRows(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if s.rows == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + var req dxRowsReq + if !decodeNumber(w, r, &req) || !dxRowsAllowedTables[req.Table] { + w.WriteHeader(http.StatusBadRequest) + return + } + if len(req.Rows) == 0 { + w.WriteHeader(http.StatusAccepted) + return + } + if err := s.rows.WritePixieRows(r.Context(), req.Table, req.Rows); err != nil { + log.WithField("table", req.Table).WithField("rows", len(req.Rows)).WithError(err).Error("dx/rows: WritePixieRows failed") + w.WriteHeader(http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusAccepted) +} + // dxManifest mirrors the wire shape of dx's manifest.Manifest (internal/manifest). // Scalars map to typed forensic_db.dx_evidence_manifest columns; the nested // collections are held as raw JSON and persisted as JSON text in String columns @@ -295,6 +389,14 @@ func decode(w http.ResponseWriter, r *http.Request, v any) bool { return json.NewDecoder(r.Body).Decode(v) == nil } +func decodeNumber(w http.ResponseWriter, r *http.Request, v any) bool { + defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxControlBodyBytes) + dec := json.NewDecoder(r.Body) + dec.UseNumber() + return dec.Decode(v) == nil +} + // ── handlers ────────────────────────────────────────────────────────── func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -353,8 +455,12 @@ func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - err := s.runner.OrderQuery(req.target(), req.Table, - time.Unix(0, req.Window[0]).UTC(), time.Unix(0, req.Window[1]).UTC(), req.QueryID) + hi := time.Unix(0, req.Window[1]).UTC() + lo := time.Unix(0, req.Window[0]).UTC() + if d := hi.Sub(lo); d <= 0 || d < minControlQueryWindow() { + lo = hi.Add(-controlExportLookback) // widen a degenerate (or floored) window + } + err := s.runner.OrderQuery(req.target(), req.Table, lo, hi, req.QueryID) if err != nil { w.WriteHeader(http.StatusBadGateway) return diff --git a/src/vizier/services/adaptive_export/internal/control/server_test.go b/src/vizier/services/adaptive_export/internal/control/server_test.go index 15b3122ad0e..ae57dbe6eb4 100644 --- a/src/vizier/services/adaptive_export/internal/control/server_test.go +++ b/src/vizier/services/adaptive_export/internal/control/server_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -45,12 +46,14 @@ func (f *fakeExporter) Remove(k activeset.Key) { f.removes = append(f.removes, k // fakeRunner records OrderQuery calls; err controls the failure path. type fakeRunner struct { - calls []string // "table|ns/pod|queryID" - err error + calls []string // "table|ns/pod|queryID" + lastStart, lastEnd time.Time + err error } func (f *fakeRunner) OrderQuery(t anomaly.Target, table string, start, end time.Time, qid string) error { f.calls = append(f.calls, table+"|"+t.Namespace+"/"+t.Pod+"|"+qid) + f.lastStart, f.lastEnd = start, end return f.err } @@ -311,3 +314,72 @@ func TestEvidenceManifest(t *testing.T) { t.Fatalf("writer error: got %d, want 502", r.StatusCode) } } + +// TestQueryWidensNarrowWindow — a control client that sends a near-zero window +// (lo≈hi, e.g. dx keying on a single finding's event_time) would capture nothing; +// the handler widens it to controlExportLookback ending at hi so the evidence +// leading up to the referral is still captured. +func TestQueryWidensNarrowWindow(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + // hi = 10_000_000_000 ns, lo = hi - 512ns → a 512ns window (passes lo= %v; got %v", minControlQueryWindow(), got) + } + // hi must be preserved (we widen the lower bound only). + if rn.lastEnd.UnixNano() != hi { + t.Errorf("hi must be preserved; want %d got %d", hi, rn.lastEnd.UnixNano()) + } +} + +// TestQueryHonoursIntentionalTightWindow — a deliberate +/-50ms span around an +// anomaly must reach the runner UNCHANGED. kubescape's alert timestamp is the +// kernel event time and travels as nanos end-to-end, so a millisecond window is +// meaningful; widening it to 600s is what made a chatty protocol (pgsql/mysql) +// return tens of thousands of rows per referral. +func TestQueryHonoursIntentionalTightWindow(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(10_000_000_000) + lo := hi - int64(100*time.Millisecond) // +/-50ms around the anomaly + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"postgres-oss","table":"pgsql_events","query_id":"q1","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 100*time.Millisecond { + t.Fatalf("tight window must pass through unchanged; want 100ms got %v", got) + } + if rn.lastStart.UnixNano() != lo { + t.Errorf("lo must be preserved; want %d got %d", lo, rn.lastStart.UnixNano()) + } +} + +// A comfortably-wide window is passed through unchanged (no over-widening). +func TestQueryWideWindowUnchanged(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(1_000_000_000_000) // 1000s in ns, so lo stays positive + lo := hi - int64(120*time.Second) + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"redis-demo","table":"dc_snoop","query_id":"q2","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 120*time.Second { + t.Errorf("wide window must pass through unchanged; want 120s got %v", got) + } +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/src/vizier/services/adaptive_export/internal/control/tls.go b/src/vizier/services/adaptive_export/internal/control/tls.go new file mode 100644 index 00000000000..cc59c8455c1 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls.go @@ -0,0 +1,128 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// TLSConfig builds the server-side *tls.Config for the control surface. +// +// If BOTH certFile and keyFile exist and load, the mounted keypair is used +// (the shared service-tls-certs the broker/PEM already carry). Otherwise an +// ephemeral in-memory self-signed cert is generated so TLS works with zero +// extra secrets — dx skip-verifies the in-cluster cert, so a self-signed cert +// is sufficient to stop the bearer JWT crossing the CNI in cleartext. +// +// The bool return reports whether the cert was self-generated (true) vs +// loaded from disk (false), for the caller's boot log. +func TLSConfig(certFile, keyFile string, hostnames ...string) (*tls.Config, bool, error) { + if certFile != "" && keyFile != "" && fileExists(certFile) && fileExists(keyFile) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, false, fmt.Errorf("load mounted keypair %s/%s: %w", certFile, keyFile, err) + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, false, nil + } + cert, err := selfSignedCert(hostnames...) + if err != nil { + return nil, false, err + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, true, nil +} + +func fileExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && !fi.IsDir() +} + +// selfSignedCert mints an ephemeral in-memory self-signed certificate: +// ECDSA P-256, 1y validity, SAN covering localhost + 127.0.0.1 + ::1 and any +// extra hostnames (the pod/node name). Nothing is written to disk; the key +// lives only in the returned tls.Certificate. +func selfSignedCert(hostnames ...string) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate ecdsa key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate serial: %w", err) + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "adaptive-export-control"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(1, 0, 0), // 1y validity + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + } + for _, h := range hostnames { + if h == "" { + continue + } + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create certificate: %w", err) + } + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + Leaf: &tmpl, + }, nil +} + +// certToPEM renders a tls.Certificate (as produced by selfSignedCert, holding a +// single DER cert + an *ecdsa.PrivateKey) as PEM cert + PEM key bytes — the +// on-disk shape of a mounted /certs/server.{crt,key} keypair. +func certToPEM(cert tls.Certificate) ([]byte, []byte, error) { + if len(cert.Certificate) == 0 { + return nil, nil, fmt.Errorf("certToPEM: empty certificate chain") + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) + ec, ok := cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("certToPEM: private key is not *ecdsa.PrivateKey") + } + der, err := x509.MarshalECPrivateKey(ec) + if err != nil { + return nil, nil, fmt.Errorf("marshal ec private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + return certPEM, keyPEM, nil +} diff --git a/src/vizier/services/adaptive_export/internal/control/tls_test.go b/src/vizier/services/adaptive_export/internal/control/tls_test.go new file mode 100644 index 00000000000..e6f53680c83 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls_test.go @@ -0,0 +1,194 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/tls" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + jwtutils "px.dev/pixie/src/shared/services/utils" +) + +// serveTLS starts the control server over TLS on 127.0.0.1:0 using the given +// *tls.Config and returns the base https URL + a shutdown func. +func serveTLS(t *testing.T, cfg *tls.Config, srv *Server) (string, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: srv.Handler(), TLSConfig: cfg} + go func() { _ = httpSrv.ServeTLS(ln, "", "") }() + return "https://" + ln.Addr().String(), func() { _ = httpSrv.Close() } +} + +func skipVerifyClient() *http.Client { + return &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, //nolint:gosec // test: dx skip-verifies the in-cluster self-signed cert + } +} + +// TestTLSConfigSelfSigned: with no mounted cert files, TLSConfig self-generates +// an in-memory cert (the default secure path when /certs is absent). +func TestTLSConfigSelfSigned(t *testing.T) { + cfg, selfSigned, err := TLSConfig("/no/such/cert.crt", "/no/such/key.key", "some-pod") + if err != nil { + t.Fatalf("TLSConfig self-gen: %v", err) + } + if !selfSigned { + t.Fatal("expected selfSigned=true when cert files are absent") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected exactly one in-memory certificate, got %+v", cfg) + } +} + +// TestTLSServesHealthz: the server serves TLS by default (self-gen path) and a +// TLS client can reach /healthz. This is T1's "no cleartext by default". +func TestTLSServesHealthz(t *testing.T) { + cfg, selfSigned, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + if !selfSigned { + t.Fatal("expected self-signed cert") + } + base, stop := serveTLS(t, cfg, New(&fakeExporter{}, nil)) + defer stop() + + resp, err := skipVerifyClient().Get(base + "/healthz") + if err != nil { + t.Fatalf("TLS GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over TLS = %d, want 200", resp.StatusCode) + } +} + +// TestTLSRejectsUnauthenticated: over TLS with a signing key configured, an +// unauthenticated control request is rejected (401). This is T1's "requires +// the bearer JWT when a signing key is present" — verified end-to-end on the +// real TLS listener, not just the handler. +func TestTLSRejectsUnauthenticated(t *testing.T) { + const key = "0123456789abcdef0123456789abcdef" + srv := New(&fakeExporter{}, nil) + srv.SetAuth(key, "vizier") + + cfg, _, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + base, stop := serveTLS(t, cfg, srv) + defer stop() + client := skipVerifyClient() + + // No bearer → 401. + resp, err := client.Post(base+"/export/start", "application/json", strings.NewReader(`{"pod":"p","t_end":1}`)) + if err != nil { + t.Fatalf("TLS POST: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated over TLS = %d, want 401", resp.StatusCode) + } + + // Valid bearer → not 401. + good, err := jwtutils.SignJWTClaims(jwtutils.GenerateJWTForService("dx", "vizier"), key) + if err != nil { + t.Fatalf("mint token: %v", err) + } + req, _ := http.NewRequest(http.MethodPost, base+"/export/start", strings.NewReader(`{"namespace":"n","pod":"p","t_end":1}`)) + req.Header.Set("Authorization", "Bearer "+good) + resp2, err := client.Do(req) + if err != nil { + t.Fatalf("TLS POST authed: %v", err) + } + resp2.Body.Close() + if resp2.StatusCode == http.StatusUnauthorized { + t.Fatal("valid bearer wrongly rejected over TLS") + } +} + +// TestTLSConfigMountedCert: when cert+key files exist, TLSConfig loads them +// (selfSigned=false) — the /certs/server.{crt,key} shared-cert path. +func TestTLSConfigMountedCert(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "server.crt") + keyPath := filepath.Join(dir, "server.key") + writePEMKeypair(t, certPath, keyPath) + + cfg, selfSigned, err := TLSConfig(certPath, keyPath, "localhost") + if err != nil { + t.Fatalf("TLSConfig mounted: %v", err) + } + if selfSigned { + t.Fatal("expected selfSigned=false when cert files exist") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected one loaded certificate, got %+v", cfg) + } +} + +// TestPlaintextPathServes: the CONTROL_INSECURE opt-out serves plain HTTP. This +// mirrors main.go's insecure branch (httpSrv.ListenAndServe with the same +// handler) — a plaintext client reaches /healthz. +func TestPlaintextPathServes(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: New(&fakeExporter{}, nil).Handler()} + go func() { _ = httpSrv.Serve(ln) }() + defer httpSrv.Close() + + resp, err := (&http.Client{Timeout: 3 * time.Second}).Get("http://" + ln.Addr().String() + "/healthz") + if err != nil { + t.Fatalf("plaintext GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("plaintext healthz = %d, want 200", resp.StatusCode) + } +} + +// writePEMKeypair mints a self-signed cert via the same helper and writes it as +// PEM cert+key files, so the mounted-cert load path can be exercised. +func writePEMKeypair(t *testing.T, certPath, keyPath string) { + t.Helper() + cert, err := selfSignedCert("localhost") + if err != nil { + t.Fatalf("selfSignedCert: %v", err) + } + certPEM, keyPEM, err := certToPEM(cert) + if err != nil { + t.Fatalf("certToPEM: %v", err) + } + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } +} diff --git a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel index 6024b7ce5c7..dc4bdca23b1 100644 --- a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel @@ -36,12 +36,14 @@ pl_go_test( name = "controller_test", srcs = [ "controller_test.go", + "order_chunk_test.go", "order_query_test.go", ], embed = [":controller"], deps = [ "//src/vizier/services/adaptive_export/internal/anomaly", "//src/vizier/services/adaptive_export/internal/kubescape", + "//src/vizier/services/adaptive_export/internal/reconcile", "//src/vizier/services/adaptive_export/internal/sink", ], ) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 5a4b44ec1c4..01e317359c0 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -33,7 +33,9 @@ import ( "context" "errors" "fmt" + "strings" "sync" + "sync/atomic" "time" log "github.com/sirupsen/logrus" @@ -135,6 +137,10 @@ type Config struct { // captures over overlapping windows. Defaulted to 30s in defaulted(). ExportAllFloor time.Duration + // OrderChunk is the sub-window the ordered path walks the capture window in. + // Defaulted in defaulted(); env ADAPTIVE_ORDER_CHUNK_SEC overrides. + OrderChunk time.Duration + // === Throughput-protection knobs === // // At high anomaly rates (many concurrent active hashes), the default @@ -207,9 +213,20 @@ func (c *Config) defaulted() Config { if out.ExportAllFloor == 0 { out.ExportAllFloor = 30 * time.Second } + if out.OrderChunk == 0 { + out.OrderChunk = defaultOrderChunk + } return out } +const ( + // defaultOrderChunk = the full control lookback: one query per table, subdividing + // only on timeout (pre-chunking every table 10x-amplifies queries on one PEM). + defaultOrderChunk = 600 * time.Second + // orderMinChunk is the adaptive-subdivision floor; a span this small that still fails is surfaced. + orderMinChunk = 1 * time.Second +) + // Controller is the live orchestrator. One instance per operator process. type Controller struct { trig Trigger @@ -243,8 +260,17 @@ type Controller struct { exportAllMu sync.Mutex exportAllAt map[string]time.Time // per-target floor for OrderExportAll (steer-all) + + // Consecutive transient failures on the ordered path; any success resets it. Above + // orderBreakerTrip captureSpan stops subdividing so a saturated PEM isn't flooded. + orderTimeoutStreak atomic.Int32 } +const ( + maxOrderSplitDepth = 3 // cap captureSpan recursion (≤2^depth leaves/chunk) + orderBreakerTrip = 8 // consecutive timeouts above which subdivision stops +) + // New wires a Controller. nil clock falls through to RealClock. // nil querier disables the rev-1 push path (controller will only // write attribution rows; expects cloud's retention plugin to write @@ -295,23 +321,78 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end return errors.New("controller: no pixie querier (operator-side push disabled)") } now := c.clock.Now() - q, err := pxl.QueryFor(table, target, start, end, now) - if err != nil { - return err + chunk := c.cfg.OrderChunk + if chunk <= 0 { + chunk = defaultOrderChunk + } + // Walk the window in OrderChunk sub-windows; captureSpan subdivides any that time out. + var readTotal, wroteTotal int + var firstErr error + for s := start; s.Before(end); s = s.Add(chunk) { + e := s.Add(chunk) + if e.After(end) { + e = end + } + qid := fmt.Sprintf("%s:%d-%d", queryID, s.Unix(), e.Unix()) + r, w, err := c.captureSpan(target, table, s, e, qid, 0) + readTotal += r + wroteTotal += w + if err != nil && firstErr == nil { + firstErr = err + } + } + recErr := "" + if firstErr != nil { + recErr = firstErr.Error() + } + // One reconcile row per table, aggregating every chunk. + c.cfg.Rec.Record(context.Background(), reconcile.Row{ + TS: now, Mode: "ordered", Table: table, + Namespace: target.Namespace, Pod: target.Pod, + WinStart: start, WinEnd: end, + ReadCount: int64(readTotal), WroteCount: int64(wroteTotal), + WriteErr: recErr, Hostname: c.cfg.Hostname, + }) + return firstErr +} + +// captureSpan captures [start,end) for one table, subdividing a transient failure into +// half-spans down to orderMinChunk. Bounded by maxOrderSplitDepth + the breaker so a +// saturated PEM isn't stormed; overlapping retries dedupe in the ReplacingMergeTree tables. +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (int, int, error) { + r, w, e := c.orderQuerySlice(target, table, start, end, queryID) + if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { + return r, w, e + } + if depth >= maxOrderSplitDepth { + return r, w, e // depth-capped: don't amplify a persistently-failing span + } + if c.orderTimeoutStreak.Load() > orderBreakerTrip { + // PEM saturated (sustained timeouts) — splitting would only add load. + log.WithFields(log.Fields{"table": table, "pod": target.Pod}). + Warn("ordered capture: circuit-breaker open (PEM saturated), not subdividing") + return r, w, e + } + log.WithError(e).WithFields(log.Fields{ + "table": table, "pod": target.Pod, "span": end.Sub(start).String(), "depth": depth, + }).Warn("ordered capture: transient failure, subdividing span") + mid := start.Add(end.Sub(start) / 2) + r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l", depth+1) + r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r", depth+1) + if e1 != nil { + return r1 + r2, w1 + w2, e1 + } + return r1 + r2, w1 + w2, e2 +} + +// orderQuerySlice runs one (target, table, [start,end)) capture and writes the rows. +// It records no reconcile row — the OrderQuery driver aggregates and records once. +func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (int, int, error) { + now := c.clock.Now() + q, qerr := pxl.QueryFor(table, target, start, end, now) + if qerr != nil { + return 0, 0, qerr } - // Background ctx with per-op timeouts mirroring pushPixieRows: a control-ordered - // capture must complete independently of any anomaly window's lifecycle. - var readCount, wroteCount int - var recErr string - defer func() { - c.cfg.Rec.Record(context.Background(), reconcile.Row{ - TS: now, Mode: "ordered", Table: table, - Namespace: target.Namespace, Pod: target.Pod, - WinStart: start, WinEnd: end, - ReadCount: int64(readCount), WroteCount: int64(wroteCount), - WriteErr: recErr, Hostname: c.cfg.Hostname, - }) - }() if c.globalSem != nil { c.globalSem <- struct{}{} defer func() { <-c.globalSem }() @@ -320,25 +401,46 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { - recErr = qerr.Error() - return qerr + if isRetriableSpanErr(qerr) { + c.orderTimeoutStreak.Add(1) // feed the saturation breaker + } + return 0, 0, qerr } - readCount = len(rows) + c.orderTimeoutStreak.Store(0) // a completed query clears the breaker if len(rows) == 0 { - return nil // nothing to persist; the read/0-wrote reconcile row still records it + return 0, 0, nil } wctx, wcancel := context.WithTimeout(context.Background(), 60*time.Second) werr := c.sink.WritePixieRows(wctx, table, rows) wcancel() if werr != nil { - recErr = werr.Error() - return werr + return len(rows), 0, werr } - wroteCount = len(rows) log.WithFields(log.Fields{ "table": table, "rows": len(rows), "pod": target.Pod, "query_id": queryID, }).Info("ordered pixie rows written to forensic_db (dx→AE /query)") - return nil + return len(rows), len(rows), nil +} + +// isRetriableSpanErr reports whether an error is a transient timeout/overload (vs. a +// structural error like a missing table) — covers ctx deadlines and gRPC status strings. +func isRetriableSpanErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + s := strings.ToLower(err.Error()) + for _, m := range []string{ + "deadline", "timeout", "exceeded", "resourceexhausted", + "resource exhausted", "unavailable", "context canceled", "context cancelled", + } { + if strings.Contains(s, m) { + return true + } + } + return false } // OrderExportAll runs a one-shot OrderQuery for EVERY configured pixie table for diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go new file mode 100644 index 00000000000..10522afd941 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -0,0 +1,226 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +// Tests for the CHUNKED + adaptively-subdividing ordered capture path — the +// durable fix for heavy tables (dc_snoop) losing the per-query deadline race under +// the OrderExportAll fan-out. Each chunk is a both-sides bounded pixie query; a +// chunk that still times out under contention is halved down to orderMinChunk. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/reconcile" +) + +// countingQuerier counts Query calls and can fail the first failN calls with a +// configurable error (to exercise adaptive subdivision) or fail every call. +type countingQuerier struct { + mu sync.Mutex + calls int + rows []map[string]any + failN int // fail the first failN calls, then succeed + failAll bool // fail every call + failErr error // error to return on a failed call +} + +func (q *countingQuerier) Query(context.Context, string) ([]map[string]any, error) { + q.mu.Lock() + defer q.mu.Unlock() + q.calls++ + if q.failAll || q.calls <= q.failN { + return nil, q.failErr + } + return q.rows, nil +} + +func (q *countingQuerier) callCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.calls +} + +// recordingRec captures every reconcile Row so a test can assert the ordered path +// records exactly ONE aggregated row per table (not one per chunk). +type recordingRec struct { + mu sync.Mutex + rows []reconcile.Row +} + +func (r *recordingRec) Record(_ context.Context, row reconcile.Row) { + r.mu.Lock() + defer r.mu.Unlock() + r.rows = append(r.rows, row) +} + +func chunkCtl(snk Sink, q PixieQuerier, rec reconcile.Recorder, chunk time.Duration) *Controller { + cfg := defaultCfg() + cfg.OrderChunk = chunk + cfg.Rec = rec + c := New(newFakeTrigger(), snk, cfg, &fakeClock{t: canonicalEventTime}) + if q != nil { + c = c.WithPixieQuerier(q) + } + return c +} + +var errDeadline = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") + +// A wide window is walked in OrderChunk-sized slices: one pixie query per chunk, +// each writing its rows. 180s window / 60s chunk = 3 bounded queries. +func TestOrderQueryChunksWideWindow(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "whoami"}}} + end := canonicalEventTime + start := end.Add(-180 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-w"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if got := q.callCount(); got != 3 { + t.Errorf("want 3 chunk queries for a 180s/60s window, got %d", got) + } + if got := snk.count("dc_snoop"); got != 3 { + t.Errorf("want 3 rows written (one per chunk), got %d", got) + } +} + +// The ordered path records exactly ONE reconcile row per table, aggregating the +// per-chunk read/wrote counts — a forensic dump reads per-table, not per-chunk. +func TestOrderQuerySingleReconcileRowPerTable(t *testing.T) { + snk := newRecordingSink() + rec := &recordingRec{} + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}} + end := canonicalEventTime + start := end.Add(-120 * time.Second) // 2 chunks + if err := chunkCtl(snk, q, rec, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-r"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if len(rec.rows) != 1 { + t.Fatalf("want 1 aggregated reconcile row, got %d", len(rec.rows)) + } + if rec.rows[0].ReadCount != 2 || rec.rows[0].WroteCount != 2 { + t.Errorf("want aggregated read=2 wrote=2 across chunks, got read=%d wrote=%d", + rec.rows[0].ReadCount, rec.rows[0].WroteCount) + } + if rec.rows[0].WriteErr != "" { + t.Errorf("clean capture must record no error, got %q", rec.rows[0].WriteErr) + } +} + +// A chunk that fails with a TRANSIENT (deadline) error is retried as narrower +// half-spans and recovers — the flaky-capture fix. The querier fails only its first +// call, so the initial full-chunk query subdivides and the halves succeed. +func TestCaptureSpanSubdividesOnTransientError(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) // single 60s chunk covers it → one initial query + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-t"); err != nil { + t.Fatalf("transient failure must recover via subdivision, got %v", err) + } + // call 1 (8s span) fails → split into two 4s halves (calls 2 & 3), both succeed. + if got := q.callCount(); got != 3 { + t.Errorf("want 3 calls (1 failed + 2 half-span retries), got %d", got) + } + if got := snk.count("dc_snoop"); got != 2 { + t.Errorf("want 2 half-span writes after subdivision, got %d", got) + } +} + +// A NON-transient error (e.g. a missing table) surfaces immediately — +// no wasteful subdivision. Exactly one query per chunk, error returned. +func TestCaptureSpanDoesNotSplitNonTransient(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errors.New("table 'no_such_table' not found")} + end := canonicalEventTime + start := end.Add(-30 * time.Second) // < one chunk → single chunk + // Uses a KNOWN table with a non-transient failure: the point is the error + // class, not the table. (Previously keyed on dx_bpf, which no longer exists — + // OrderQuery now rejects it before the querier runs, so nothing subdivided.) + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-n") + if err == nil { + t.Fatal("non-transient error must surface") + } + if got := q.callCount(); got != 1 { + t.Errorf("non-transient error must NOT subdivide; want 1 call, got %d", got) + } +} + +// A persistently-timing-out span subdivides down to orderMinChunk and then surfaces +// the error instead of looping forever — the recursion terminates at the floor. +func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-4 * time.Second) // 4s → 2s → 1s (floor), bounded call count + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-f") + if err == nil { + t.Fatal("a span that never succeeds must ultimately surface the error") + } + // 4s→(2s,2s)→each (1s,1s): calls = 1 + 2 + 4 = 7, finite. Assert it stayed bounded. + if got := q.callCount(); got == 0 || got > 15 { + t.Errorf("subdivision must terminate at orderMinChunk with a bounded call count, got %d", got) + } +} + +// A persistently-timing-out multi-chunk window must NOT explode into a query storm. +// Without guards, 10 chunks each subdividing 60s→1s ≈ 10×64 = 640 queries against a +// saturated PEM. The depth cap (≤2^3 leaves/chunk) + circuit-breaker (stop +// subdividing after orderBreakerTrip consecutive timeouts) bound it hard. +func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-600 * time.Second) // 10 chunks @ 60s, all time out + _ = chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-storm") + got := q.callCount() + if got > 60 { + t.Errorf("depth-cap + circuit-breaker must bound the storm; got %d calls (want <=60, ungrafted would be ~640)", got) + } + if got < 10 { + t.Errorf("must still attempt each of the 10 chunks at least once; got %d", got) + } +} + +// A healthy PEM (queries succeed) must NOT trip the breaker — subdivision stays +// available for genuinely-oversized windows. A querier that fails ONCE then succeeds +// still subdivides and recovers (breaker reset by the success). +func TestCircuitBreakerResetsOnSuccess(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-reset"); err != nil { + t.Fatalf("single transient failure must recover (breaker must not latch); got %v", err) + } + if snk.count("dc_snoop") < 1 { + t.Errorf("recovered subdivision must write rows; got %d", snk.count("dc_snoop")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile.go b/src/vizier/services/adaptive_export/internal/pxl/compile.go index cdd21c5313c..82d312c36b9 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile.go @@ -71,8 +71,6 @@ func CompilePassthrough(table string, window time.Duration) (string, error) { // PodEnrichPxL. var darkVectorTables = map[string]bool{ "dc_snoop": true, "creds_change": true, - "dx_vfs_events": true, "dx_unlink": true, "dx_dlookup": true, - "dx_mprotect": true, "dx_bpf": true, "dx_ptrace": true, } // IsDarkVector reports whether table is a pid-keyed dx tracepoint table. @@ -95,8 +93,18 @@ const darkProcStatsWindow = "-2m" // aggregator/kelvin asid, not the per-PEM asid of the data, so pid+asid never // matches. df.pod here is the BARE pod name (proc.ctx['pod']). Best-effort: blank // for host/transient pids (correct — no pod). +// NodeHostname is the AE pod's k8s node (NODE_NAME), stamped as a literal onto the +// pid-keyed dark tables. The process_stats-merge node resolution misses transient +// attack pids (most dc_snoop rows blank), leaving the table un-px-readable; AE is +// node-local, so its node is the correct shard key for every row it captures. +var NodeHostname string + func PodEnrichPxL(table string) string { if darkVectorTables[table] { + hostLine := "df.hostname = df.node\n" + if NodeHostname != "" { + hostLine = "df.hostname = '" + escapePxL(NodeHostname) + "'\n" + } // process_stats is the COST of the dark-table merge: a busy node samples // every live pid every ~10-30s, so a wide window is a huge scan that // competes with the fast native-table queries for the shared query-slot @@ -108,12 +116,23 @@ func PodEnrichPxL(table string) string { return "proc = px.DataFrame(table='process_stats', start_time='" + darkProcStatsWindow + "')\n" + "proc.pod = proc.ctx['pod']\n" + "proc.namespace = proc.ctx['namespace']\n" + + // node resolved from the SAME process_stats upid the pod/ns come from, so + // dark-vector rows (dc_snoop et al.) carry hostname and become px-readable + // (#136). Transient attack pids that miss process_stats resolve blank — + // the same accepted limitation as pod/ns above. + "proc.node = px.upid_to_node_name(proc.upid)\n" + "proc.pid = px.upid_to_pid(proc.upid)\n" + - "proc = proc.groupby(['pod', 'namespace', 'pid']).agg()\n" + - "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + "proc = proc.groupby(['pod', 'namespace', 'node', 'pid']).agg()\n" + + "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + + hostLine } return "df.namespace = px.upid_to_namespace(df.upid)\n" + - "df.pod = px.upid_to_pod_name(df.upid)\n" + "df.pod = px.upid_to_pod_name(df.upid)\n" + + // hostname = the capture node — the leading ORDER BY column on every + // socket_tracer table. AE left it empty (only stack_trace stamped it), so + // px reads of these tables (and the #136 order-UUID views) could not filter + // by hostname and the pushdown prefix (hostname,event_time) was unusable. + "df.hostname = px.upid_to_node_name(df.upid)\n" } // Render fills a CompilePassthrough template with the precise [sliceStart, diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile_test.go b/src/vizier/services/adaptive_export/internal/pxl/compile_test.go index e5a16cc3e56..8d0065aab54 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile_test.go @@ -95,7 +95,7 @@ func TestPodEnrichPxL_DarkVsNative(t *testing.T) { if !strings.Contains(native, "px.upid_to_pod_name(df.upid)") { t.Errorf("native table must resolve pod via upid: %q", native) } - for _, tbl := range []string{"dc_snoop", "creds_change", "dx_bpf", "dx_ptrace"} { + for _, tbl := range []string{"dc_snoop", "creds_change"} { q := PodEnrichPxL(tbl) if strings.Contains(q, "df.upid") { t.Errorf("%s (pid-keyed) must NOT reference df.upid: %q", tbl, q) diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index 4f9d8d6d37c..69703f658a8 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -63,21 +63,17 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim var b strings.Builder b.WriteString(pxSetMaxRows) b.WriteString("import px\n") - b.WriteString("df = px.DataFrame(table='" + pixieSourceFor(table) + "', start_time='" + relStart + "')\n") + // Bound the source scan on both sides; the df.time_ < sliceEnd filter trims the exact upper bound. + dfArgs := "table='" + pixieSourceFor(table) + "', start_time='" + relStart + "'" + if relEnd := relEndBound(now, sliceEnd); relEnd != "" { + dfArgs += ", end_time='" + relEnd + "'" + } + b.WriteString("df = px.DataFrame(" + dfArgs + ")\n") b.WriteString("df = df[df.time_ >= px.int64_to_time(" + strconv.FormatInt(sliceStart.UnixNano(), 10) + ")]\n") b.WriteString("df = df[df.time_ < px.int64_to_time(" + strconv.FormatInt(sliceEnd.UnixNano(), 10) + ")]\n") - // Native tables: px.upid_to_pod_name returns "/" (carnot: - // metadata_ops.h UPIDToPodNameUDF::Exec → absl::Substitute("$0/$1", ns, name)), - // not the bare pod name. Dark-vector tracepoint tables (pid-keyed) resolve pod - // via a process_stats pid-merge instead and yield a BARE pod name (dx#126). + // px.upid_to_pod_name yields "/"; dark-vector tables resolve pod via a pid-merge (bare name). if table == "stack_trace" { - // stack_trace is the CANONICAL native continuous profiler (stack_traces.beta, - // upid-keyed — NOT a pid tracepoint, so NOT a dark-vector pid-merge). Resolve - // pod/namespace/container/hostname exactly like the export preset - // (script/presets/stack_trace.pxl) and stamp event_time = time_ so the CH - // stack_trace row is complete. df.ctx['pod'] is the NAMESPACED "/" - // key (verified live), so the pod filter is namespaced — same as the native - // upid_to_pod_name path below. + // Native profiler (stack_traces.beta): resolve pod/ns/container from ctx, stamp event_time. b.WriteString("df.namespace = df.ctx['namespace']\n") b.WriteString("df.pod = df.ctx['pod']\n") b.WriteString("df.container = df.ctx['container']\n") @@ -94,22 +90,11 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim } } } else if IsDarkVector(table) { - // Dark-vector tracepoints emit a RAW kernel pid. The malignant transient - // pids an incident actually produces — an attack's whoami/cat/getent - // children — are too short-lived to land in process_stats, so their - // pod/namespace resolves BLANK; a pod (or even namespace) filter drops - // exactly the evidence, which is why the dark tables came back empty. - // The AE is node-local (pem-direct → the node's own PEM), so the query is - // already scoped to the alert's node. - // - // ORDER MATTERS: drop the infra/self comms FIRST (env-driven, no recompile), - // THEN do the process_stats pid-merge. The node's dark stream is huge - // (Formatter/vector/runc/... thousands of rows per window); merging every - // one against process_stats is the query that timed out and silently - // dropped dc_snoop. Filtering comm first shrinks the merge to the handful - // of workload rows (bash/redis/whoami/cat), so the dark capture completes. + // Node-scoped (transient attack pids resolve blank ns, so no pod filter). Drop + // own-stack comms before the pid-merge to keep it cheap, then drop infra namespaces. b.WriteString(darkCommExclusion(table)) b.WriteString(PodEnrichPxL(table)) + b.WriteString(darkNamespaceExclusion()) } else { b.WriteString(PodEnrichPxL(table)) if t.Namespace != "" { @@ -130,11 +115,16 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim return b.String(), nil } -// pixieSourceFor returns the Pixie table a builtin is sourced FROM when it -// differs from the ClickHouse table it is written TO. stack_trace is written to -// CH as 'stack_trace' but sourced from the CANONICAL native continuous profiler -// 'stack_traces.beta' — the always-on Pixie profiler, NOT an AE-invented table. -// (Dotted-name DataFrames compile fine in a direct query; verified live.) +// relEndBound returns a relative end_time ("-s"), or "" when sliceEnd is at/after now. +func relEndBound(now, sliceEnd time.Time) string { + gap := now.Sub(sliceEnd) + if gap < time.Second { + return "" // at/after now → default end_time (scan to now) + } + return "-" + strconv.FormatInt(int64(gap/time.Second), 10) + "s" +} + +// pixieSourceFor maps a CH table to the pixie table it's read from (stack_trace ← stack_traces.beta). func pixieSourceFor(table string) string { if table == "stack_trace" { return "stack_traces.beta" @@ -142,18 +132,12 @@ func pixieSourceFor(table string) string { return table } -// darkVectorHasComm lists the dark-vector tables that carry a `comm` column, so -// the infra-comm exclusion only emits for those (stack_trace is upid-only). +// Dark-vector tables carrying a comm column (so the comm exclusion applies). var darkVectorHasComm = map[string]bool{ - "dc_snoop": true, "creds_change": true, "dx_vfs_events": true, - "dx_unlink": true, "dx_dlookup": true, "dx_mprotect": true, - "dx_bpf": true, "dx_ptrace": true, -} + "dc_snoop": true, "creds_change": true} -// darkExcludeCommsDefault is the node's own infra/self comms dropped from the -// node-scoped dark capture so the workload's activity stands out. Overridable at -// runtime via DC_SNOOP_EXCLUDE_COMMS (csv) — a process can be added without a -// recompile. Kept in sync with script.presets defaultExcludeComms. +// Own-stack + node/system comms dropped from the node-scoped dark capture; workload +// comms (redis-*, etc.) are never listed. Override via DC_SNOOP_EXCLUDE_COMMS (csv). var darkExcludeCommsDefault = []string{ "pem", "kelvin", "containerd", "containerd-shim", "runc", "node-agent", "runc:[2:INIT]", "runc:[1:CHILD]", @@ -165,10 +149,26 @@ var darkExcludeCommsDefault = []string{ "ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd", "vector-worker", "metrics-server", "local-path-prov", "portmap", "(udev-worker)", "systemd-resolve", "systemd-timesyn", + "systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel", + "loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled", + "grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy", + "pause", "systemd-logind", +} + +// Kernel-thread families whose names carry a variable suffix (kworker/u8:3) that +// exact match misses; dropped via px.contains. +var darkExcludeCommSubstrings = []string{ + "kworker", "ksoftirqd", "migration", "rcu_", "kthreadd", "kdevtmpfs", + "kcompactd", "khugepaged", "kswapd", "watchdog", "cpuhp", "ksmd", "irq/", +} + +// Infra namespaces dropped from the node-scoped dark capture. Blank-namespace rows +// (transient attack children) survive. Override via DC_SNOOP_EXCLUDE_NAMESPACES. +var darkExcludeNamespacesDefault = []string{ + "pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch", + "kube-system", "kube-public", "kube-node-lease", "local-path-storage", } -// darkCommExclusion builds the infra-comm drop filter for a dark-vector table -// that has a comm column. Returns "" for comm-less tables (stack_trace). func darkCommExclusion(table string) string { if !darkVectorHasComm[table] { return "" @@ -186,6 +186,26 @@ func darkCommExclusion(table string) string { for _, c := range comms { b.WriteString("df = df[df.comm != '" + escapePxL(c) + "']\n") } + for _, s := range darkExcludeCommSubstrings { + b.WriteString("df = df[px.logicalNot(px.contains(df.comm, '" + escapePxL(s) + "'))]\n") + } + return b.String() +} + +func darkNamespaceExclusion() string { + nss := darkExcludeNamespacesDefault + if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" { + nss = nil + for _, s := range strings.Split(v, ",") { + if s = strings.TrimSpace(s); s != "" { + nss = append(nss, s) + } + } + } + var b strings.Builder + for _, ns := range nss { + b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n") + } return b.String() } diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index 562ea794cc0..6ca0023fd61 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -18,6 +18,7 @@ package pxl import ( "errors" + "strconv" "strings" "testing" "time" @@ -257,14 +258,15 @@ func TestEscapePxL_TableDriven(t *testing.T) { // df = df[df.time_ < ...] 1 // df.namespace = px.upid_to_namespace(...) 1 // df.pod = px.upid_to_pod_name(...) 1 +// df.hostname = px.upid_to_node_name(...) 1 // df = df[df.namespace == '...'] 1 // df = df[df.pod == '...'] 1 // px.display(df, '...') 1 -// (trailing newline → empty 11th split) 1 +// (trailing newline → empty 12th split) 1 // -// Total: 10 statements + trailing empty == strings.Split == 11 entries. +// Total: 11 statements + trailing empty == strings.Split == 12 entries. func TestQueryFor_RejectsInjectionInTargetFields(t *testing.T) { - const wantLines = 11 + const wantLines = 12 cases := []struct { name string @@ -336,7 +338,105 @@ func TestQueryFor_PodOnlyRegexEscapesQuoteMetaInjection(t *testing.T) { if err != nil { t.Fatalf("QueryFor: %v", err) } - if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 9 { + if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 10 { t.Fatalf("pod-only path injection succeeded:\n%s", q) } } + +// TestQueryFor_EndTimeBoundsPastWindow — a window whose upper bound is in the past +// must emit a relative end_time so the PEM scan is bounded on BOTH sides (not +// [sliceStart, now]). The precise upper bound is still enforced by the df.time_ < +// nanos post-filter. +func TestQueryFor_EndTimeBoundsPastWindow(t *testing.T) { + // sliceEnd 2 minutes before now → end_time must appear. + end := fixedNow.Add(-2 * time.Minute) + start := fixedNow.Add(-7 * time.Minute) + q, err := QueryFor("dc_snoop", target, start, end, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "end_time='-120s'") { + t.Fatalf("past-window query must bound the source scan with end_time='-120s'; got:\n%s", q) + } + // exact upper bound still trimmed precisely in nanos. + if !strings.Contains(q, "df = df[df.time_ < px.int64_to_time("+ + strconv.FormatInt(end.UnixNano(), 10)+")]") { + t.Fatalf("precise nanos upper-bound filter must remain; got:\n%s", q) + } +} + +// TestQueryFor_NoEndTimeAtLiveEdge — a window that reaches now must NOT emit +// end_time (scan to the live edge), preserving the pre-chunking behavior for the +// most-recent slice. +func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedNow.Add(-1*time.Minute), fixedNow, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if strings.Contains(q, "end_time=") { + t.Fatalf("live-edge window must not bound end_time; got:\n%s", q) + } +} + +// TestQueryFor_DarkNamespaceExclusion — the node-scoped dark capture (dc_snoop) +// must drop infra namespaces (pl, kube-system, …) while KEEPING blank-namespace +// transient rows (the attack's short-lived children). Mirrors the shipped preset. +func TestQueryFor_DarkNamespaceExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + // namespace drops present for infra + for _, ns := range []string{"pl", "kube-system", "clickhouse"} { + if !strings.Contains(q, "df = df[df.namespace != '"+ns+"']") { + t.Errorf("dark capture must drop infra namespace %q; got:\n%s", ns, q) + } + } + // must NOT pin to the alert pod's namespace (node-scoped keeps blank + other workloads) + if strings.Contains(q, "df = df[df.namespace == '") { + t.Errorf("dark capture must not pin df.namespace ==; got:\n%s", q) + } + // host/CNI comm drops present + for _, c := range []string{"host-local", "systemd-udevd", "tailscaled", "kubevuln"} { + if !strings.Contains(q, "df = df[df.comm != '"+c+"']") { + t.Errorf("dark capture must drop host/CNI comm %q; got:\n%s", c, q) + } + } +} + +// TestQueryFor_DarkNamespaceExclusion_EnvOverride — DC_SNOOP_EXCLUDE_NAMESPACES +// replaces the default list. +func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) { + t.Setenv("DC_SNOOP_EXCLUDE_NAMESPACES", "foo,bar") + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "df = df[df.namespace != 'foo']") || !strings.Contains(q, "df = df[df.namespace != 'bar']") { + t.Errorf("env override must emit foo/bar drops; got:\n%s", q) + } + if strings.Contains(q, "df = df[df.namespace != 'pl']") { + t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q) + } +} + +// dc_snoop drops kernel-thread families (variable suffix) via px.logicalNot(px.contains), +// keeps workload comms (redis-server), and doesn't pin the alert pod's namespace. +func TestQueryFor_DarkCommSubstringExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + for _, sub := range []string{"kworker", "ksoftirqd", "rcu_"} { + want := "df = df[px.logicalNot(px.contains(df.comm, '" + sub + "'))]" + if !strings.Contains(q, want) { + t.Errorf("want kernel-thread drop %q; got:\n%s", want, q) + } + } + if strings.Contains(q, "df.comm != 'redis-server'") || strings.Contains(q, "df.comm, 'redis") { + t.Errorf("workload comm redis-* must NOT be excluded; got:\n%s", q) + } + if !strings.Contains(q, "df = df[df.comm != 'pause']") { + t.Errorf("want exact drop of 'pause'; got:\n%s", q) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/tables.go b/src/vizier/services/adaptive_export/internal/pxl/tables.go index d04f107fa2f..6bad717007d 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/tables.go +++ b/src/vizier/services/adaptive_export/internal/pxl/tables.go @@ -81,12 +81,6 @@ var builtinTables = []TableSpec{ {Name: "dc_snoop", Protocol: "tracepoint (dentry lookup, V1/V2)"}, {Name: "creds_change", Protocol: "tracepoint (commit_creds priv-esc, V7)"}, {Name: "stack_trace", Protocol: "profiler (stack_traces.beta, V9)"}, - {Name: "dx_vfs_events", Protocol: "tracepoint"}, - {Name: "dx_unlink", Protocol: "tracepoint"}, - {Name: "dx_dlookup", Protocol: "tracepoint"}, - {Name: "dx_mprotect", Protocol: "tracepoint"}, - {Name: "dx_bpf", Protocol: "tracepoint"}, - {Name: "dx_ptrace", Protocol: "tracepoint"}, } // Registry is the extension surface for users to register their own diff --git a/src/vizier/services/adaptive_export/internal/pxl/tables_test.go b/src/vizier/services/adaptive_export/internal/pxl/tables_test.go index d0d8a38ed54..e410c61149c 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/tables_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/tables_test.go @@ -27,11 +27,12 @@ import ( // kafka_events.beta, amqp_events, mux_events, tls_events, conn_stats). // Update this guard if the spec adds / removes a table. // 13 socket_tracer tables + 9 dark-vector tables: dc_snoop (V1/V2), creds_change -// (V7), stack_trace (V9) — canonical schemas, active — plus dx_vfs_events, -// dx_unlink, dx_dlookup, dx_mprotect, dx_bpf, dx_ptrace reserved for the bpftraces -// still to be written (entlein/dx#126). Update this guard if the spec changes. +// (V7), stack_trace (V9) — canonical schemas, active. The six reserved dx_* +// tracepoint tables (dx_vfs_events/dx_unlink/dx_dlookup/dx_mprotect/dx_bpf/ +// dx_ptrace) were removed: no bpftrace was ever deployed for them, so every +// steer-all query on them could only fail PxL compile. Update if the spec changes. func TestBuiltinTables_Count(t *testing.T) { - const want = 22 + const want = 16 if got := len(builtinTables); got != want { t.Fatalf("builtinTables = %d entries, want %d", got, want) } diff --git a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel index 0445d9211f4..8ffeb29b8e4 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel @@ -21,6 +21,8 @@ go_library( name = "trigger", srcs = [ "clickhouse.go", + "dedup.go", + "metrics.go", "watermark.go", ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/trigger", @@ -28,6 +30,8 @@ go_library( deps = [ "//src/vizier/services/adaptive_export/internal/chhttp", "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_sirupsen_logrus//:logrus", ], ) @@ -37,10 +41,15 @@ pl_go_test( srcs = [ "clickhouse_internal_test.go", "clickhouse_test.go", + "dedup_test.go", "fingerprint_bench_test.go", + "lookback_test.go", "oracle_test.go", "watermark_test.go", ], embed = [":trigger"], - deps = ["//src/vizier/services/adaptive_export/internal/kubescape"], + deps = [ + "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus/testutil", + ], ) diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go index 80e03f3b942..1f548e37cc2 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go @@ -84,13 +84,48 @@ type Config struct { // hardcoded to 5s, which under any backlog caused every poll to // time out mid-stream → watermark never advanced. HTTPTimeout time.Duration + + // Lookback (#97 / F8 / AE-9): when > 0, each poll re-scans + // [watermark-Lookback, ∞) instead of the strict [watermark, ∞) and + // dedupes re-seen rows by content fingerprint, so an out-of-order / + // clock-skewed / restart-buried row that lands within the window is + // still processed EXACTLY ONCE (no drop, no duplicate). Rows below + // watermark-Lookback stay dropped — the documented bound. 0 keeps + // the legacy strict high-water-mark behavior (anything below the + // watermark is dropped forever). Production default is 300s via + // ADAPTIVE_TRIGGER_LOOKBACK_SEC in cmd/main.go; the zero value here + // is legacy so existing callers/tests are unchanged. + Lookback time.Duration + + // MaxSkew is the wall-clock poison clamp (#97): a row whose + // NORMALIZED event_time is more than MaxSkew past now is still + // emitted once, but never advances the watermark, so a single + // corrupted/oversized timestamp (the 1.78e18 leftover of loadtest + // E8) cannot jump the cursor past all real data and silently halt + // the trigger. Also applied to the persisted watermark at load, so + // an ALREADY-poisoned cursor self-recovers on restart without the + // manual `ALTER TABLE trigger_watermark DELETE`. <=0 → 1h. + MaxSkew time.Duration + + // DedupMaxEntries caps the lookback dedup set (memory bound). An + // in-window fingerprint evicted by capacity may re-emit once, so + // size it >= the max rows expected per lookback window. + // <=0 → 4*PollLimit. + DedupMaxEntries int } +// defaultMaxSkew is the default wall-clock poison-clamp bound (#97): +// an event_time more than this far in the future is implausible. +const defaultMaxSkew = time.Hour + // ClickHouseHTTP polls forensic_db.
over the ClickHouse HTTP // interface, scoped to a single node. type ClickHouseHTTP struct { cfg Config client *http.Client + // now is the wall clock used by the poison clamp (#97). + // Injectable for deterministic tests; time.Now in production. + now func() time.Time } // New validates Config and returns a ready trigger. @@ -143,9 +178,19 @@ func New(cfg Config) (*ClickHouseHTTP, error) { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 30 * time.Second } + if cfg.Lookback < 0 { + return nil, fmt.Errorf("trigger: Lookback must be >= 0 (got %v)", cfg.Lookback) + } + if cfg.MaxSkew <= 0 { + cfg.MaxSkew = defaultMaxSkew + } + if cfg.DedupMaxEntries <= 0 { + cfg.DedupMaxEntries = 4 * cfg.PollLimit + } return &ClickHouseHTTP{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, + now: time.Now, }, nil } @@ -169,11 +214,15 @@ func (t *ClickHouseHTTP) Subscribe(ctx context.Context) (<-chan kubescape.Event, func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { defer close(out) // Watermark uses event_time as the cursor PLUS a set of row - // fingerprints already pushed at that exact event_time. This - // closes the race where two kubescape rows share the same - // event_time but the second arrives after our previous poll: the - // query is `event_time >= watermark` (inclusive) and we skip rows - // whose fingerprint we have already seen at the boundary. + // fingerprints already pushed. In legacy strict mode (Lookback==0) + // the query is `event_time >= watermark` (inclusive) and the + // fingerprint set covers only the exact boundary event_time — + // closing the race where two kubescape rows share the same + // event_time but the second arrives after our previous poll. With + // a bounded lookback (#97, the F8/AE-9 fix) the query starts at + // max(0, watermark-Lookback) and the fingerprint set is a bounded + // LRU over the whole re-scanned window, so out-of-order / skewed / + // restart-buried rows inside the window are captured exactly once. // // Cold-start order: persistent store > InitialWatermark > 0. // The persistent store is the production answer to "operator @@ -203,7 +252,39 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // pre-fix persisted seconds watermark (or a non-seconds InitialWatermark) // is interpreted on the same scale as chNormEventTimeNanos in the SQL. watermark = normalizeEventTimeNanos(watermark) + maxSkewNS := uint64(t.cfg.MaxSkew.Nanoseconds()) + lookbackNS := uint64(t.cfg.Lookback.Nanoseconds()) + // Self-recovery from an ALREADY-poisoned persisted cursor (#97 T1): + // a pre-fix deployment could have persisted a far-future watermark + // (loadtest E8's leftover 1.78e18-style value). Clamp it to + // wall-clock so fresh rows flow again on restart WITHOUT the manual + // `ALTER TABLE trigger_watermark DELETE WHERE 1=1` + redeploy. + if nowNS := uint64(t.now().UnixNano()); watermark > nowNS+maxSkewNS { + log.WithFields(log.Fields{"watermark": watermark, "clamped_to": nowNS}). + Warn("trigger: persisted watermark is implausibly far in the future — clamping to wall-clock (poison recovery, #97)") + watermark = nowNS + } + wmGauge := metricWatermarkNS.WithLabelValues(t.cfg.Table, t.cfg.Hostname) + wmGauge.Set(float64(watermark)) + // Dedup state. Strict mode (Lookback==0) keeps the legacy exact + // boundary set; lookback mode dedupes the whole re-scanned window + // with a bounded LRU (#97). rejectedSeen exists only in strict mode: + // a clamp-rejected row never falls below the cursor, so without a + // fingerprint record it would re-emit on every poll. seenAtBoundary := map[string]bool{} + var seenInWindow *dedupLRU + var rejectedSeen *dedupLRU + if lookbackNS > 0 { + seenInWindow = newDedupLRU(t.cfg.DedupMaxEntries) + } else { + rejectedSeen = newDedupLRU(t.cfg.DedupMaxEntries) + } + // catchup lifts a poll's lower bound above the sliding lookback + // floor while an in-window backlog is wider than PollLimit: without + // it every poll would re-fetch the same fully-deduped first + // PollLimit rows and never reach deeper into the window. Cleared as + // soon as a poll returns under capacity (back to full-window scans). + var catchup uint64 ticker := time.NewTicker(t.cfg.PollInterval) defer ticker.Stop() @@ -253,7 +334,21 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { }() pollOnce := func() { - rows, maxSeen, err := t.fetchSince(ctx, watermark) + // Bounded lookback (#97): scan from max(0, watermark-Lookback) + // so rows that landed BELOW the cursor (out-of-order, clock + // skew, restart burial) are still fetched; the dedup LRU makes + // re-seen rows exactly-once. Lookback==0 → legacy strict HWM. + queryFrom := watermark + if lookbackNS > 0 { + queryFrom = 0 + if watermark > lookbackNS { + queryFrom = watermark - lookbackNS + } + if catchup > queryFrom { + queryFrom = catchup + } + } + rows, maxFetched, err := t.fetchSince(ctx, queryFrom) // Partial-read tolerance: when the body read is cut short by // HTTP timeout / connection reset, fetchSince returns the rows // it managed to parse + err. We still process those rows so @@ -267,6 +362,19 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { log.WithError(err).WithField("partial_rows", len(rows)). Warn("trigger: poll partial — advancing on what parsed") } + // Wall-clock poison clamp (#97): any normalized event_time past + // now+MaxSkew must never advance the cursor. acceptedMax is the + // advancement target — the max normalized event_time among rows + // that PASS the clamp. With no poison rows it equals maxFetched, + // so the monotonic happy path is byte-identical to before. + skewLimit := uint64(t.now().UnixNano()) + maxSkewNS + acceptedMax := uint64(0) + for _, row := range rows { + if evn := normalizeEventTimeNanos(row.EventTime); evn <= skewLimit && evn > acceptedMax { + acceptedMax = evn + } + } + wmAtPollStart := watermark nextSeen := map[string]bool{} // Periodic in-loop save: when pollOnce is draining a large // initial backlog, the watermark advances long before the @@ -276,45 +384,122 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // with the time-based throttle inside flushWatermark, this // produces at most one persistent INSERT per WatermarkSaveInterval. const saveEveryN = 256 - skippedAtBoundary := 0 + skippedSeen := 0 + emitted := 0 for i, row := range rows { fp := rowFingerprint(row) // Cursor comparisons are in NORMALIZED nanos (F8): the raw // event_time unit is not enforced, so compare on the same scale - // as the SQL filter (chNormEventTimeNanos) and maxSeen. + // as the SQL filter (chNormEventTimeNanos) and acceptedMax. evn := normalizeEventTimeNanos(row.EventTime) - if evn == watermark && seenAtBoundary[fp] { - skippedAtBoundary++ - continue // already pushed in a prior poll at this exact boundary + if lookbackNS > 0 { + if seenInWindow.Contains(fp) { + skippedSeen++ + continue // already pushed in a prior scan of this window + } + } else { + if evn == watermark && seenAtBoundary[fp] { + skippedSeen++ + continue // already pushed in a prior poll at this exact boundary + } + if rejectedSeen.Contains(fp) { + continue // clamp-rejected row re-fetched (it never sinks below the cursor) + } } - ev, err := kubescape.Extract(row) - if err != nil { - log.WithError(err).Debug("trigger: skip incomplete row") + poison := evn > skewLimit + ev, exErr := kubescape.Extract(row) + if exErr != nil { + log.WithError(exErr).Debug("trigger: skip incomplete row") + // Register the fingerprint anyway (lookback / poison): + // the row can never become extractable, and without a + // record it would be re-fetched + re-logged every poll + // for as long as it stays above the scan floor. + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) + } continue } - // Promote the per-row (normalized) event_time into the watermark - // immediately so flushWatermark below can persist mid-drain. - if evn > watermark { - watermark = evn - dirty = true + if poison { + // Emit the row once (it may be a real anomaly with a + // mangled timestamp) but do NOT let it advance the + // cursor: one 1.78e18 row must not jump the watermark + // past all real seconds rows (F8 halt). + metricEventTimeRejected.Inc() + log.WithFields(log.Fields{ + "event_time": row.EventTime, + "normalized": evn, + "skew_limit": skewLimit, + }).Warn("trigger: event_time beyond wall-clock skew bound — processing row WITHOUT advancing watermark (poison clamp, #97)") + } else { + if evn < wmAtPollStart { + // A row the legacy strict HWM would have dropped — + // captured via the lookback (T2). Observable proof + // the fix is doing work (T3). + metricBelowWatermark.Inc() + } + // Promote the per-row (normalized) event_time into the watermark + // immediately so flushWatermark below can persist mid-drain. + if evn > watermark { + watermark = evn + dirty = true + wmGauge.Set(float64(watermark)) + } + } + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) } select { case out <- ev: case <-ctx.Done(): return } - if evn == maxSeen { + emitted++ + if !poison && evn == acceptedMax { nextSeen[fp] = true } if i > 0 && i%saveEveryN == 0 { flushWatermark() } } - if maxSeen > watermark { - watermark = maxSeen + if lookbackNS > 0 { + if acceptedMax > watermark { + watermark = acceptedMax + dirty = true + wmGauge.Set(float64(watermark)) + } + // Paging within the window: a saturated response means the + // window holds more rows than PollLimit — lift the floor so + // the next poll pages FORWARD instead of re-fetching the + // same deduped prefix forever. + if len(rows) >= t.cfg.PollLimit { + if emitted == 0 && skippedSeen == len(rows) { + // Every row in the saturated page was already seen — + // step past the page entirely (lookback analog of the + // legacy 1ns boundary escape). + catchup = maxFetched + 1 + } else if acceptedMax > catchup { + catchup = acceptedMax + } + } else { + catchup = 0 + } + // Entries below the sliding floor can never be re-fetched; + // evict them so the LRU stays at ~window size. + floor := uint64(0) + if watermark > lookbackNS { + floor = watermark - lookbackNS + } + seenInWindow.EvictBelow(floor) + } else if acceptedMax > watermark { + watermark = acceptedMax seenAtBoundary = nextSeen dirty = true - } else if maxSeen == watermark { + wmGauge.Set(float64(watermark)) + } else if acceptedMax == watermark { // no progress this tick — preserve boundary set, optionally extend for fp := range nextSeen { seenAtBoundary[fp] = true @@ -329,10 +514,11 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // the next poll, which is acceptable: the fingerprint dedup already // tolerates boundary overlap, and we prefer forward progress over // an infinite loop. - if skippedAtBoundary > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { + if skippedSeen > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { watermark++ seenAtBoundary = map[string]bool{} dirty = true + wmGauge.Set(float64(watermark)) log.WithField("watermark", watermark). Warn("trigger: boundary paging escape — advanced watermark by 1ns to unblock poll") } diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup.go b/src/vizier/services/adaptive_export/internal/trigger/dedup.go new file mode 100644 index 00000000000..ca1c5dda9a0 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup.go @@ -0,0 +1,98 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "container/list" + +// dedupLRU is a bounded, insertion-ordered set of row fingerprints, +// each tagged with the row's normalized event_time (nanos). It is the +// #97 (F8/AE-9) extension of the old single-boundary `seenAtBoundary` +// map: with a bounded lookback the trigger re-fetches every row in +// [watermark-Lookback, watermark] on each poll, so dedup must cover the +// whole window, not just the exact watermark boundary. +// +// Eviction is two-fold: +// - EvictBelow(floor): entries whose event_time has slid below the +// lookback floor can never be returned by the SELECT again, so they +// are dropped eagerly to keep the set at ~window size. +// - capacity: Add evicts the OLDEST INSERTION when over max, bounding +// memory even if the window holds more rows than expected. An +// in-window entry evicted by capacity may cause one duplicate emit — +// the documented trade-off for bounded memory (size it >= the max +// rows per window; default 4*PollLimit). +// +// Not goroutine-safe; owned by the single poll loop. +type dedupLRU struct { + max int + ll *list.List // front = oldest insertion + items map[string]*list.Element +} + +type dedupEntry struct { + fp string + evn uint64 // normalized event_time (nanos) +} + +func newDedupLRU(capacity int) *dedupLRU { + if capacity <= 0 { + capacity = 1 + } + return &dedupLRU{max: capacity, ll: list.New(), items: map[string]*list.Element{}} +} + +// Contains reports whether fp was Added and not yet evicted. +func (d *dedupLRU) Contains(fp string) bool { + _, ok := d.items[fp] + return ok +} + +// Add records fp with its normalized event_time. No-op if already +// present. Evicts oldest insertions while over capacity. +func (d *dedupLRU) Add(fp string, evn uint64) { + if _, ok := d.items[fp]; ok { + return + } + d.items[fp] = d.ll.PushBack(dedupEntry{fp: fp, evn: evn}) + for d.ll.Len() > d.max { + d.removeElement(d.ll.Front()) + } +} + +// EvictBelow drops entries with evn < floor, popping from the oldest +// insertion. Insertion order tracks the poll's ORDER BY event_time, so +// in the common case this removes exactly the expired prefix. A late +// arrival (low evn inserted after a higher one) may survive behind a +// newer entry until capacity eviction — harmless: Contains on an +// expired fp only suppresses a row the SELECT can no longer return. +func (d *dedupLRU) EvictBelow(floor uint64) { + for e := d.ll.Front(); e != nil; { + if e.Value.(dedupEntry).evn >= floor { + return + } + next := e.Next() + d.removeElement(e) + e = next + } +} + +// Len returns the number of live entries. +func (d *dedupLRU) Len() int { return d.ll.Len() } + +func (d *dedupLRU) removeElement(e *list.Element) { + delete(d.items, e.Value.(dedupEntry).fp) + d.ll.Remove(e) +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go new file mode 100644 index 00000000000..c139b21831c --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go @@ -0,0 +1,92 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "testing" + +func TestDedupLRU_AddContains(t *testing.T) { + d := newDedupLRU(8) + if d.Contains("a") { + t.Fatalf("empty LRU claims to contain a") + } + d.Add("a", 100) + d.Add("b", 200) + if !d.Contains("a") || !d.Contains("b") { + t.Fatalf("added fingerprints not found") + } + if d.Len() != 2 { + t.Fatalf("Len = %d, want 2", d.Len()) + } + // Duplicate Add is a no-op (no double-entry, no reorder). + d.Add("a", 100) + if d.Len() != 2 { + t.Fatalf("duplicate Add changed Len to %d", d.Len()) + } +} + +func TestDedupLRU_CapacityEvictsOldestInsertion(t *testing.T) { + d := newDedupLRU(3) + d.Add("a", 1) + d.Add("b", 2) + d.Add("c", 3) + d.Add("d", 4) // over capacity → "a" (oldest insertion) evicted + if d.Contains("a") { + t.Fatalf("oldest entry not evicted at capacity") + } + for _, fp := range []string{"b", "c", "d"} { + if !d.Contains(fp) { + t.Fatalf("entry %q evicted unexpectedly", fp) + } + } + if d.Len() != 3 { + t.Fatalf("Len = %d, want 3", d.Len()) + } +} + +func TestDedupLRU_EvictBelow(t *testing.T) { + d := newDedupLRU(8) + d.Add("a", 100) + d.Add("b", 200) + d.Add("c", 300) + d.EvictBelow(250) + if d.Contains("a") || d.Contains("b") { + t.Fatalf("entries below floor survived EvictBelow") + } + if !d.Contains("c") { + t.Fatalf("entry at/above floor was evicted") + } + // EvictBelow stops at the first entry >= floor (prefix semantics): + // a late arrival (low evn inserted AFTER a higher one) survives — + // documented as harmless. + d.Add("late", 50) + d.EvictBelow(250) + if !d.Contains("late") { + t.Fatalf("late-arrival entry behind a newer one should survive prefix eviction") + } +} + +func TestDedupLRU_ZeroCapacityIsSafe(t *testing.T) { + d := newDedupLRU(0) // clamped to 1 + d.Add("a", 1) + if !d.Contains("a") { + t.Fatalf("single entry not retained") + } + d.Add("b", 2) + if d.Contains("a") || !d.Contains("b") { + t.Fatalf("capacity-1 eviction wrong: a=%v b=%v", d.Contains("a"), d.Contains("b")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go new file mode 100644 index 00000000000..8fa153001fa --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go @@ -0,0 +1,321 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Bounded-lookback + wall-clock poison-clamp tests (#97 / F8 / AE-9). +// No live ClickHouse: a stub HTTP server implements the trigger's +// JSONEachRow contract INCLUDING the `>= ` watermark predicate, +// so re-poll semantics (the essence of lookback) are exercised for real. + +package trigger + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// fakeCH is a stub ClickHouse HTTP endpoint that stores rows and, like +// the real server, only returns rows whose NORMALIZED event_time is >= +// the bound parsed out of the trigger's SELECT. +type fakeCH struct { + mu sync.Mutex + rows []fakeRow + srv *httptest.Server +} + +type fakeRow struct { + eventTime uint64 // raw, unit-ambiguous — exactly like production + ruleID string + pid int +} + +var boundRE = regexp.MustCompile(`>= (\d+) ORDER`) + +func newFakeCH(t *testing.T) *fakeCH { + f := &fakeCH{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + m := boundRE.FindStringSubmatch(q) + if m == nil { + t.Errorf("query without >= bound: %q", q) + w.WriteHeader(400) + return + } + bound, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + t.Errorf("unparseable bound in query %q: %v", q, err) + w.WriteHeader(400) + return + } + f.mu.Lock() + var out []fakeRow + for _, row := range f.rows { + if normalizeEventTimeNanos(row.eventTime) >= bound { + out = append(out, row) + } + } + f.mu.Unlock() + sort.Slice(out, func(i, j int) bool { + return normalizeEventTimeNanos(out[i].eventTime) < normalizeEventTimeNanos(out[j].eventTime) + }) + for _, row := range out { + fmt.Fprintf(w, + `{"RuleID":%q,"RuntimeK8sDetails":"{\"podName\":\"p-1\",\"podNamespace\":\"ns\"}","RuntimeProcessDetails":"{\"processTree\":{\"pid\":%d,\"comm\":\"c\"}}","event_time":"%d","hostname":"node-1"}`+"\n", + row.ruleID, row.pid, row.eventTime) + } + })) + return f +} + +func (f *fakeCH) add(r fakeRow) { + f.mu.Lock() + f.rows = append(f.rows, r) + f.mu.Unlock() +} + +func (f *fakeCH) close() { f.srv.Close() } + +// testBase is a fixed "now" for deterministic clamp behavior: +// 2026-05-29T… ≈ 1.7805e9 seconds. +const testBase = uint64(1_780_500_000) + +func fixedNow() time.Time { return time.Unix(int64(testBase), 0) } + +// newLookbackTrigger builds a trigger against the fake server with the +// #97 config (300s lookback) and a pinned wall clock. +func newLookbackTrigger(t *testing.T, f *fakeCH, hostname string, lookback time.Duration) *ClickHouseHTTP { + t.Helper() + tr, err := New(Config{ + Endpoint: f.srv.URL, + Hostname: hostname, + PollInterval: 20 * time.Millisecond, + Lookback: lookback, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + tr.now = fixedNow // deterministic poison clamp + return tr +} + +// TestTrigger_LookbackCapturesLateArrivalExactlyOnce — T2: a row that +// lands BELOW the watermark but inside the lookback window is processed +// exactly once (no drop, no duplicate over many re-polls), and a row +// below watermark-lookback stays dropped (the documented bound). Also +// asserts ae_trigger_below_watermark_total increments (T3). +func TestTrigger_LookbackCapturesLateArrivalExactlyOnce(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) // head row → watermark = testBase + + belowBefore := testutil.ToFloat64(metricBelowWatermark) + + tr := newLookbackTrigger(t, f, "node-lb", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + // Wait for the head row so the watermark is at testBase. + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Late arrival 60s below the watermark (inside the 300s window) and + // one 400s below (outside the window). + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + f.add(fakeRow{eventTime: testBase - 400, ruleID: "R3", pid: 333}) + + got := map[uint64]int{} + deadline := time.Now().Add(400 * time.Millisecond) // ~20 re-polls of the same window + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 1 { + t.Errorf("late-arrival row emitted %d times, want exactly 1 (T2)", got[222]) + } + if got[333] != 0 { + t.Errorf("row below watermark-lookback emitted %d times, want 0 (documented bound)", got[333]) + } + if got[111] != 0 { + t.Errorf("head row re-emitted %d times after initial delivery (window dedup failed)", got[111]) + } + if delta := testutil.ToFloat64(metricBelowWatermark) - belowBefore; delta < 1 { + t.Errorf("ae_trigger_below_watermark_total delta = %v, want >= 1", delta) + } +} + +// TestTrigger_PoisonRowDoesNotHalt — T1 (the F8 non-halt guarantee): a +// row carrying the real E8 poison timestamp (1.78e18-style far-future +// vs the pinned clock) is clamp-rejected from advancing the watermark, +// the reject metric increments, the watermark gauge stays wall-clock- +// bounded, and SUBSEQUENT seconds rows are still processed — no manual +// watermark reset needed. +func TestTrigger_PoisonRowDoesNotHalt(t *testing.T) { + // The exact leftover value from loadtest E8's poisoned watermark. + // Normalized it stays 1.781559e18 ns ≈ 12 days past the pinned + // clock (1.7805e9 s) — beyond the 1h MaxSkew. + const poisonET = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase - 10, ruleID: "R1", pid: 111}) + + rejBefore := testutil.ToFloat64(metricEventTimeRejected) + + tr := newLookbackTrigger(t, f, "node-poison", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Inject the poison row; it is emitted once (real anomaly, mangled + // timestamp) but must not advance the cursor. + f.add(fakeRow{eventTime: poisonET, ruleID: "RPOISON", pid: 666}) + select { + case ev := <-ch: + if ev.Target.PID != 666 { + t.Fatalf("expected poison row emission, got PID %d", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("poison row was dropped entirely; want emitted-once-without-advance") + } + if delta := testutil.ToFloat64(metricEventTimeRejected) - rejBefore; delta < 1 { + t.Errorf("ae_trigger_event_time_rejected_total delta = %v, want >= 1", delta) + } + + // THE F8 guarantee: a fresh seconds row AFTER the poison must flow. + // Under the old strict HWM the cursor sat at 1.78e18 and this row + // was below it forever (25/25 ticks at n_anomalies=0 in E8). + f.add(fakeRow{eventTime: testBase + 5, ruleID: "R2", pid: 222}) + var got222 int + deadline := time.Now().Add(600 * time.Millisecond) + for time.Now().Before(deadline) && got222 == 0 { + select { + case ev := <-ch: + if ev.Target.PID == 222 { + got222++ + } + case <-time.After(20 * time.Millisecond): + } + } + if got222 != 1 { + t.Fatalf("post-poison seconds row emitted %d times, want 1 (T1 non-halt)", got222) + } + + // Watermark gauge stays wall-clock-bounded: it advanced to the real + // row (testBase+5 s), NOT to the poison value. + wantWM := float64(normalizeEventTimeNanos(testBase + 5)) + if got := testutil.ToFloat64(metricWatermarkNS.WithLabelValues("kubescape_logs", "node-poison")); got != wantWM { + t.Errorf("ae_trigger_watermark_ns = %v, want %v (wall-clock-bounded, not poison)", got, wantWM) + } +} + +// TestTrigger_PoisonPersistedWatermarkSelfRecovers — the E8 recovery +// scenario without the manual ALTER TABLE … DELETE: a pre-fix deployment +// left a far-future watermark behind; on start the trigger clamps it to +// wall-clock and fresh rows flow again. +func TestTrigger_PoisonPersistedWatermarkSelfRecovers(t *testing.T) { + const poisonWM = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-recover", 300*time.Second) + tr.cfg.InitialWatermark = poisonWM // simulates the poisoned persisted cursor + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("recovered event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("fresh row not delivered — poisoned persisted watermark was not clamped (still halted)") + } +} + +// TestTrigger_LookbackZeroIsStrictHWM — T4: LOOKBACK=0 preserves the +// legacy strict high-water-mark exactly — the poll bound IS the +// watermark (no window subtraction) and a below-watermark row stays +// dropped. (The monotonic happy path itself is pinned by the existing +// clickhouse_test.go suite, which runs with the zero-value Lookback.) +func TestTrigger_LookbackZeroIsStrictHWM(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-strict", 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // A late arrival below the watermark: with strict HWM the SELECT + // bound equals the watermark, so it is never fetched again → dropped. + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + got := map[uint64]int{} + deadline := time.Now().Add(300 * time.Millisecond) + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 0 { + t.Errorf("strict mode emitted a below-watermark row %d times; want 0 (legacy behavior)", got[222]) + } + if got[111] != 0 { + t.Errorf("strict mode re-emitted the boundary row %d times; want 0", got[111]) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/metrics.go b/src/vizier/services/adaptive_export/internal/trigger/metrics.go new file mode 100644 index 00000000000..4fc9dee9eda --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Watermark observability (#97 / F8 / AE-9). Registered on the DEFAULT +// prometheus registry via promauto — the same pattern the rest of pixie +// uses (e.g. query_broker's queryExec* summaries) — and served by the +// shared services/metrics /metrics handler wired up in cmd/main.go. +// Before these existed a watermark halt was completely invisible: writes +// stopped, no error, no signal (loadtest E8). +var ( + // metricWatermarkNS tracks the trigger's current cursor in + // normalized unix NANOS, per (table, hostname). A flat gauge while + // kubescape rows keep arriving is the F8 silent-halt signature — + // alert on it. + metricWatermarkNS = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ae_trigger_watermark_ns", + Help: "Current trigger high-water-mark cursor in normalized unix nanoseconds, per (table, hostname).", + }, []string{"table", "hostname"}) + + // metricBelowWatermark counts rows processed with a normalized + // event_time BELOW the poll-start watermark — i.e. out-of-order / + // clock-skewed / restart-buried rows the legacy strict HWM silently + // dropped and the bounded lookback now captures. + metricBelowWatermark = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_below_watermark_total", + Help: "Rows seen with event_time below the prior watermark that the bounded lookback captured (strict HWM would have dropped them).", + }) + + // metricEventTimeRejected counts poison clamps: rows whose + // normalized event_time was implausibly far in the future + // (> now + MaxSkew) and were therefore barred from advancing the + // watermark. + metricEventTimeRejected = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_event_time_rejected_total", + Help: "Rows whose normalized event_time exceeded now+max-skew and were rejected from advancing the watermark (poison clamp).", + }) +)