From 3b6a9ef4316e8c8f961b8ca96f70ff34008a5e7b Mon Sep 17 00:00:00 2001 From: George Kalisse <20505232+george-kalisse-sada@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:01:56 -0400 Subject: [PATCH] Variant Configs & Sweep Tools --- .gitignore | 2 + .../config/agentic_benchmark_config.yaml | 150 ++++ .../k8s_agents/config/variants/baseline.yaml | 346 +++++++++ .../k8s_agents/config/variants/c4a_arm.yaml | 399 ++++++++++ .../k8s_agents/config/variants/c4d_amd.yaml | 347 +++++++++ .../config/variants/hyperdisk_200gb.yaml | 348 +++++++++ .../config/variants/kubelet_pulls.yaml | 354 +++++++++ .../k8s_agents/config/variants/larger_vm.yaml | 347 +++++++++ .../config/variants/multi_node.yaml | 327 ++++++++ .../config/variants/overlay_none.yaml | 347 +++++++++ .../config/variants/sched_tuning.yaml | 347 +++++++++ .../data/k8s_agents/config/variants/swap.yaml | 355 +++++++++ .../data/k8s_agents/config/variants/thp.yaml | 356 +++++++++ .../tuning/kubelet_parallel_pulls.yaml | 9 + .../tuning/sched_tuner_daemonset.yaml.j2 | 49 ++ .../config/variants/tuning/swap_config.yaml | 9 + .../config/variants/tuning/thp_config.yaml | 7 + .../variants/variant_reference_guide.md | 305 ++++++++ perfkitbenchmarker/scripts/agentic/analyze.py | 718 ++++++++++++++++++ perfkitbenchmarker/scripts/agentic/sweep.py | 565 ++++++++++++++ 20 files changed, 5687 insertions(+) create mode 100644 perfkitbenchmarker/data/k8s_agents/config/agentic_benchmark_config.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/baseline.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/c4a_arm.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/c4d_amd.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/hyperdisk_200gb.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/kubelet_pulls.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/larger_vm.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/multi_node.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/overlay_none.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/sched_tuning.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/swap.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/thp.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/tuning/sched_tuner_daemonset.yaml.j2 create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml create mode 100644 perfkitbenchmarker/data/k8s_agents/config/variants/variant_reference_guide.md create mode 100644 perfkitbenchmarker/scripts/agentic/analyze.py create mode 100644 perfkitbenchmarker/scripts/agentic/sweep.py diff --git a/.gitignore b/.gitignore index 1e1c6fe077..6f0c9cb603 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ /.idea /*git_ignore* .DS_Store +.adk +tmp/ diff --git a/perfkitbenchmarker/data/k8s_agents/config/agentic_benchmark_config.yaml b/perfkitbenchmarker/data/k8s_agents/config/agentic_benchmark_config.yaml new file mode 100644 index 0000000000..74e16578c0 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/agentic_benchmark_config.yaml @@ -0,0 +1,150 @@ +# Agentic Benchmark Configuration for GKE +# Used with: --benchmark_config_file=perfkitbenchmarker/data/k8s_agents/config/agentic_benchmark_config.yaml +# +# User/environment-specific flags that MUST be passed on CLI: +# --project= +# --owner= +# --gce_network_name=-agentic-vpc +# --gke_additional_flags="--workload-pool=.svc.id.goog,--subnetwork=-agentic-subnet,--enable-master-authorized-networks,--master-authorized-networks=$(curl -s ifconfig.me)/32" +# +# Per-run flags: +# --run_stage=provision|prepare|run,cleanup|teardown +# --run_uri= +# --temp_dir= +# +# Benchmark-specific sweep parameters (vary per run): +# --k8s_python_density_concurrent_sandbox_count=N +# --k8s_snapshot_preload_mb=N +# etc. + +# =========================================================================== +# Shared configuration (defined once, referenced by all benchmarks via YAML +# anchors). PKB ignores top-level keys that don't match a benchmark name. +# =========================================================================== + +_shared_flags: &shared_flags + # --- Cluster creation flags --- + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-private-nodes" + - "--enable-ip-alias" + - "--master-ipv4-cidr=172.16.0.0/28" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.36.0-gke.3070003" + gke_enable_shielded_nodes: false + gce_subnet_region: "us-central1" + + # --- Agentic workload flags --- + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + +_shared_cluster: &shared_cluster + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-central1-a + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-central1-a + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +_shared_registry: &shared_registry + cloud: GCP + spec: + GCP: + zone: us-central1-a + + +_shared_container_specs: &shared_container_specs + adk_agent: + image: agentic/adk-agent + +# =========================================================================== +# Benchmark definitions (each references the shared anchors above) +# =========================================================================== + +k8s_python_density: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_chromium_density: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_payload: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_qps: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_snapshot: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_warmpool: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster + +k8s_deletion: + flags: + <<: *shared_flags + container_registry: + <<: *shared_registry + container_specs: + <<: *shared_container_specs + container_cluster: + <<: *shared_cluster diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/baseline.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/baseline.yaml new file mode 100644 index 0000000000..dcb093d3ee --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/baseline.yaml @@ -0,0 +1,346 @@ +# UC-B Baseline: c4-standard-8, gVisor, all defaults +# No tuning knobs applied. This is the reference measurement. + +k8s_python_density: + description: > + UC-B Baseline: Python sandbox density on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Baseline: Chromium Browser Density on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Baseline: Payload Transfer on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Baseline: QPS Saturation on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Baseline: Warmpool Scale-Up on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Baseline: Deletion & Cleanup on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Baseline: Pod Snapshot saturation on c4-standard-8 with default settings. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "baseline" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/c4a_arm.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/c4a_arm.yaml new file mode 100644 index 0000000000..9c3387af47 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/c4a_arm.yaml @@ -0,0 +1,399 @@ +# UC-B Variant 9: C4A (ARM) +# Knob: c4a-standard-8 (Arm Neoverse) +# Expected: Different gVisor syscall overhead on ARM ISA +# PREREQUISITES: +# 1. Run gke_prerequisites.py with --target_arch=arm64 +# 2. Uses --node-architecture-taint-behavior=none on BOTH node pools +# to prevent the automatic kubernetes.io/arch=arm64:NoSchedule taint +# that blocks amd64-default pods from scheduling. +# +# WHY on both pools: +# - Default pool: ADK agent, router, agent-sandbox-controller need to schedule here +# - Sandbox pool: sandbox pods need to schedule here +# Without removing the taint from both, pods have nowhere to go. + +k8s_python_density: + description: > + UC-B Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-9: C4A (ARM Neoverse) c4a-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--node-architecture-taint-behavior=none" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--node-architecture-taint-behavior=none" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4a-arm" + container_remote_build_config: "perfkitbenchmarker/data/docker/agentic/adk-agent/cloudbuild-arm64.yaml" + target_arch: "arm64" + container_cluster_architecture: + - "linux/arm64" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4a-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/c4d_amd.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/c4d_amd.yaml new file mode 100644 index 0000000000..b7db98ae85 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/c4d_amd.yaml @@ -0,0 +1,347 @@ +# UC-B Variant 8: C4D (AMD) +# Knob: c4d-standard-8 (AMD EPYC) +# Expected: Different gVisor overhead on AMD microarchitecture + +k8s_python_density: + description: > + UC-B Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-8: C4D (AMD EPYC) c4d-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4d-amd" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4d-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/hyperdisk_200gb.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/hyperdisk_200gb.yaml new file mode 100644 index 0000000000..eda7ea9dd3 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/hyperdisk_200gb.yaml @@ -0,0 +1,348 @@ +# UC-B Variant 4: Larger hyperdisk-balanced (200GB) +# Knob: 200GB hyperdisk-balanced boot disk (2x default) +# Expected: More IOPS/throughput for pod creation and image layer extraction +# Note: C4 machines only support hyperdisk-balanced, NOT pd-ssd + +k8s_python_density: + description: > + UC-B Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-4: hyperdisk-balanced 200GB boot disk on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "hyperdisk-200gb" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 200 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/kubelet_pulls.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/kubelet_pulls.yaml new file mode 100644 index 0000000000..7f4e7e413d --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/kubelet_pulls.yaml @@ -0,0 +1,354 @@ +# UC-B Variant 1: Kubelet image pull + GC tuning +# Knobs: maxParallelImagePulls=5, imageGc thresholds lowered +# Expected: Faster warm pool provisioning via parallel pulls + aggressive GC + +k8s_python_density: + description: > + UC-B Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-1: Kubelet parallel image pulls + image GC tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "kubelet-pulls" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/larger_vm.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/larger_vm.yaml new file mode 100644 index 0000000000..a6071c80bb --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/larger_vm.yaml @@ -0,0 +1,347 @@ +# UC-B Variant 3: Larger VM (c4-standard-16) +# Knob: 16 vCPUs instead of 8 +# Expected: Higher density ceiling before CEL degrades + +k8s_python_density: + description: > + UC-B Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-3: c4-standard-16 (2x CPU headroom). + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "c4-standard-16" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-16 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/multi_node.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/multi_node.yaml new file mode 100644 index 0000000000..d186a6a3a3 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/multi_node.yaml @@ -0,0 +1,327 @@ +# UC-B Variant 10: Multi-node sandbox pool (2 nodes) +# Knob: vm_count: 2 on sandbox node pool (default: 1) +# Expected: Higher density ceiling, faster warmpool provisioning, +# better scheduling throughput across 2 nodes + +k8s_python_density: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + Var-10: 2-node sandbox pool (c4-standard-8 x2) for multi-node scaling. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + container_cluster_version: "1.35.5-gke.1057002" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "multi-node-2x" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 2 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/overlay_none.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/overlay_none.yaml new file mode 100644 index 0000000000..aee0e2a129 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/overlay_none.yaml @@ -0,0 +1,347 @@ +# UC-B Variant 2: gVisor overlay2=none +# Knob: dev.gvisor.spec.overlay2=none (applied via kubectl patch after Prepare) +# Expected: Possible improvement in import CEL (fewer FS layers) + +k8s_python_density: + description: > + UC-B Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-2: gVisor overlay2=none on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "overlay-none" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/sched_tuning.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/sched_tuning.yaml new file mode 100644 index 0000000000..7562222f60 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/sched_tuning.yaml @@ -0,0 +1,347 @@ +# UC-B Variant 6: Kernel scheduler tuning +# Knobs: sched_migration_cost_ns, sched_min_granularity_ns (applied via DaemonSet) +# Expected: Reduced context switching at high density + +k8s_python_density: + description: > + UC-B Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-6: Kernel scheduler tuning on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "sched-tuning" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/swap.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/swap.yaml new file mode 100644 index 0000000000..dc6ac6131c --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/swap.yaml @@ -0,0 +1,355 @@ +# UC-B Variant 7: Swap enabled +# Knobs: Node swap + swappiness=15 +# Expected: Higher density ceiling at cost of tail latency +# Requires: GKE >= 1.34.1-gke.1341000, Burstable QoS pods only + +k8s_python_density: + description: > + UC-B Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-7: Node swap enabled on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "swap" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/thp.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/thp.yaml new file mode 100644 index 0000000000..601d37a111 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/thp.yaml @@ -0,0 +1,356 @@ +# UC-B Variant 5: Transparent Hugepages (THP) enabled system-wide +# Knob: THP=always + defrag=defer +# Expected: Reduced TLB misses for Python memory-intensive workloads +# Risk: Higher memory usage, possible compaction latency +# Note: Original Kata variant removed - GKE --sandbox only supports gvisor. + +k8s_python_density: + description: > + UC-B Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_chromium_density: + description: > + UC-C Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_payload: + description: > + UC-D Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_qps: + description: > + UC-F Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_warmpool: + description: > + UC-E Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_deletion: + description: > + UC-G Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor + +k8s_snapshot: + description: > + UC-A Var-5: Transparent Hugepages (THP=always) on c4-standard-8. + flags: + gke_additional_flags: + - "--enable-pod-snapshots" + - "--enable-dataplane-v2" + - "--enable-ip-alias" + - "--release-channel=None" + gke_additional_nodepool_flags: + - "--max-pods-per-node=250" + - "--system-config-from-file=perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml" + - "--no-enable-autoupgrade" + - "--no-enable-autorepair" + container_cluster_version: "1.36.0-gke.3070003" + gce_subnet_region: "us-east1" + k8s_agentic_namespace: "agentic" + agent_sandbox_version: "v0.4.6" + k8s_agentic_gvisor: true + k8s_agentic_agent_api_url: "http://localhost:8080" + k8s_agentic_benchmark_note: "thp-always" + container_registry: + cloud: GCP + spec: + GCP: + zone: us-east1-c + container_specs: + adk_agent: + image: agentic/adk-agent + container_cluster: + cloud: GCP + type: Kubernetes + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 50 + nodepools: + sandbox: + vm_count: 1 + vm_spec: + GCP: + machine_type: c4-standard-8 + zone: us-east1-c + boot_disk_type: hyperdisk-balanced + boot_disk_size: 100 + sandbox_config: + type: gvisor diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml new file mode 100644 index 0000000000..820876341e --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/kubelet_parallel_pulls.yaml @@ -0,0 +1,9 @@ +# Kubelet config for parallel image pulls + image GC tuning +# Applied via --system-config-from-file on node pool creation +# +# GKE-supported kubelet fields only. +# serializeImagePulls, registryPullQPS, registryBurst are NOT exposed by GKE. +kubeletConfig: + maxParallelImagePulls: 5 + imageGcHighThresholdPercent: 70 + imageGcLowThresholdPercent: 50 diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/sched_tuner_daemonset.yaml.j2 b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/sched_tuner_daemonset.yaml.j2 new file mode 100644 index 0000000000..12c5864fd7 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/sched_tuner_daemonset.yaml.j2 @@ -0,0 +1,49 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sched-tuner + namespace: {{ ns }} +spec: + selector: + matchLabels: + app: sched-tuner + template: + metadata: + labels: + app: sched-tuner + spec: + nodeSelector: + pkb_nodepool: sandbox + tolerations: + - key: "sandbox.gke.io/runtime" + operator: "Equal" + value: "gvisor" + effect: "NoSchedule" + hostPID: true + initContainers: + - name: sysctl-tuner + image: busybox:1.36 + command: ["sh", "-c"] + args: + - | + echo 2212926 > /proc/sys/kernel/sched_migration_cost_ns + echo 4656091 > /proc/sys/kernel/sched_min_granularity_ns + echo 26403982 > /proc/sys/kernel/sched_wakeup_granularity_ns + echo "Scheduler tunables applied." + securityContext: + privileged: true + volumeMounts: + - name: proc + mountPath: /proc + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + resources: + requests: + cpu: "1m" + memory: "4Mi" + volumes: + - name: proc + hostPath: + path: /proc diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml new file mode 100644 index 0000000000..0b569f5d90 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/swap_config.yaml @@ -0,0 +1,9 @@ +# Node swap configuration for UC-B Variant 7 +# Applied via --system-config-from-file (NOT --linux-config) +# Requires GKE >= 1.34.1-gke.1341000 +# Only Burstable QoS pods can use swap +linuxConfig: + swapConfig: + enabled: true + sysctl: + vm.swappiness: '15' diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml new file mode 100644 index 0000000000..c60bb171ba --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/tuning/thp_config.yaml @@ -0,0 +1,7 @@ +# Transparent Hugepages configuration +# THP=always can reduce TLB misses for memory-intensive workloads +# Default is 'madvise' (only MADV_HUGEPAGE regions) +# Risk: may increase memory usage and cause latency spikes during compaction +linuxConfig: + transparentHugepageEnabled: TRANSPARENT_HUGEPAGE_ENABLED_ALWAYS + transparentHugepageDefrag: TRANSPARENT_HUGEPAGE_DEFRAG_DEFER diff --git a/perfkitbenchmarker/data/k8s_agents/config/variants/variant_reference_guide.md b/perfkitbenchmarker/data/k8s_agents/config/variants/variant_reference_guide.md new file mode 100644 index 0000000000..20d9b649e3 --- /dev/null +++ b/perfkitbenchmarker/data/k8s_agents/config/variants/variant_reference_guide.md @@ -0,0 +1,305 @@ +# Optimization Variant Reference Guide + +## Baseline Configuration + +All variants are compared against this reference: + +| Parameter | Baseline Value | +|-----------|---------------| +| Machine type | c4-standard-8 (4 cores / 8 vCPUs, 30 GB RAM) | +| Boot disk | hyperdisk-balanced, 100 GB (sandbox pool) | +| Runtime | gVisor (GKE Sandbox) | +| GKE version | 1.35.5-gke.1057002 | +| Platform | xemu (Google's hardware-accelerated, default) | +| Filesystem | lisafs + directfs + exclusive (all defaults) | +| Overlay | root:self (disk-backed, host page cache) | +| THP | madvise (default) | +| Swap | disabled | +| Kubelet | default settings | +| Scheduler | default kernel tunables | +| Sandbox nodes | 1 | + +--- + +## Variant Descriptions + +### var1 — Kubelet Parallel Image Pulls + Image GC + +**What changes vs baseline:** +- `maxParallelImagePulls: 5` (default: 1 — serial pulls) +- `imageGcHighThresholdPercent: 70` (default: 85) +- `imageGcLowThresholdPercent: 50` (default: 80) +- Applied via `--system-config-from-file` on sandbox node pool creation + +**What it tests:** Whether parallelizing container image pulls and aggressively garbage-collecting unused images improves pod creation throughput. The hypothesis is that serial image pulls bottleneck warm pool provisioning when many pods start simultaneously. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | None | Measures in-sandbox execution, not pod creation | +| k8s_chromium_density | None | Same — measures browser interaction after pod is running | +| k8s_payload | None | Measures data transfer, not pod creation | +| k8s_qps | Indirect | Faster image pulls -> faster warm pool refill -> sustained QPS | +| **k8s_warmpool** | **Direct** | **Parallel pulls directly accelerate bulk pod provisioning** | +| k8s_deletion | None | Image GC may help disk cleanup, but deletion is IPAM-bound | +| k8s_snapshot | Indirect | Faster pod creation after restore, but restore is CRIU-bound | + +--- + +### var2 — gVisor overlay2=none + +**What changes vs baseline:** +- `dev.gvisor.spec.overlay2: none` (default: `root:self`) +- Applied via `kubectl patch sandboxtemplate` post-prepare +- Removes the overlay filesystem layer — sandbox writes go directly to the container's rootfs + +**What it tests:** Whether removing the overlay2 filesystem layer reduces I/O overhead for file-heavy operations. The hypothesis is that overlay adds latency to every file open/stat/read through extra indirection layers. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | Harmful | Import CEL worsens — loses page cache sharing | +| k8s_chromium_density | Likely harmful | Chrome loads many shared libs — same cache loss | +| k8s_payload | Possible | Stdout write path may differ, but payload is memory to stdout | +| k8s_qps | None | QPS measures scheduling, not filesystem | +| k8s_warmpool | None | Pod creation doesn't depend on overlay mode | +| k8s_deletion | None | Deletion is pod lifecycle, not filesystem | +| k8s_snapshot | Unknown | Snapshot captures filesystem state — overlay=none may reduce snapshot size but slow restore | + +--- + +### var3 — c4-standard-16 (2x CPU) + +**What changes vs baseline:** +- Machine type: `c4-standard-16` (8 cores / 16 vCPUs, 60 GB RAM) +- Both default pool and sandbox pool upgraded +- 2x CPU, 2x RAM vs baseline + +**What it tests:** Whether doubling available compute resources directly reduces execution latency and raises the density ceiling. This is the "throw hardware at it" variant — establishes whether the bottleneck is CPU-bound. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| **k8s_python_density** | **Strong improvement** | **CPU-bound workload directly benefits from 2x cores** | +| **k8s_chromium_density** | **Strong improvement** | **Chrome rendering + JS execution are CPU-bound** | +| **k8s_payload** | **Moderate improvement** | **base64 serialization is CPU-bound** | +| **k8s_qps** | **Moderate improvement** | **More CPU headroom for concurrent sandbox lifecycle ops** | +| **k8s_warmpool** | **Moderate improvement** | **Faster container init, more scheduling headroom** | +| k8s_deletion | Minor | Deletion is control-plane/IPAM bound, not node CPU | +| **k8s_snapshot** | **Moderate improvement** | **CRIU restore + gVisor init are CPU-intensive** | + +--- + +### var4 — Hyperdisk-balanced 200 GB + +**What changes vs baseline:** +- Sandbox pool boot disk: `hyperdisk-balanced 200 GB` (default: 100 GB) +- Larger disk = more baseline IOPS (hyperdisk scales with size) +- No change to machine type + +**What it tests:** Whether increased disk I/O throughput improves pod creation speed and file-heavy operations. Hyperdisk-balanced IOPS scale with provisioned size, so 200 GB provides ~2x the IOPS of 100 GB. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | None | CPU-bound workload, disk irrelevant after pod starts | +| k8s_chromium_density | Minor | Chrome writes temp files, but execution is CPU-bound | +| k8s_payload | None | Payload goes through stdout, not disk | +| k8s_qps | Minor | Pod creation involves image layer extraction (disk I/O) | +| **k8s_warmpool** | **Moderate** | **Bulk pod creation extracts image layers — disk IOPS matters** | +| k8s_deletion | Minor | Pod cleanup involves filesystem teardown | +| **k8s_snapshot** | **Direct** | **Snapshot write/restore is GCS-bound but local staging uses disk** | + +--- + +### var5 — Transparent Hugepages (THP=always) + +**What changes vs baseline:** +- `transparentHugepageEnabled: ALWAYS` (default: `madvise`) +- `transparentHugepageDefrag: DEFER` (default: varies) +- Applied via `--system-config-from-file` on sandbox node pool + +**What it tests:** Whether using 2 MB huge pages instead of 4 KB pages reduces TLB (Translation Lookaside Buffer) misses for memory-intensive workloads. The hypothesis is that Python's memory allocator and gVisor's sentry benefit from fewer page table entries. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | Harmful at density | Compaction stalls worsen with more pods | +| k8s_chromium_density | Likely harmful | Chrome's multi-process model amplifies compaction | +| k8s_payload | Unknown | Large memory allocations might benefit, but compaction risk | +| k8s_qps | Harmful | Compaction during pod creation adds latency | +| k8s_warmpool | Harmful | Bulk pod creation triggers heavy compaction | +| k8s_deletion | None | Deletion doesn't allocate memory | +| k8s_snapshot | Unknown | Snapshot restore allocates large memory blocks — could help or hurt | + +--- + +### var6 — Kernel Scheduler Tuning + +**What changes vs baseline:** +- `sched_migration_cost_ns: 2212926` (higher = less eager migration between cores) +- `sched_min_granularity_ns: 4656091` (larger time slices) +- `sched_wakeup_granularity_ns: 26403982` (less preemption on wakeup) +- Applied via privileged DaemonSet writing to `/proc/sys/kernel/sched_*` + +**What it tests:** Whether reducing context-switch frequency improves throughput for many concurrent containers. The hypothesis is that at high density, the default scheduler switches between gVisor sentry processes too aggressively, wasting CPU on context switches. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | Harmful | Larger time slices increase scheduling unfairness at high density | +| k8s_chromium_density | Likely harmful | Chrome's multi-process model needs responsive scheduling | +| k8s_payload | Neutral | Single-threaded transfer, scheduling less relevant | +| k8s_qps | Harmful | Slower preemption delays sandbox claim processing | +| k8s_warmpool | Unknown | Could help or hurt bulk pod init depending on contention pattern | +| k8s_deletion | None | Deletion is API-server/kubelet bound, not scheduler | +| k8s_snapshot | Unknown | CRIU restore is single-threaded, may not be affected | + +--- + +### var7 — Swap Enabled + +**What changes vs baseline:** +- `swapConfig.enabled: true` +- `vm.swappiness: 15` (low — only swap under memory pressure) +- Applied via `--system-config-from-file` +- Only Burstable QoS pods can use swap (Guaranteed QoS pods are excluded) + +**What it tests:** Whether enabling swap allows higher pod density by offloading cold memory pages to disk. The hypothesis is that at very high density, some sandbox memory (e.g., imported-but-idle Python modules) can be swapped out to make room for more pods. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | Neutral (unless memory-bound) | Only helps if density exceeds RAM capacity | +| k8s_chromium_density | **Possible** | **Chrome is memory-hungry (~100-200 MB/instance) — swap could extend density ceiling** | +| k8s_payload | None | Payload size is small relative to RAM | +| k8s_qps | None | QPS is scheduling-bound, not memory-bound | +| k8s_warmpool | Minor | More pods can fit if memory is the constraint | +| k8s_deletion | None | Deletion doesn't allocate memory | +| k8s_snapshot | Minor | Swap could allow larger preload sizes before OOM | + +--- + +### var8 — C4D AMD (c4d-standard-8) + +**What changes vs baseline:** +- Machine type: `c4d-standard-8` (AMD EPYC, same vCPU/RAM as baseline) +- Different CPU microarchitecture (AMD vs Intel) +- Same core count, same memory + +**What it tests:** Whether AMD's CPU microarchitecture handles gVisor's syscall interception more efficiently than Intel's. gVisor's sentry traps every syscall via hardware virtualization extensions — AMD and Intel implement these differently (AMD-V vs VT-x). + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| **k8s_python_density** | **Strong improvement** | **Syscall interception is the core overhead — AMD handles it better** | +| **k8s_chromium_density** | **Strong improvement** | **Chrome makes many syscalls — same benefit** | +| **k8s_payload** | **Moderate improvement** | **stdout write is a syscall through gVisor** | +| **k8s_qps** | **Moderate improvement** | **Sandbox lifecycle involves many syscalls** | +| **k8s_warmpool** | **Minor** | **Pod creation is control-plane bound, not syscall bound** | +| k8s_deletion | Minor | Deletion is API-server bound | +| **k8s_snapshot** | **Moderate improvement** | **CRIU restore + gVisor init involve heavy syscall activity** | + +--- + +### var9 — C4A ARM (c4a-standard-8) + +**What changes vs baseline:** +- Machine type: `c4a-standard-8` (ARM Neoverse, same vCPU/RAM as baseline) +- Entirely different ISA (ARM64 vs x86_64) +- Requires ARM container images (cross-compiled via QEMU) +- `--node-architecture-taint-behavior=none` on both pools + +**What it tests:** Whether ARM's instruction set architecture provides different gVisor overhead characteristics. ARM has a different syscall ABI and virtualization model — gVisor's sentry may behave differently. + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| **k8s_python_density** | **Unknown** | **ARM syscall ABI differs — could be better or worse** | +| **k8s_chromium_density** | **Unknown** | **Chrome on ARM is less mature — may have different perf profile** | +| k8s_payload | Unknown | ARM memory bandwidth differs | +| k8s_qps | Unknown | Scheduling overhead may differ | +| k8s_warmpool | Unknown | Pod creation speed on ARM nodes | +| k8s_deletion | Unknown | Likely similar to x86 (API-server bound) | +| k8s_snapshot | Unknown | CRIU on ARM may have different characteristics | + +--- + +### var10 — Multi-Node Sandbox Pool (2x c4-standard-8) + +**What changes vs baseline:** +- Sandbox node pool: `vm_count: 2` (default: 1) +- Same machine type per node (c4-standard-8) +- Default pool stays at 1 node +- Total sandbox capacity: 2x CPU, 2x RAM, 2x disk across 2 nodes + +**What it tests:** Whether distributing sandbox pods across multiple nodes improves provisioning speed, scheduling throughput, and density ceiling. Every previous test ran on a single sandbox node — this tests whether the bottleneck is per-node resource saturation or control-plane scheduling. + +Key questions this variant answers: +- Does warmpool provisioning scale linearly with nodes? (2 nodes = 2x pods/sec?) +- Does QPS saturation point double with 2 nodes? +- Does deletion/IP reclamation improve with pods spread across nodes? +- Is the GKE scheduler or the node the bottleneck at high density? + +| Benchmark | Expected Influence | Rationale | +|-----------|-------------------|-----------| +| k8s_python_density | Moderate | Pods spread across 2 nodes — each node at ~half density, less CPU contention | +| **k8s_warmpool** | **Strong** | **Pod creation parallelized across 2 kubelets** | +| **k8s_qps** | **Strong** | **Warm pool spans 2 nodes — 2x capacity before drain** | +| k8s_payload | Minor | Payload transfer is per-sandbox, not affected by node count | +| k8s_chromium_density | Moderate | Chrome pods spread across nodes — more total RAM | +| **k8s_deletion** | **Moderate** | **IPAM reclamation distributed across 2 nodes** | +| **k8s_snapshot** | **Moderate** | **Snapshot/restore parallelized across nodes** | + +--- + +## Expansion Plan: Which Variants to Test Per Benchmark + +Based on the analysis above, here's the recommended test matrix. Only variants with a plausible positive or unknown influence are included — proven-harmful variants (var2, var5, var6) are excluded. + +### Priority Matrix + +| Benchmark | Priority Variants | Why | +|-----------|------------------|-----| +| **k8s_warmpool** | var1, var3, var4, var8, var10 | Pod creation speed: kubelet (var1), CPU (var3), disk IOPS (var4), AMD (var8), multi-node (var10) | +| **k8s_qps** | var3, var8, var10 | Scheduling throughput: CPU-bound; multi-node doubles capacity | +| **k8s_payload** | var3, var8 | Serialization is CPU-bound, syscall overhead matters | +| **k8s_chromium_density** | var3, var7, var8 | CPU (var3), memory ceiling (var7), AMD syscalls (var8) | +| **k8s_deletion** | var4, var10, baseline | Disk cleanup + IPAM — multi-node distributes pressure | +| **k8s_snapshot** | var3, var4, var8 | CRIU restore is CPU+disk, AMD may help | + +### Variants to Skip for Non-Density Benchmarks + +| Variant | Reason to Skip | +|---------|---------------| +| var2 (overlay2=none) | Harmful everywhere — loses page cache sharing | +| var5 (THP=always) | Harmful at density — compaction stalls | +| var6 (sched tuning) | Harmful — defaults are better for containers | + +### Suggested Sweep Values Per Benchmark + +| Benchmark | Sweep Parameter | Suggested Values | Rationale | +|-----------|----------------|-----------------|-----------| +| k8s_python_density | concurrent_sandbox_count | 1, 10, 50, 100, 150, 200 | Find saturation curve | +| k8s_chromium_density | concurrent_sessions | 1, 2, 4, 8, 12, 16 | Chrome is much heavier per instance | +| k8s_payload | payload_size_mb | 0.01, 0.1, 1, 5, 10, 50 | Find bandwidth saturation | +| k8s_qps | target_qps | 1, 5, 10, 20, 50, 100 | Find scheduling saturation | +| k8s_warmpool | target_replicas | 10, 50, 100, 200, 500 | Find provisioning ceiling | +| k8s_deletion | batch_size | 10, 50, 100, 200, 500 | Find cleanup bottleneck | +| k8s_snapshot | preload_mb | 10, 50, 100, 500, 1000 | Find snapshot size limit | + +### Execution Order Recommendation + +1. **k8s_warmpool** (var1, var3, var4, var8, var10) — fastest to run, no agent API needed +2. **k8s_qps** (var3, var8, var10) — quick runs, tests scheduling +3. **k8s_payload** (var3, var8) — moderate duration +4. **k8s_chromium_density** (var3, var7, var8) — needs chromium warm pool +5. **k8s_deletion** (baseline, var4, var10) — needs bulk provisioning first +6. **k8s_snapshot** (var3, var4, var8) — needs snapshot infrastructure + +### Full Test Matrix (Y = Run) + +| Variant | k8s_python_density | k8s_warmpool | k8s_qps | k8s_payload | k8s_chromium_density | k8s_deletion | k8s_snapshot | +|---------|:--:|:--:|:--:|:--:|:--:|:--:|:--:| +| baseline | Y | Y | Y | Y | Y | Y | Y | +| var1 (kubelet) | Y | Y | | | | | | +| var2 (overlay) | Y | | | | | | | +| var3 (2x CPU) | Y | Y | Y | Y | Y | | Y | +| var4 (disk) | Y | Y | | | | Y | Y | +| var5 (THP) | Y | | | | | | | +| var6 (sched) | Y | | | | | | | +| var7 (swap) | Y | | | | Y | | | +| var8 (AMD) | Y | Y | Y | Y | Y | | Y | +| var9 (ARM) | Y | | | | | | | +| var10 (multi-node) | Y | Y | Y | | | Y | Y | diff --git a/perfkitbenchmarker/scripts/agentic/analyze.py b/perfkitbenchmarker/scripts/agentic/analyze.py new file mode 100644 index 0000000000..ef5913a21b --- /dev/null +++ b/perfkitbenchmarker/scripts/agentic/analyze.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +"""Optimization Sweep Results Analyzer. + +Parses PKB NDJSON results from all variants, computes deltas vs baseline, +and generates a markdown report with tables and recommendations. + +Auto-discovers all metrics from the data -- no hardcoded metric lists. +Supports all agentic benchmarks via a benchmark registry with auto-discovery +fallback for unknown benchmarks. + +Usage: + python perfkitbenchmarker/scripts/agentic/analyze.py --benchmark k8s_python_density + python perfkitbenchmarker/scripts/agentic/analyze.py --benchmark k8s_qps --results-dir results/pkb/k8s_qps + python perfkitbenchmarker/scripts/agentic/analyze.py --benchmark k8s_payload --output-dir reports/payload + +Outputs: + reports//_report.md -- Full markdown report + reports//_raw_data.csv -- Raw data for further analysis +""" + +import argparse +import csv +import json +import os +import sys +from collections import defaultdict +from pathlib import Path + + +# ============================================================ +# Benchmark Registry +# ============================================================ + +BENCHMARK_REGISTRY = { + "k8s_python_density": { + "sweep_label": "density", + "metric_prefix": "k8s_python_density_", + "verdict_metric": "k8s_python_density_sandbox_total_cel_mean", + "description": "Python Sandbox Density (UC-B)", + }, + "k8s_chromium_density": { + "sweep_label": "density", + "metric_prefix": "k8s_chromium_density_", + "verdict_metric": "k8s_chromium_density_interaction_mean", + "description": "Chromium Browser Density (UC-C)", + }, + "k8s_payload": { + "sweep_label": "payload_size_mb", + "metric_prefix": "k8s_payload_", + "verdict_metric": "k8s_payload_sandbox_transfer_time_mean", + "description": "Payload Transfer (UC-D)", + }, + "k8s_qps": { + "sweep_label": "target_qps", + "metric_prefix": "k8s_qps_", + "verdict_metric": "k8s_qps_ttfe_mean", + "description": "QPS Saturation (UC-F)", + }, + "k8s_warmpool": { + "sweep_label": "target_replicas", + "metric_prefix": "k8s_warmpool_", + "verdict_metric": "k8s_warmpool_total_time_to_ready", + "description": "Warmpool Scale-Up (UC-E)", + }, + "k8s_deletion": { + "sweep_label": "batch_size", + "metric_prefix": "k8s_deletion_", + "verdict_metric": "k8s_deletion_total_drain_time", + "description": "Deletion & Cleanup (UC-G)", + }, + "k8s_snapshot": { + "sweep_label": "preload_mb", + "metric_prefix": "k8s_snapshot_", + "verdict_metric": "k8s_snapshot_ttfe_p50", + "description": "Pod Snapshot (UC-A)", + }, +} + + +# ============================================================ +# Variant Descriptions +# ============================================================ + +VARIANT_DESC = { + "baseline": "Baseline (c4-standard-8, gVisor defaults)", + "kubelet_pulls": "Kubelet tuning (maxParallelImagePulls=5, image GC)", + "overlay_none": "gVisor overlay2=none", + "larger_vm": "c4-standard-16 (2x CPU)", + "hyperdisk_200gb": "hyperdisk-balanced 200GB", + "thp": "Transparent Hugepages (THP=always)", + "sched_tuning": "Kernel scheduler tuning", + "swap": "Swap enabled (swappiness=15)", + "c4d_amd": "C4D AMD (c4d-standard-8)", + "c4a_arm": "C4A ARM (c4a-standard-8)", + "multi_node": "Multi-node sandbox pool (2x c4-standard-8)", + "combined": "Combined best knobs", +} + + +# ============================================================ +# Parsing +# ============================================================ + + +def parse_labels(labels_str): + """Parse PKB pipe-delimited labels into a dict.""" + result = {} + if not labels_str: + return result + for part in labels_str.split(","): + part = part.strip().strip("|") + if ":" in part: + key, _, val = part.partition(":") + result[key.strip()] = val.strip() + return result + + +def short_name(metric, prefix): + """Strip the benchmark prefix to get a short display name.""" + if metric.startswith(prefix): + return metric[len(prefix):] + return metric + + +def auto_discover_prefix(metrics): + """Auto-discover the common metric prefix from a set of metric names. + + Finds the longest common prefix ending with underscore. + """ + if not metrics: + return "" + prefix = os.path.commonprefix(list(metrics)) + last_underscore = prefix.rfind("_") + if last_underscore > 0: + return prefix[:last_underscore + 1] + return prefix + + +def auto_discover_sweep_label(results_dir): + """Auto-discover the sweep label by scanning NDJSON records. + + Looks at the labels field of the first benchmark metric found and + returns the first label key that looks like a sweep parameter + (numeric value, not 'note' or 'machine_type'). + """ + skip_labels = {"note", "machine_type", "gvisor", "namespace"} + runs_dir = Path(results_dir) / "runs" + if not runs_dir.is_dir(): + return "density" + + for variant_dir in sorted(runs_dir.iterdir()): + if not variant_dir.is_dir(): + continue + for pattern in [ + variant_dir / "perfkitbenchmarker_results.json", + ]: + if not pattern.is_file(): + continue + with open(pattern, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + labels = parse_labels(record.get("labels", "")) + for key, val in labels.items(): + if key in skip_labels: + continue + try: + float(val) + return key + except (ValueError, TypeError): + continue + return "density" # fallback + + +def load_variant_data(results_dir, metric_prefix, sweep_label): + """Load all variant NDJSON files into a structured dict. + + Groups records by (sweep_val, run_id) to distinguish separate + runs at the same sweep value; keys are suffixed with _runN. + + Returns: + dict: {variant: {sweep_key: {metric: value}}} + """ + all_data = {} + + runs_dir = Path(results_dir) / "runs" + if not runs_dir.is_dir(): + print(f" No runs/ directory found in {results_dir}") + return all_data + + for variant_dir in sorted(runs_dir.iterdir()): + if not variant_dir.is_dir(): + continue + variant_name = variant_dir.name + + results_file = variant_dir / "perfkitbenchmarker_results.json" + if not results_file.is_file(): + results_file = None + + if not results_file: + print(f" SKIP {variant_name}: no results file found") + continue + + print(f" Loading {variant_name} from {results_file}") + + # Phase 1: Group records by (sweep_val, wall_time_s) + run_groups = defaultdict(dict) # (sweep_val, wall_time_s) -> {metric: value} + line_count = 0 + metric_count = 0 + + with open(results_file, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + + line_count += 1 + metric = record.get("metric", "") + value = record.get("value") + labels = parse_labels(record.get("labels", "")) + + if not metric.startswith(metric_prefix): + continue + + sweep_val = labels.get(sweep_label) + if sweep_val is None: + continue + + try: + sweep_val = float(sweep_val) + if sweep_val == int(sweep_val): + sweep_val = int(sweep_val) + except (ValueError, TypeError): + pass + + # run_id is a unique UUID prefix per Run() invocation + rid = labels.get("run_id", "") + + group_key = (sweep_val, rid) + if metric in run_groups[group_key]: + print(f" WARNING: duplicate metric {metric} in group {group_key}") + run_groups[group_key][metric] = value + metric_count += 1 + + if not run_groups: + print(f" WARNING: no benchmark metrics found ({line_count} total lines)") + continue + + # Phase 2: Assign display keys (plain or _runN suffixed) + sweep_val_runs = defaultdict(list) + for (sv, wt), metrics in sorted(run_groups.items()): + sweep_val_runs[sv].append((wt, metrics)) + + variant_data = {} + for sv, runs in sorted(sweep_val_runs.items()): + for idx, (wt, metrics) in enumerate(runs, 1): + key = f"{sv}_run{idx}" + variant_data[key] = metrics + + all_data[variant_name] = variant_data + sweep_keys = sorted(variant_data.keys(), key=lambda k: (str(k), k)) + metrics_found = set() + for d in variant_data.values(): + metrics_found.update(d.keys()) + print(f" {metric_count} samples, {len(metrics_found)} unique metrics, " + f"{len(run_groups)} runs, keys: {sweep_keys}") + + return all_data + +def discover_metrics(all_data): + """Auto-discover all metrics across all variants and sweep values.""" + all_metrics = set() + for variant_data in all_data.values(): + for sweep_data in variant_data.values(): + all_metrics.update(sweep_data.keys()) + return sorted(all_metrics) + + +# ============================================================ +# Analysis +# ============================================================ + + +def compute_deltas(all_data, baseline_name="baseline"): + """Compute percentage deltas vs baseline for each variant/sweep_value/metric.""" + baseline = all_data.get(baseline_name) + if not baseline: + print(f"WARNING: baseline '{baseline_name}' not found in data") + return {} + + deltas = {} + for variant, sweep_values in all_data.items(): + if variant == baseline_name: + continue + deltas[variant] = {} + for sweep_val, metrics in sweep_values.items(): + deltas[variant][sweep_val] = {} + baseline_metrics = baseline.get(sweep_val, {}) + for metric, value in metrics.items(): + base_val = baseline_metrics.get(metric) + if base_val is not None and base_val != 0: + delta_pct = ((value - base_val) / abs(base_val)) * 100 + else: + delta_pct = None + deltas[variant][sweep_val][metric] = { + "value": value, + "baseline": base_val, + "delta_pct": delta_pct, + } + return deltas + + +def format_delta(delta_pct): + """Format delta percentage with direction indicator.""" + if delta_pct is None: + return "N/A" + sign = "+" if delta_pct > 0 else "" + if delta_pct < -5: + indicator = " faster" + elif delta_pct > 5: + indicator = " slower" + else: + indicator = " (flat)" + return f"{sign}{delta_pct:.1f}%{indicator}" + + +def format_value(value): + """Format a metric value for display.""" + if value is None: + return "\u2014" + if isinstance(value, float): + if value >= 1000: + return f"{value:.1f}" + return f"{value:.2f}" + return str(value) + + +def format_value_with_delta(value, delta_pct): + """Format a value with its delta percentage.""" + if value is None: + return "\u2014" + base = format_value(value) + if delta_pct is not None: + sign = "+" if delta_pct > 0 else "" + return f"{base} ({sign}{delta_pct:.1f}%)" + return base + + +# ============================================================ +# Report Generation +# ============================================================ + + +def generate_report(benchmark, description, sweep_label, metric_prefix, + verdict_metric, all_data, deltas, all_metrics, output_dir): + """Generate the full markdown report.""" + os.makedirs(output_dir, exist_ok=True) + + baseline = all_data.get("baseline", {}) + sweep_vals = sorted(set(sv for v in all_data.values() for sv in v.keys())) + variants = [v for v in sorted(all_data.keys()) if v != "baseline"] + + lines = [] + lines.append(f"# {description} Optimization \u2014 Results Report") + lines.append("") + lines.append("## Overview") + lines.append("") + lines.append(f"- **Benchmark**: {benchmark}") + lines.append(f"- **Sweep parameter**: {sweep_label}") + lines.append(f"- **Sweep values tested**: {sweep_vals}") + lines.append(f"- **Variants**: {len(all_data)} ({len(variants)} + baseline)") + lines.append(f"- **Metrics discovered**: {len(all_metrics)}") + lines.append("") + + # Variant descriptions + lines.append("## Variants Tested") + lines.append("") + lines.append("| Variant | Description |") + lines.append("|---------|-------------|") + lines.append(f"| baseline | {VARIANT_DESC.get('baseline', '')} |") + for v in variants: + lines.append(f"| {v} | {VARIANT_DESC.get(v, '')} |") + lines.append("") + + # Baseline absolute values + lines.append("## Baseline Absolute Values") + lines.append("") + + baseline_metrics = [m for m in all_metrics if any(m in baseline.get(sv, {}) for sv in sweep_vals)] + if baseline_metrics: + header = "| Metric | " + " | ".join(f"{sweep_label}={sv}" for sv in sweep_vals) + " |" + sep = "|--------|" + "|".join("------:" for _ in sweep_vals) + "|" + lines.append(header) + lines.append(sep) + for metric in baseline_metrics: + row = f"| {short_name(metric, metric_prefix)} |" + for sv in sweep_vals: + val = baseline.get(sv, {}).get(metric) + row += f" {format_value(val)} |" + lines.append(row) + lines.append("") + + # Summary comparison + lines.append("## Summary: Delta vs Baseline") + lines.append("") + lines.append("Percentage change vs baseline. Negative = faster/less (better for latency metrics).") + lines.append("Threshold: \u00b15% considered significant.") + lines.append("") + + for sv in sweep_vals: + lines.append(f"### {sweep_label} = {sv}") + lines.append("") + + active_variants = [v for v in variants if sv in all_data.get(v, {})] + if not active_variants: + lines.append("*No variant data at this sweep value.*") + lines.append("") + continue + + available_metrics = [m for m in all_metrics + if any(m in deltas.get(v, {}).get(sv, {}) for v in active_variants)] + + if not available_metrics: + lines.append("*No metrics available.*") + lines.append("") + continue + + header = "| Metric | " + " | ".join(active_variants) + " |" + sep = "|--------|" + "|".join("------:" for _ in active_variants) + "|" + lines.append(header) + lines.append(sep) + + for metric in available_metrics: + row = f"| {short_name(metric, metric_prefix)} |" + for v in active_variants: + d_info = deltas.get(v, {}).get(sv, {}).get(metric, {}) + delta = d_info.get("delta_pct") + row += f" {format_delta(delta)} |" + lines.append(row) + lines.append("") + + # Per-variant detailed analysis + lines.append("## Per-Variant Detailed Analysis") + lines.append("") + + for variant in variants: + v_data = all_data.get(variant, {}) + v_deltas = deltas.get(variant, {}) + if not v_data: + continue + + lines.append(f"### {variant}: {VARIANT_DESC.get(variant, '')}") + lines.append("") + + v_sweep_vals = sorted(v_data.keys()) + available_metrics = [m for m in all_metrics + if any(m in v_data.get(sv, {}) for sv in v_sweep_vals)] + + if not available_metrics: + lines.append("*No metrics available.*") + lines.append("") + continue + + header = "| Metric | " + " | ".join(f"{sweep_label}={sv}" for sv in v_sweep_vals) + " |" + sep = "|--------|" + "|".join("------:" for _ in v_sweep_vals) + "|" + lines.append(header) + lines.append(sep) + + for metric in available_metrics: + row = f"| {short_name(metric, metric_prefix)} |" + for sv in v_sweep_vals: + val = v_data.get(sv, {}).get(metric) + d_info = v_deltas.get(sv, {}).get(metric, {}) + delta = d_info.get("delta_pct") + row += f" {format_value_with_delta(val, delta)} |" + lines.append(row) + lines.append("") + + # Verdict + lines.append("## Verdict & Recommendations") + lines.append("") + lines.append("### Per-Variant Verdict") + lines.append("") + lines.append("| Variant | Verdict | Key Finding |") + lines.append("|---------|---------|-------------|") + + for variant in variants: + v_deltas = deltas.get(variant, {}) + if not v_deltas: + lines.append(f"| {variant} | \u2753 No data | \u2014 |") + continue + + verdict_deltas = [] + for sv, metrics in v_deltas.items(): + d_info = metrics.get(verdict_metric, {}) + if d_info.get("delta_pct") is not None: + verdict_deltas.append(d_info["delta_pct"]) + + if verdict_deltas: + avg_delta = sum(verdict_deltas) / len(verdict_deltas) + verdict_short = short_name(verdict_metric, metric_prefix) + if avg_delta < -5: + verdict = "\u2705 Improvement" + finding = f"Avg {avg_delta:.1f}% better {verdict_short}" + elif avg_delta > 5: + verdict = "\u274c Regression" + finding = f"Avg {avg_delta:.1f}% worse {verdict_short}" + else: + verdict = "\u2796 Neutral" + finding = f"Avg {avg_delta:.1f}% delta (within noise)" + else: + verdict = "\u2753 Insufficient data" + finding = f"No {short_name(verdict_metric, metric_prefix)} data" + + lines.append(f"| {variant} | {verdict} | {finding} |") + + lines.append("") + + report_path = os.path.join(output_dir, f"{benchmark}_report.md") + with open(report_path, "w") as f: + f.write("\n".join(lines)) + print(f"\nReport written to: {report_path}") + return report_path + + +def export_csv(benchmark, metric_prefix, sweep_label, all_data, deltas, + all_metrics, output_dir): + """Export raw data as CSV for further analysis.""" + os.makedirs(output_dir, exist_ok=True) + csv_path = os.path.join(output_dir, f"{benchmark}_raw_data.csv") + + rows = [] + for variant, sweep_values in sorted(all_data.items()): + for sweep_val, metrics in sorted(sweep_values.items()): + for metric in all_metrics: + value = metrics.get(metric) + if value is None: + continue + d_info = deltas.get(variant, {}).get(sweep_val, {}).get(metric, {}) + rows.append({ + "variant": variant, + "sweep_label": sweep_label, + "sweep_value": sweep_val, + "metric": metric, + "metric_short": short_name(metric, metric_prefix), + "value": value, + "baseline_value": d_info.get("baseline", ""), + "delta_pct": round(d_info["delta_pct"], 2) if d_info.get("delta_pct") is not None else "", + }) + + if rows: + fieldnames = ["variant", "sweep_label", "sweep_value", "metric", + "metric_short", "value", "baseline_value", "delta_pct"] + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"CSV written to: {csv_path} ({len(rows)} rows)") + return csv_path + + +# ============================================================ +# Main +# ============================================================ + + +def main(): + parser = argparse.ArgumentParser(description="Optimization Sweep Results Analyzer") + parser.add_argument( + "--benchmark", required=True, + help="Benchmark name (e.g. k8s_python_density, k8s_qps). " + "Used to auto-configure metric prefix, sweep label, and paths.", + ) + parser.add_argument( + "--results-dir", default=None, + help="Directory containing variant subdirectories. " + "Default: results/pkb/", + ) + parser.add_argument( + "--output-dir", default=None, + help="Output directory for report and CSV. " + "Default: reports/", + ) + parser.add_argument( + "--baseline", default="baseline", + help="Name of the baseline variant (default: baseline)", + ) + args = parser.parse_args() + + benchmark = args.benchmark + results_dir = args.results_dir or os.path.join("results", "pkb", benchmark) + output_dir = args.output_dir or os.path.join("reports", benchmark) + + # Look up benchmark in registry, fall back to auto-discovery + registry = BENCHMARK_REGISTRY.get(benchmark, {}) + description = registry.get("description", benchmark) + metric_prefix = registry.get("metric_prefix", f"{benchmark}_") + verdict_metric = registry.get("verdict_metric", "") + sweep_label = registry.get("sweep_label", "") + + print("=" * 60) + print("Optimization Sweep Results Analyzer") + print("=" * 60) + print(f"Benchmark: {benchmark}") + print(f"Description: {description}") + print(f"Results dir: {results_dir}") + print(f"Output dir: {output_dir}") + print(f"Baseline: {args.baseline}") + print() + + if not os.path.isdir(results_dir): + print(f"ERROR: Results directory not found: {results_dir}") + sys.exit(1) + + # Auto-discover sweep label if not in registry + if not sweep_label: + print("Auto-discovering sweep label...") + sweep_label = auto_discover_sweep_label(results_dir) + print(f"Sweep label: {sweep_label}") + + # Load data + print("\nLoading variant data...") + all_data = load_variant_data(results_dir, metric_prefix, sweep_label) + + if not all_data: + # Try auto-discovering metric prefix + print(f"No data found with prefix '{metric_prefix}'. Auto-discovering...") + all_metrics_raw = set() + runs_dir = Path(results_dir) / "runs" + if runs_dir.is_dir(): + for variant_dir in sorted(runs_dir.iterdir()): + if not variant_dir.is_dir(): + continue + for pattern in [ + variant_dir / "perfkitbenchmarker_results.json", + ]: + if not pattern.is_file(): + continue + with open(pattern, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + m = record.get("metric", "") + if m: + all_metrics_raw.add(m) + + if all_metrics_raw: + metric_prefix = auto_discover_prefix(all_metrics_raw) + print(f"Auto-discovered prefix: '{metric_prefix}'") + all_data = load_variant_data(results_dir, metric_prefix, sweep_label) + + if not all_data: + print("ERROR: No data loaded. Check results directory.") + sys.exit(1) + + print(f"\nLoaded {len(all_data)} variants: {sorted(all_data.keys())}") + + # Auto-discover all metrics + all_metrics = discover_metrics(all_data) + print(f"Discovered {len(all_metrics)} unique metrics") + + # If verdict metric not set, try to pick a reasonable default + if not verdict_metric and all_metrics: + for m in all_metrics: + if "mean" in m and "wall_time" not in m: + verdict_metric = m + break + if not verdict_metric: + verdict_metric = all_metrics[0] + print(f"Auto-selected verdict metric: {verdict_metric}") + elif verdict_metric: + print(f"Verdict metric: {verdict_metric}") + + # Compute deltas + print("\nComputing deltas vs baseline...") + deltas = compute_deltas(all_data, args.baseline) + + # Generate report + print("\nGenerating report...") + report_path = generate_report( + benchmark, description, sweep_label, metric_prefix, + verdict_metric, all_data, deltas, all_metrics, output_dir, + ) + + # Export CSV + print("\nExporting CSV...") + csv_path = export_csv( + benchmark, metric_prefix, sweep_label, + all_data, deltas, all_metrics, output_dir, + ) + + print() + print("=" * 60) + print("DONE") + print("=" * 60) + print(f" Report: {report_path}") + print(f" CSV: {csv_path}") + print() + print("View the report:") + print(f" cat {report_path}") + + +if __name__ == "__main__": + main() diff --git a/perfkitbenchmarker/scripts/agentic/sweep.py b/perfkitbenchmarker/scripts/agentic/sweep.py new file mode 100644 index 0000000000..d2d4888fa8 --- /dev/null +++ b/perfkitbenchmarker/scripts/agentic/sweep.py @@ -0,0 +1,565 @@ +#!/usr/bin/env python3 +"""Optimization Sweep Runner - Multi-Benchmark. + +Runs baseline + variant benchmarks for any GKE agentic benchmark. +Drains warm pool between sweep levels for clean measurements. + +Usage: + python perfkitbenchmarker/scripts/agentic/sweep.py --benchmark k8s_python_density --variant baseline --stages run --sweep-values 1,4,8,16 + python perfkitbenchmarker/scripts/agentic/sweep.py --benchmark k8s_payload --variant baseline --stages run --sweep-values 0.01,0.1,1,5,10 + python perfkitbenchmarker/scripts/agentic/sweep.py --benchmark k8s_python_density --variant baseline --stages run --sweep-values 50,100 -- --k8s_python_density_sample_count=50 + python perfkitbenchmarker/scripts/agentic/sweep.py --benchmark k8s_python_density --all --stages provision,prepare,run,teardown +""" + +import argparse +import re +import logging +import os +import subprocess +import sys +import time +import urllib.request +import yaml +from jinja2 import Template as Jinja2Template + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + + +# ============================================================ +# Path Resolution +# ============================================================ +# Script lives in: perfkitbenchmarker/data/k8s_agents/config/optimization/ +# Repo root has: pkb.py + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _find_repo_root(): + """Walk up from script location to find the repo root (contains pkb.py).""" + d = SCRIPT_DIR + for _ in range(10): + if os.path.isfile(os.path.join(d, "pkb.py")) and os.path.isdir(os.path.join(d, "perfkitbenchmarker")): + return d + parent = os.path.dirname(d) + if parent == d: + break + d = parent + return None + + +REPO_ROOT = _find_repo_root() +if not REPO_ROOT: + logger.error( + "Cannot find pkb.py. Ensure this script is inside the PerfKitBenchmarker repo." + ) + sys.exit(1) + + +# ============================================================ +# Project Configuration +# ============================================================ + +PROJECT = "sada-gke-benchmarking2" +REGION = "us-east1" +OWNER = "george-kalisse" +NETWORK = "george-agentic-vpc" +SUBNET = "george-agentic-subnet" +NAMESPACE = "agentic" + +# CONFIG_DIR points to the variant config YAML files +CONFIG_DIR = os.path.join(REPO_ROOT, "perfkitbenchmarker", "data", + "k8s_agents", "config", "variants") +RESULTS_BASE = os.path.join(REPO_ROOT, "results", "pkb") +TUNING_DIR = os.path.join(CONFIG_DIR, "tuning") +PKB_CMD = "pkb.py" + + +# ============================================================ +# Benchmark Registry +# ============================================================ +# sweep_flag: PKB flag that receives the sweep value +# warmpool: SandboxWarmPool name to drain (None = skip) +# pod_label: Label selector for warm pool pods +# default_sweep: Default sweep values if --sweep-values not given +# drain_between: Whether to drain warm pool between sweep levels + +BENCHMARKS = { + "k8s_python_density": { + "sweep_flag": "--k8s_python_density_concurrent_sandbox_count", + "warmpool": "python-sandbox-warmpool", + "pod_label": "sandbox=python-sandbox-example", + "default_sweep": [1, 4, 8, 16, 32], + "drain_between": True, + }, + "k8s_chromium_density": { + "sweep_flag": "--k8s_chromium_density_concurrent_sessions", + "warmpool": "chromium-sandbox-warmpool", + "pod_label": "sandbox=chromium-sandbox-example", + "default_sweep": [1, 2, 4, 8], + "drain_between": True, + }, + "k8s_payload": { + "sweep_flag": "--k8s_payload_size_mb", + "warmpool": "python-sandbox-warmpool", + "pod_label": "sandbox=python-sandbox-example", + "default_sweep": [0.01, 0.1, 1, 5, 10], + "drain_between": True, + }, + "k8s_qps": { + "sweep_flag": "--k8s_qps_target_qps", + "warmpool": "python-sandbox-warmpool", + "pod_label": "sandbox=python-sandbox-example", + "default_sweep": [1, 5, 10, 20, 50], + "drain_between": True, + }, + "k8s_warmpool": { + "sweep_flag": "--k8s_warmpool_target_replicas", + "warmpool": "python-sandbox-warmpool", + "pod_label": "sandbox=python-sandbox-example", + "default_sweep": [10, 50, 100, 200, 500], + "drain_between": True, + }, + "k8s_deletion": { + "sweep_flag": "--k8s_deletion_batch_size", + "warmpool": "python-sandbox-warmpool", + "pod_label": "sandbox=python-sandbox-example", + "default_sweep": [10, 50, 100, 200, 500], + "drain_between": True, + }, + "k8s_snapshot": { + "sweep_flag": "--k8s_snapshot_preload_mb", + "warmpool": None, + "pod_label": "app=snapshot-benchmark-workload", + "default_sweep": [10, 50, 100, 500, 1000], + "drain_between": False, + }, +} + + +# ============================================================ +# Variant Registry +# ============================================================ + +ALL_VARIANTS = [ + "baseline", + "kubelet_pulls", + "overlay_none", + "larger_vm", + "hyperdisk_200gb", + "thp", + "sched_tuning", + "swap", + "c4d_amd", + "c4a_arm", + "multi_node", + "combined", +] + +VARIANT_CONFIGS = { + "baseline": "baseline.yaml", + "kubelet_pulls": "kubelet_pulls.yaml", + "overlay_none": "overlay_none.yaml", + "larger_vm": "larger_vm.yaml", + "hyperdisk_200gb": "hyperdisk_200gb.yaml", + "thp": "thp.yaml", + "sched_tuning": "sched_tuning.yaml", + "swap": "swap.yaml", + "c4d_amd": "c4d_amd.yaml", + "c4a_arm": "c4a_arm.yaml", + "multi_node": "multi_node.yaml", + "combined": "combined.yaml", +} + +VARIANT_DESC = { + "baseline": "Baseline: c4-standard-8, gVisor, defaults", + "kubelet_pulls": "Kubelet parallel pulls + image GC", + "overlay_none": "gVisor overlay2=none", + "larger_vm": "c4-standard-16 (2x CPU)", + "hyperdisk_200gb": "hyperdisk-balanced 200GB", + "thp": "Transparent Hugepages (THP=always)", + "sched_tuning": "Kernel scheduler tuning", + "swap": "Swap enabled (swappiness=15)", + "c4d_amd": "C4D AMD (c4d-standard-8)", + "c4a_arm": "C4A ARM (c4a-standard-8)", + "multi_node": "Multi-node sandbox pool (2x c4-standard-8)", + "combined": "Combined best knobs", +} + + +# ============================================================ +# Helpers +# ============================================================ + + +def get_my_ip(): + try: + return ( + urllib.request.urlopen("https://ifconfig.me", timeout=10) + .read() + .decode() + .strip() + ) + except Exception: + return ( + urllib.request.urlopen("https://api.ipify.org", timeout=10) + .read() + .decode() + .strip() + ) + + +def run_cmd(cmd, check=True): + logger.info("CMD: %s", cmd) + result = subprocess.run(cmd, shell=True) + if check and result.returncode != 0: + logger.error("Command failed (rc=%d)", result.returncode) + sys.exit(1) + return result.returncode + + +def drain_warmpool(warmpool_name, pod_label): + """Drain warm pool to 0 and delete lingering claims.""" + if not warmpool_name: + return + logger.info("Draining warm pool %s to 0", warmpool_name) + run_cmd( + "kubectl patch sandboxwarmpool " + + warmpool_name + + " -n " + + NAMESPACE + + ' --type=merge -p \'{"spec":{"replicas":0}}\'', + check=False, + ) + run_cmd( + "kubectl delete sandboxclaims --all -n " + + NAMESPACE + + " --ignore-not-found=true", + check=False, + ) + for _ in range(30): + result = subprocess.run( + "kubectl get pods -n " + + NAMESPACE + + " -l " + + pod_label + + " --no-headers 2>/dev/null | wc -l", + shell=True, + capture_output=True, + text=True, + ) + try: + count = int(result.stdout.strip()) + except ValueError: + count = 0 + if count == 0: + logger.info("Warm pool drained (0 pods)") + return + logger.info(" Draining... %d pods remaining", count) + time.sleep(3) + logger.warning("Drain timeout: pods may still be terminating") + + +def _read_yaml_flags(config_path, benchmark, flag_name, flag_type="list"): + """Read a single flag from a variant YAML config file. + + Args: + config_path: Path to the variant YAML file. + benchmark: Benchmark key to look up (e.g. 'k8s_snapshot'). + flag_name: Flag name under the 'flags:' section. + flag_type: 'list' returns a list of strings, 'string' returns + a single string or None. + + Returns: + list[str] when flag_type='list' (empty list if not found). + str or None when flag_type='string' (None if not found). + """ + empty = [] if flag_type == "list" else None + try: + with open(config_path, "r") as f: + data = yaml.safe_load(f) + if not data: + return empty + # Look up the specific benchmark key first + bench_config = data.get(benchmark) + if bench_config is None: + # Fall back to first benchmark key (backward compat) + for _, bench_config in data.items(): + if isinstance(bench_config, dict): + break + else: + return empty + if not isinstance(bench_config, dict): + return empty + fl = bench_config.get("flags", {}) + if not isinstance(fl, dict): + return empty + value = fl.get(flag_name) + if value is None: + return empty + if flag_type == "list": + if isinstance(value, list): + return [str(v).strip() for v in value if v] + return [str(value).strip()] + else: + return str(value).strip() + except Exception as e: + logger.warning( + "Could not read %s from %s: %s", flag_name, config_path, e + ) + return empty + + +def run_pkb( + benchmark, variant, stage, extra_flags=None, passthrough_flags=None, check=True +): + """Run PKB with the given benchmark, variant, and stage.""" + config = os.path.join(CONFIG_DIR, VARIANT_CONFIGS[variant]) + uri = re.sub(r"[^a-zA-Z0-9]", "", variant)[:12] + results = os.path.join(RESULTS_BASE, benchmark) + os.makedirs(results, exist_ok=True) + + parts = [ + "python " + PKB_CMD, + "--benchmarks=" + benchmark, + "--run_stage=" + stage, + "--project=" + PROJECT, + "--benchmark_config_file=" + config, + "--run_uri=" + uri, + "--json_write_mode=a", + "--temp_dir=" + results, + ] + + if "provision" in stage: + parts.append("--owner=" + OWNER) + parts.append("--gce_network_name=" + NETWORK) + my_ip = get_my_ip() + gke_flag_list = [ + "--workload-pool=" + PROJECT + ".svc.id.goog", + "--subnetwork=" + SUBNET, + "--enable-master-authorized-networks", + "--master-authorized-networks=" + my_ip + "/32", + ] + yaml_gke_flags = _read_yaml_flags(config, benchmark, "gke_additional_flags") + for yf in yaml_gke_flags: + if yf not in gke_flag_list: + gke_flag_list.append(yf) + parts.append("--gke_additional_flags=" + ",".join(gke_flag_list)) + yaml_nodepool_flags = _read_yaml_flags(config, benchmark, "gke_additional_nodepool_flags") + if yaml_nodepool_flags: + parts.append("--gke_additional_nodepool_flags=" + ",".join(yaml_nodepool_flags)) + + if "teardown" in stage: + parts.append("--gce_network_name=" + NETWORK) + + if extra_flags: + parts.append(extra_flags) + + if passthrough_flags: + parts.extend(passthrough_flags) + + cmd = " ".join(parts) + return run_cmd(cmd, check=check) + + +# ============================================================ +# Stage Handlers +# ============================================================ + + +def do_provision(benchmark, variant, passthrough_flags=None): + logger.info("=== PROVISION: %s / %s ===", benchmark, variant) + run_pkb(benchmark, variant, "provision", passthrough_flags=passthrough_flags) + + +def do_prepare(benchmark, variant, passthrough_flags=None): + logger.info("=== PREPARE: %s / %s ===", benchmark, variant) + run_pkb(benchmark, variant, "prepare", passthrough_flags=passthrough_flags) + apply_variant_extras(variant) + + +def do_run(benchmark, variant, sweep_values, passthrough_flags=None): + """Run the sweep across all values.""" + bench_cfg = BENCHMARKS[benchmark] + sweep_flag = bench_cfg["sweep_flag"] + warmpool = bench_cfg["warmpool"] + pod_label = bench_cfg["pod_label"] + do_drain = bench_cfg["drain_between"] + + logger.info( + "=== SWEEP: %s / %s -- values: %s ===", benchmark, variant, sweep_values + ) + failed = [] + try: + for val in sweep_values: + logger.info("--- %s=%s ---", sweep_flag, val) + extra = sweep_flag + "=" + str(val) + rc = run_pkb( + benchmark, + variant, + "run,cleanup", + extra_flags=extra, + passthrough_flags=passthrough_flags, + check=False, + ) + if rc != 0: + logger.warning("value=%s failed (rc=%d), continuing", val, rc) + failed.append(val) + + if do_drain and warmpool: + logger.info("Draining between sweep levels...") + drain_warmpool(warmpool, pod_label) + time.sleep(5) + finally: + if do_drain and warmpool: + logger.info("Final safety drain after sweep") + drain_warmpool(warmpool, pod_label) + + if failed: + logger.warning("Failed sweep values: %s", failed) + else: + logger.info("All sweep values completed successfully") + + +def do_teardown(benchmark, variant, passthrough_flags=None): + logger.info("=== TEARDOWN: %s / %s ===", benchmark, variant) + run_pkb(benchmark, variant, "teardown", passthrough_flags=passthrough_flags) + + +def apply_variant_extras(variant): + """Apply variant-specific configuration after Prepare.""" + if variant == "overlay_none": + logger.info("Patching SandboxTemplate: overlay2=none") + patch_json = ( + '{"spec":{"podTemplate":{"metadata":{"annotations":' + '{"dev.gvisor.spec.overlay2":"none"}}}}}' + ) + run_cmd( + "kubectl patch sandboxtemplate python-sandbox-template" + " -n " + NAMESPACE + " --type=merge -p '" + patch_json + "'", + check=False, + ) + time.sleep(10) + + if variant == "sched_tuning": + logger.info("Applying scheduler tuning DaemonSet") + j2_path = os.path.join(TUNING_DIR, "sched_tuner_daemonset.yaml.j2") + with open(j2_path, "r") as f: + rendered = Jinja2Template(f.read()).render(ns=NAMESPACE) + rendered_path = os.path.join(TUNING_DIR, "tmp", "sched_tuner_daemonset.yaml") + os.makedirs(os.path.dirname(rendered_path), exist_ok=True) + with open(rendered_path, "w") as f: + f.write(rendered) + run_cmd("kubectl apply -f " + rendered_path, check=False) + run_cmd( + "kubectl rollout status daemonset/sched-tuner -n " + + NAMESPACE + + " --timeout=60s", + check=False, + ) + time.sleep(5) + + +# ============================================================ +# Main +# ============================================================ + + +def main(): + parser = argparse.ArgumentParser( + description="Optimization Sweep Runner - Multi-Benchmark", + epilog="Pass additional PKB flags after --: sweep_optimization.py ... -- --flag1=val1 --flag2=val2", + ) + parser.add_argument( + "--benchmark", + required=True, + choices=sorted(BENCHMARKS.keys()), + help="PKB benchmark name to run.", + ) + parser.add_argument( + "--variant", + action="append", + default=[], + help="Variant to run (can specify multiple).", + ) + parser.add_argument("--all", action="store_true", help="Run all variants.") + parser.add_argument( + "--stages", + default="provision,prepare,run,teardown", + help="Comma-separated stages (default: provision,prepare,run,teardown).", + ) + parser.add_argument( + "--sweep-values", + default=None, + help="Comma-separated sweep values. Defaults vary by benchmark.", + ) + + args, passthrough = parser.parse_known_args() + passthrough_flags = [f for f in passthrough if f != "--"] if passthrough else None + + benchmark = args.benchmark + bench_cfg = BENCHMARKS[benchmark] + + if getattr(args, "all"): + variants = list(ALL_VARIANTS) + elif args.variant: + variants = args.variant + else: + logger.error("Specify --variant or --all") + logger.error("Available: %s", ", ".join(ALL_VARIANTS)) + sys.exit(1) + + for v in variants: + if v not in VARIANT_CONFIGS: + logger.error( + "Unknown variant: %s. Available: %s", v, ", ".join(ALL_VARIANTS) + ) + sys.exit(1) + + stages = [s.strip() for s in args.stages.split(",")] + valid_stages = {"provision", "prepare", "run", "teardown"} + for s in stages: + if s not in valid_stages: + logger.error( + "Unknown stage: %s. Valid: %s", s, ", ".join(sorted(valid_stages)) + ) + sys.exit(1) + + if args.sweep_values: + raw = [v.strip() for v in args.sweep_values.split(",")] + sweep_values = [float(v) if "." in v else int(v) for v in raw] + else: + sweep_values = list(bench_cfg["default_sweep"]) + + logger.info("OPTIMIZATION SWEEP") + logger.info(" Benchmark: %s", benchmark) + logger.info(" Sweep flag: %s", bench_cfg["sweep_flag"]) + logger.info(" Variants: %s", variants) + logger.info(" Stages: %s", stages) + logger.info(" Sweep: %s", sweep_values) + if passthrough_flags: + logger.info(" Passthrough: %s", passthrough_flags) + logger.info(" Repo root: %s", REPO_ROOT) + + for variant in variants: + logger.info("") + logger.info("=" * 60) + logger.info(" %s / %s: %s", benchmark, variant, VARIANT_DESC.get(variant, "")) + logger.info("=" * 60) + + if "provision" in stages: + do_provision(benchmark, variant, passthrough_flags) + if "prepare" in stages: + do_prepare(benchmark, variant, passthrough_flags) + if "run" in stages: + do_run(benchmark, variant, sweep_values, passthrough_flags) + if "teardown" in stages: + do_teardown(benchmark, variant, passthrough_flags) + + logger.info("=== %s / %s COMPLETE ===", benchmark, variant) + + logger.info("ALL DONE. Results in: %s/", os.path.join(RESULTS_BASE, benchmark)) + + +if __name__ == "__main__": + main()