diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml new file mode 100644 index 000000000000..5e4b7cd28c16 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -0,0 +1,119 @@ +substitutions: + _ZONE: "us-west4-a" + _VM_NAME: "gcs-benchmark-runner-us-west4-a" + _ULIMIT: "65536" + _PROCESSES: "48" + _COROS: "1" + _FILE_SIZE_MIB: "10240" + _CHUNK_SIZE_KIB: "102400" + _ROUNDS: "3" + _BUCKET_TYPE: "zonal" + _ZONAL_BUCKET: "gcs-read-bench-zb-us-west4-a" + _REGIONAL_BUCKET: "gcs-read-bench-rb-us-west4" + _PR_NUMBER: "" + _REPO: "googleapis/google-cloud-python" + +steps: + # Step 0: Generate a temporary SSH key for this build run and register with OS Login + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + gcloud compute os-login ssh-keys add \ + --key-file=/workspace/.ssh/google_compute_engine.pub \ + --ttl=1h + waitFor: ["-"] + + # Step 1: Package google-cloud-storage directory for direct transfer to VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "package-code" + entrypoint: "bash" + args: + - "-c" + - | + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage + waitFor: ["-"] + + # Step 2: Start VM, run benchmark directly via private internal IP SSH, fetch results, and clean up + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-benchmark-on-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + cleanup() { + set +e + echo "Stopping VM ${_VM_NAME}..." + gcloud compute instances stop "${_VM_NAME}" --zone="${_ZONE}" --quiet + echo "Removing temporary build SSH key from OS Login profile..." + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + } + trap cleanup EXIT + + echo "Starting standing VM ${_VM_NAME} in zone ${_ZONE}..." + gcloud compute instances start "${_VM_NAME}" --zone="${_ZONE}" + + echo "Waiting for VM ${_VM_NAME} to become accessible over internal SSH..." + SSH_READY=0 + for i in $(seq 1 20); do + if gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" 2>/dev/null; then + echo "VM internal SSH connection established successfully." + SSH_READY=1 + break + fi + echo "Waiting for VM internal SSH availability... (attempt $$i/20)" + sleep 10 + done + + if [ $$SSH_READY -ne 1 ]; then + echo "ERROR: VM internal SSH connection could not be established." >&2 + exit 1 + fi + + echo "Copying package archive to VM over internal IP..." + gcloud compute scp /workspace/google-cloud-storage.tar.gz \ + "${_VM_NAME}":~ --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + echo "Executing benchmark test suite directly on VM via SSH..." + set +e + gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ + --command="tar -xzf google-cloud-storage.tar.gz && cd google-cloud-storage && ulimit -n ${_ULIMIT}; PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} ROUNDS=${_ROUNDS} BUCKET_TYPE=${_BUCKET_TYPE} ZONAL_BUCKET=${_ZONAL_BUCKET} REGIONAL_BUCKET=${_REGIONAL_BUCKET} bash cloudbuild/run_benchmark_tests.sh" + TEST_EXIT_CODE=$? + set -e + + # Copy JSON report back from VM to Cloud Build workspace + mkdir -p /workspace/report + echo "Fetching benchmark result JSON from VM..." + gcloud compute scp "${_VM_NAME}":~/bench_result.json /workspace/report/bench_result.json \ + --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine 2>/dev/null || true + + exit $$TEST_EXIT_CODE + waitFor: + - "generate-ssh-key" + - "package-code" + + # Step 4: Display benchmark performance summary table in Cloud Build logs + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "display-benchmark-results" + entrypoint: "python3" + args: + - "packages/google-cloud-storage/cloudbuild/display_benchmark_results.py" + - "/workspace/report/bench_result.json" + waitFor: + - "run-benchmark-on-vm" + +timeout: "3600s" + +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true + pool: + name: "projects/${PROJECT_ID}/locations/us-west4/workerPools/benchmark-worker-pool" diff --git a/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py b/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py new file mode 100644 index 000000000000..d22b3f44cbb9 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py @@ -0,0 +1,81 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper script to format and display GCS benchmark performance results.""" + +import json +import os +import sys + + +def display_results(result_path: str) -> None: + """Reads benchmark JSON result and prints a formatted summary table.""" + if not os.path.exists(result_path): + print( + f"ERROR: Benchmark result file not found at {result_path}", file=sys.stderr + ) + sys.exit(1) + + with open(result_path) as f: + data = json.load(f) + + if not isinstance(data, dict): + print( + "ERROR: Invalid JSON structure in benchmark result file.", file=sys.stderr + ) + sys.exit(1) + + benchmarks = data.get("benchmarks", []) + if not isinstance(benchmarks, list) or not benchmarks: + print("No benchmarks found in result file.") + sys.exit(0) + + print("\n" + "=" * 88) + print(" GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS") + print("=" * 88) + header = f"| {'Workload Pattern':<36} | {'Avg Throughput':<17} | {'Network Bandwidth':<22} | {'CPU Usage':<9} |" + print(header) + print("|" + "-" * 38 + "|" + "-" * 19 + "|" + "-" * 24 + "|" + "-" * 11 + "|") + for b in benchmarks: + if not isinstance(b, dict): + continue + name = ( + b.get("name", "") + .replace("test_downloads_multi_proc_multi_coro[", "") + .replace("]", "") + ) + extra = b.get("extra_info", {}) + if not isinstance(extra, dict): + extra = {} + avg_mib = extra.get("avg_throughput_mib_s", "N/A") + net_mb = extra.get("net_throughput_mb_s") + if net_mb: + try: + net_str = ( + f"{float(net_mb):,.1f} MB/s ({float(net_mb) * 0.008:.1f} Gbps)" + ) + except Exception: + net_str = str(net_mb) + else: + net_str = "N/A" + cpu = extra.get("cpu_max_global", "N/A") + print( + f"| {name:<36} | {str(avg_mib) + ' MiB/s':<17} | {net_str:<22} | {str(cpu):<9} |" + ) + print("=" * 88 + "\n") + + +if __name__ == "__main__": + path = sys.argv[1] if len(sys.argv) > 1 else "/workspace/report/bench_result.json" + display_results(path) diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh new file mode 100755 index 000000000000..ad42d0cddcf4 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# ============================================================================== +# Automated Google Cloud Storage Read Microbenchmark Runner +# Intended for GitHub CI/CD & GCE High-Bandwidth Tier-1 VMs (C4/N2/C3 series) +# Location: packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +# ============================================================================== + +set -eo pipefail + +# Configurable defaults +PROCESSES="${PROCESSES:-48}" +COROS="${COROS:-1}" +FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default +CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default +ROUNDS="${ROUNDS:-3}" # Run benchmark 3 rounds by default +BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath, "regional" uses REST/gRPC standard +ZONAL_BUCKET="${ZONAL_BUCKET:-${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}}" +REGIONAL_BUCKET="${REGIONAL_BUCKET:-${DEFAULT_STANDARD_BUCKET:-gcs-read-bench-rb-us-west4}}" +if [ -n "${TARGET_BUCKET:-}" ]; then + if [ "${BUCKET_TYPE}" = "regional" ]; then + REGIONAL_BUCKET="${TARGET_BUCKET}" + else + ZONAL_BUCKET="${TARGET_BUCKET}" + fi +fi +# Ensure HOME is exported for gRPC / ALTS Application Default Credentials +export HOME="${HOME:-/root}" +OUTPUT_JSON_PATH="${OUTPUT_JSON_PATH:-${OUT_JSON:-${HOME}/bench_result.json}}" +rm -f "${OUTPUT_JSON_PATH}" 2>/dev/null || true +UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" + +echo "========================================================================" +echo " GCS Read Microbenchmark Runner (gRPC BidiReadObject / REST)" +echo " Processes: ${PROCESSES}" +echo " Coroutines/proc: ${COROS}" +echo " File Size: ${FILE_SIZE_MIB} MiB" +echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Rounds: ${ROUNDS}" +echo " Bucket Type: ${BUCKET_TYPE}" +echo " Zonal Bucket: gs://${ZONAL_BUCKET}" +echo " Regional Bucket: gs://${REGIONAL_BUCKET}" +echo " Output JSON Path: ${OUTPUT_JSON_PATH}" +echo " Upload GCS Path: ${UPLOAD_GCS_PREFIX:-None}" +echo "========================================================================" + +export DEFAULT_RAPID_ZONAL_BUCKET="${ZONAL_BUCKET}" +export DEFAULT_STANDARD_BUCKET="${REGIONAL_BUCKET}" +export PROCESSES="${PROCESSES}" +export COROS="${COROS}" +export FILE_SIZE_MIB="${FILE_SIZE_MIB}" +export CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB}" +export ROUNDS="${ROUNDS}" +export BUCKET_TYPE="${BUCKET_TYPE}" + +# Navigate to package directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}/.." + +echo "--- 1. Setting up Python environment ---" +# Ensure python3-pip and python3-venv are present on the VM +if ! command -v pip3 &>/dev/null || ! python3 -c "import venv" 2>/dev/null; then + echo "Installing python3-pip and python3-venv on VM..." + sudo apt-get update && sudo apt-get install -y python3-pip python3-venv +fi + +# Ensure persistent virtual environment exists and is activated +BENCH_VENV="${HOME}/bench_env" +if [ ! -d "${BENCH_VENV}" ]; then + echo "Creating virtual environment at ${BENCH_VENV}..." + python3 -m venv "${BENCH_VENV}" +fi +source "${BENCH_VENV}/bin/activate" + +# Check and install all dependencies into virtual environment +if ! python3 -c "import pytest, psutil, yaml, google.cloud.storage" 2>/dev/null; then + echo "Installing dependencies into virtual environment..." + pip install --upgrade pip + pip install -e ".[grpc,testing]" +fi + +echo "--- 2. Executing pytest benchmark suite (${ROUNDS} rounds) ---" +set +e +python3 -m pytest --benchmark-json="${OUTPUT_JSON_PATH}" \ + -rA \ + tests/perf/microbenchmarks/time_based/reads/test_reads.py +TEST_EXIT_CODE=$? +set -e + +if [ -s "${OUTPUT_JSON_PATH}" ]; then + DISPLAY_SCRIPT="${SCRIPT_DIR}/display_benchmark_results.py" + if [ ! -f "${DISPLAY_SCRIPT}" ]; then + DISPLAY_SCRIPT="cloudbuild/display_benchmark_results.py" + fi + python3 "${DISPLAY_SCRIPT}" "${OUTPUT_JSON_PATH}" + + if [ -n "${UPLOAD_GCS_PREFIX}" ]; then + GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" + echo "Uploading JSON report to ${GCS_DEST}..." + gcloud storage cp "${OUTPUT_JSON_PATH}" "${GCS_DEST}" + fi +fi + +echo "--- Benchmark Run Complete ---" +exit $TEST_EXIT_CODE diff --git a/packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/config.py b/packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/config.py index a7bf67f465d3..add541c6596a 100644 --- a/packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/config.py +++ b/packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/config.py @@ -33,13 +33,39 @@ def _get_params() -> Dict[str, List[TimeBasedReadParameters]]: config = yaml.safe_load(f) common_params = config["common"] - bucket_types = common_params["bucket_types"] - file_sizes_mib = common_params["file_sizes_mib"] - chunk_sizes_kib = common_params["chunk_sizes_kib"] - num_ranges = common_params["num_ranges"] - rounds = common_params["rounds"] - duration = common_params["duration"] - warmup_duration = common_params["warmup_duration"] + bucket_types = ( + [b.strip() for b in os.environ["BUCKET_TYPE"].split(",") if b.strip()] + if "BUCKET_TYPE" in os.environ + else common_params["bucket_types"] + ) + file_sizes_mib = ( + [int(s.strip()) for s in os.environ["FILE_SIZE_MIB"].split(",") if s.strip()] + if "FILE_SIZE_MIB" in os.environ + else common_params["file_sizes_mib"] + ) + chunk_sizes_kib = ( + [int(c.strip()) for c in os.environ["CHUNK_SIZE_KIB"].split(",") if c.strip()] + if "CHUNK_SIZE_KIB" in os.environ + else common_params["chunk_sizes_kib"] + ) + num_ranges = ( + [int(r.strip()) for r in os.environ["NUM_RANGES"].split(",") if r.strip()] + if "NUM_RANGES" in os.environ + else common_params["num_ranges"] + ) + rounds = ( + int(os.environ["ROUNDS"]) if "ROUNDS" in os.environ else common_params["rounds"] + ) + duration = ( + int(os.environ["DURATION"]) + if "DURATION" in os.environ + else common_params["duration"] + ) + warmup_duration = ( + int(os.environ["WARMUP_DURATION"]) + if "WARMUP_DURATION" in os.environ + else common_params["warmup_duration"] + ) bucket_map = { "zonal": os.environ.get( @@ -51,12 +77,25 @@ def _get_params() -> Dict[str, List[TimeBasedReadParameters]]: ), } + env_processes = ( + [int(p.strip()) for p in os.environ["PROCESSES"].split(",") if p.strip()] + if "PROCESSES" in os.environ + else None + ) + env_coros = ( + [int(c.strip()) for c in os.environ["COROS"].split(",") if c.strip()] + if "COROS" in os.environ + else None + ) + for workload in config["workload"]: workload_name = workload["name"] params[workload_name] = [] pattern = workload["pattern"] - processes = workload["processes"] - coros = workload["coros"] + processes = ( + env_processes if env_processes is not None else workload["processes"] + ) + coros = env_coros if env_coros is not None else workload["coros"] # Create a product of all parameter combinations product = itertools.product(