diff --git a/README.md b/README.md index 4125e05..d316f96 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ The general flow looks as follows: * Then to start any benchmark run you do `make start CLUSTER_NAME=my-prombenchy BENCH_NAME= SCENARIO=./manifests/scenarios/gmp`. This will setup node-pool and your collector (e.g. as daemon set or separate pod - up to you, as long as you do correct node section!) You can start as many scenarios as you want on the single cluster (make sure to use unique `BENCH_NAME` though!). The scenario is a path to the "collector" manifest, so anything that will scrape `./manifests/load/avalanche.exampletarget.yaml`. Feel free to adjust anything in `./manifests/scenarios/` or add your own. You are also welcome to create custom scenarios under `scenarios/`, store them locally or propose to this repo. + + Available scenarios under `./manifests/scenarios/`: + * `./manifests/scenarios/gmp`: Google Cloud Managed Service for Prometheus collector (`prometheus-engine/prometheus`). + * `./manifests/scenarios/gmp-otel`: Go OpenTelemetry Collector (`otelcol-contrib`) with `prometheus` receiver and `googlemanagedprometheus` exporter. + * `./manifests/scenarios/otap-rust`: Rust OTAP Dataflow Engine (`df_engine`) with `urn:otel:receiver:prometheus` (`prometheus/scrape.rs`), `urn:otel:extension:gcp_auth`, and `urn:otel:exporter:otlp_grpc` exporting to Google Cloud Telemetry (`https://telemetry.googleapis.com:443`). `prombenchy` setup uses separate meta-monitoring containers in `core` namespace: * Separate Prometheus for gathering metrics about core resources and collectors (available locally, but also sends all to GCM). Make sure your pod has `app=collector` label and relevant port name has `-ins` suffix, to be scraped by this core Prometheus. There is also a dashboard you can apply to GCM in `./dashboards/`. diff --git a/manifests/core/1b_prometheus.yaml b/manifests/core/1b_prometheus.yaml index 732c0d6..2087ef6 100644 --- a/manifests/core/1b_prometheus.yaml +++ b/manifests/core/1b_prometheus.yaml @@ -119,6 +119,15 @@ data: - source_labels: [__meta_kubernetes_pod_container_port_name] regex: .*-ins$ action: keep + - action: replace + source_labels: [__meta_kubernetes_pod_container_port_name] + regex: otap-ins + target_label: __metrics_path__ + replacement: /api/v1/metrics + - action: replace + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + regex: (.+) + target_label: __metrics_path__ - action: replace source_labels: [__meta_kubernetes_pod_label_benchmark] target_label: benchmark diff --git a/manifests/core/8b_parca.yaml b/manifests/core/8b_parca.yaml index 2402cda..5a41de4 100644 --- a/manifests/core/8b_parca.yaml +++ b/manifests/core/8b_parca.yaml @@ -31,7 +31,7 @@ data: regex: collector - action: keep source_labels: [__meta_kubernetes_pod_container_name] - regex: prometheus|otel-collector + regex: prometheus|otel-collector|otap-collector - action: replace source_labels: [__meta_kubernetes_pod_label_app] target_label: job diff --git a/manifests/load/avalanche.exampletarget.yaml b/manifests/load/avalanche.exampletarget.yaml index cb60a60..9bdca30 100644 --- a/manifests/load/avalanche.exampletarget.yaml +++ b/manifests/load/avalanche.exampletarget.yaml @@ -45,3 +45,19 @@ spec: name: metrics nodeSelector: role: {{ .Env.BENCH_NAME }}-work +--- +apiVersion: v1 +kind: Service +metadata: + name: avalanche + namespace: {{ .Env.BENCH_NAME }} + labels: + app: avalanche +spec: + clusterIP: None + selector: + app: avalanche + ports: + - name: metrics + port: 9001 + targetPort: metrics diff --git a/manifests/scenarios/otap-rust/1_collector.yaml b/manifests/scenarios/otap-rust/1_collector.yaml new file mode 100644 index 0000000..8fecc99 --- /dev/null +++ b/manifests/scenarios/otap-rust/1_collector.yaml @@ -0,0 +1,198 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: collector + namespace: {{ .Env.BENCH_NAME }} + annotations: + iam.gke.io/gcp-service-account: gmp-prombench@{{ .Env.PROJECT_ID }}.iam.gserviceaccount.com +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Env.BENCH_NAME }}:collector +rules: +- resources: + - endpoints + - nodes + - nodes/metrics + - pods + - services + apiGroups: [""] + verbs: ["get", "list", "watch"] +- resources: + - configmaps + apiGroups: [""] + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Env.BENCH_NAME }}:collector +roleRef: + name: {{ .Env.BENCH_NAME }}:collector + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: +- name: collector + namespace: {{ .Env.BENCH_NAME }} + kind: ServiceAccount +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otap-collector-config + namespace: {{ .Env.BENCH_NAME }} +data: + config.yaml: | + version: otel_dataflow/v1 + engine: {} + groups: + default: + pipelines: + main: + policies: + resources: + core_allocation: + type: core_count + count: 1 + channel_capacity: + control: + node: 100 + pipeline: 100 + pdata: 256 + extensions: + gcp_auth: + type: urn:otel:extension:gcp_auth + config: {} + nodes: + receiver: + type: urn:otel:receiver:prometheus + config: + job_name: avalanche + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + enable_open_metrics: true + max_data_points_per_batch: 200 + dns_sd_targets: + - "avalanche.{{ .Env.BENCH_NAME }}.svc.cluster.local:9001" + resource_attrs: + type: urn:otel:processor:attribute + config: + apply_to: ["resource"] + actions: + - action: upsert + key: "gcp.project_id" + value: "{{ .Env.PROJECT_ID }}" + - action: upsert + key: "cloud.availability_zone" + value: "{{ .Env.ZONE }}" + - action: upsert + key: "k8s.cluster.name" + value: "{{ .Env.CLUSTER_NAME }}" + - action: upsert + key: "k8s.namespace.name" + value: "{{ .Env.BENCH_NAME }}" + exporter: + type: urn:otel:exporter:otlp_grpc + capabilities: + bearer_token_provider: gcp_auth + config: + grpc_endpoint: "https://telemetry.googleapis.com:443" + compression: gzip + connections: + - from: receiver + to: resource_attrs + - from: resource_attrs + to: exporter +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: collector + namespace: {{ .Env.BENCH_NAME }} + labels: + benchmark: {{ .Env.BENCH_NAME }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: collector + template: + metadata: + labels: + app: collector + app.kubernetes.io/name: collector + benchmark: {{ .Env.BENCH_NAME }} + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + prometheus.io/path: /api/v1/metrics + spec: + serviceAccountName: collector + automountServiceAccountToken: true + containers: + - name: otap-collector + image: gcr.io/gpe-test-1/otap-df-engine:prometheus-receiver-gcp-auth-v0.2.0 + args: + - "--config=/conf/config.yaml" + - "--num-cores=1" + - "--http-admin-bind=0.0.0.0:8888" + env: + - name: MALLOC_CONF + value: "prof:true,prof_active:true,lg_prof_sample:19" + volumeMounts: + - name: otap-collector-config + mountPath: /conf + readOnly: true + ports: + - name: otap-ins # -ins, tells core Prometheus to scrape it. + containerPort: 8888 + livenessProbe: + httpGet: + path: /api/v1/livez + port: 8888 + readinessProbe: + httpGet: + path: /api/v1/readyz + port: 8888 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + volumes: + - name: otap-collector-config + configMap: + name: otap-collector-config + items: + - key: config.yaml + path: config.yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + securityContext: + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + nodeSelector: + role: {{ .Env.BENCH_NAME }}-work diff --git a/manifests/scenarios/otap-rust/Dockerfile b/manifests/scenarios/otap-rust/Dockerfile new file mode 100644 index 0000000..a630c2b --- /dev/null +++ b/manifests/scenarios/otap-rust/Dockerfile @@ -0,0 +1,18 @@ +# Build from the open-telemetry/otel-arrow repository root (branch bwplotka/prometheus-receiver-gcp-auth): +# docker build -f manifests/scenarios/otap-rust/Dockerfile \ +# -t gcr.io/gpe-test-1/otap-df-engine:prometheus-receiver-gcp-auth-v0.2.0 /path/to/otel-arrow +FROM rust:1.99-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +COPY . . + +WORKDIR /workspace/rust/otap-dataflow +RUN cargo build --locked --release --features gcp-auth --bin df_engine + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +COPY --from=builder /workspace/rust/otap-dataflow/target/release/df_engine /df_engine +USER 1000:1000 +ENTRYPOINT ["/df_engine"]