scenarios: add otap-rust collector scenario

README.md

@@ -15,6 +15,11 @@ The general flow looks as follows:
 * Then to start any benchmark run you do `make start CLUSTER_NAME=my-prombenchy BENCH_NAME=<name of benchmark, also k8s namespace> SCENARIO=./manifests/scenarios/gmp`. This will setup node-pool and your collector (e.g. as daemon set or separate pod - up to you, as long as you do correct node section!)
 
     You can start as many scenarios as you want on the single cluster (make sure to use unique `BENCH_NAME` though!). The scenario is a path to the "collector" manifest, so anything that will scrape `./manifests/load/avalanche.exampletarget.yaml`. Feel free to adjust anything in `./manifests/scenarios/` or add your own. You are also welcome to create custom scenarios under `scenarios/`, store them locally or propose to this repo.
+
+    Available scenarios under `./manifests/scenarios/`:
+    * `./manifests/scenarios/gmp`: Google Cloud Managed Service for Prometheus collector (`prometheus-engine/prometheus`).
+    * `./manifests/scenarios/gmp-otel`: Go OpenTelemetry Collector (`otelcol-contrib`) with `prometheus` receiver and `googlemanagedprometheus` exporter.
+    * `./manifests/scenarios/otap-rust`: Rust OTAP Dataflow Engine (`df_engine`) with `urn:otel:receiver:prometheus` (`prometheus/scrape.rs`), `urn:otel:extension:gcp_auth`, and `urn:otel:exporter:otlp_grpc` exporting to Google Cloud Telemetry (`https://telemetry.googleapis.com:443`).
 
     `prombenchy` setup uses separate meta-monitoring containers in `core` namespace:
     * Separate Prometheus for gathering metrics about core resources and collectors (available locally, but also sends all to GCM). Make sure your pod has `app=collector` label and relevant port name has `-ins` suffix, to be scraped by this core Prometheus. There is also a dashboard you can apply to GCM in `./dashboards/`.

manifests/core/1b_prometheus.yaml

@@ -119,6 +119,15 @@ data:
       - source_labels: [__meta_kubernetes_pod_container_port_name]
         regex: .*-ins$
         action: keep
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_container_port_name]
+        regex: otap-ins
+        target_label: __metrics_path__
+        replacement: /api/v1/metrics
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        regex: (.+)
+        target_label: __metrics_path__
       - action: replace
         source_labels: [__meta_kubernetes_pod_label_benchmark]
         target_label: benchmark

manifests/core/8b_parca.yaml

@@ -31,7 +31,7 @@ data:
             regex: collector
           - action: keep
             source_labels: [__meta_kubernetes_pod_container_name]
-            regex: prometheus|otel-collector
+            regex: prometheus|otel-collector|otap-collector
           - action: replace 
             source_labels: [__meta_kubernetes_pod_label_app]
             target_label: job

manifests/load/avalanche.exampletarget.yaml

@@ -45,3 +45,19 @@ spec:
           name: metrics
       nodeSelector:
         role: {{ .Env.BENCH_NAME }}-work
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: avalanche
+  namespace: {{ .Env.BENCH_NAME }}
+  labels:
+    app: avalanche
+spec:
+  clusterIP: None
+  selector:
+    app: avalanche
+  ports:
+  - name: metrics
+    port: 9001
+    targetPort: metrics

manifests/scenarios/otap-rust/1_collector.yaml

@@ -0,0 +1,198 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: collector
+  namespace: {{ .Env.BENCH_NAME }}
+  annotations:
+    iam.gke.io/gcp-service-account: gmp-prombench@{{ .Env.PROJECT_ID }}.iam.gserviceaccount.com
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Env.BENCH_NAME }}:collector
+rules:
+- resources:
+  - endpoints
+  - nodes
+  - nodes/metrics
+  - pods
+  - services
+  apiGroups: [""]
+  verbs: ["get", "list", "watch"]
+- resources:
+  - configmaps
+  apiGroups: [""]
+  verbs: ["get"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Env.BENCH_NAME }}:collector
+roleRef:
+  name: {{ .Env.BENCH_NAME }}:collector
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- name: collector
+  namespace: {{ .Env.BENCH_NAME }}
+  kind: ServiceAccount
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otap-collector-config
+  namespace: {{ .Env.BENCH_NAME }}
+data:
+  config.yaml: |
+    version: otel_dataflow/v1
+    engine: {}
+    groups:
+      default:
+        pipelines:
+          main:
+            policies:
+              resources:
+                core_allocation:
+                  type: core_count
+                  count: 1
+              channel_capacity:
+                control:
+                  node: 100
+                  pipeline: 100
+                pdata: 256
+            extensions:
+              gcp_auth:
+                type: urn:otel:extension:gcp_auth
+                config: {}
+            nodes:
+              receiver:
+                type: urn:otel:receiver:prometheus
+                config:
+                  job_name: avalanche
+                  scrape_interval: 15s
+                  scrape_timeout: 10s
+                  metrics_path: /metrics
+                  enable_open_metrics: true
+                  max_data_points_per_batch: 200
+                  dns_sd_targets:
+                    - "avalanche.{{ .Env.BENCH_NAME }}.svc.cluster.local:9001"
+              resource_attrs:
+                type: urn:otel:processor:attribute
+                config:
+                  apply_to: ["resource"]
+                  actions:
+                    - action: upsert
+                      key: "gcp.project_id"
+                      value: "{{ .Env.PROJECT_ID }}"
+                    - action: upsert
+                      key: "cloud.availability_zone"
+                      value: "{{ .Env.ZONE }}"
+                    - action: upsert
+                      key: "k8s.cluster.name"
+                      value: "{{ .Env.CLUSTER_NAME }}"
+                    - action: upsert
+                      key: "k8s.namespace.name"
+                      value: "{{ .Env.BENCH_NAME }}"
+              exporter:
+                type: urn:otel:exporter:otlp_grpc
+                capabilities:
+                  bearer_token_provider: gcp_auth
+                config:
+                  grpc_endpoint: "https://telemetry.googleapis.com:443"
+                  compression: gzip
+            connections:
+              - from: receiver
+                to: resource_attrs
+              - from: resource_attrs
+                to: exporter
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: collector
+  namespace: {{ .Env.BENCH_NAME }}
+  labels:
+    benchmark: {{ .Env.BENCH_NAME }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: collector
+  template:
+    metadata:
+      labels:
+        app: collector
+        app.kubernetes.io/name: collector
+        benchmark: {{ .Env.BENCH_NAME }}
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
+        prometheus.io/path: /api/v1/metrics
+    spec:
+      serviceAccountName: collector
+      automountServiceAccountToken: true
+      containers:
+      - name: otap-collector
+        image: gcr.io/gpe-test-1/otap-df-engine:prometheus-receiver-gcp-auth-v0.2.0
+        args:
+          - "--config=/conf/config.yaml"
+          - "--num-cores=1"
+          - "--http-admin-bind=0.0.0.0:8888"
+        env:
+        - name: MALLOC_CONF
+          value: "prof:true,prof_active:true,lg_prof_sample:19"
+        volumeMounts:
+        - name: otap-collector-config
+          mountPath: /conf
+          readOnly: true
+        ports:
+        - name: otap-ins # -ins, tells core Prometheus to scrape it.
+          containerPort: 8888
+        livenessProbe:
+          httpGet:
+            path: /api/v1/livez
+            port: 8888
+        readinessProbe:
+          httpGet:
+            path: /api/v1/readyz
+            port: 8888
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          privileged: false
+      volumes:
+      - name: otap-collector-config
+        configMap:
+          name: otap-collector-config
+          items:
+          - key: config.yaml
+            path: config.yaml
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                - arm64
+                - amd64
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      tolerations:
+      - effect: NoExecute
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
+      securityContext:
+        runAsGroup: 1000
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      nodeSelector:
+        role: {{ .Env.BENCH_NAME }}-work

manifests/scenarios/otap-rust/Dockerfile

@@ -0,0 +1,18 @@
+# Build from the open-telemetry/otel-arrow repository root (branch bwplotka/prometheus-receiver-gcp-auth):
+#   docker build -f manifests/scenarios/otap-rust/Dockerfile \
+#     -t gcr.io/gpe-test-1/otap-df-engine:prometheus-receiver-gcp-auth-v0.2.0 /path/to/otel-arrow
+FROM rust:1.99-bookworm AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace
+COPY . .
+
+WORKDIR /workspace/rust/otap-dataflow
+RUN cargo build --locked --release --features gcp-auth --bin df_engine
+
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /workspace/rust/otap-dataflow/target/release/df_engine /df_engine
+USER 1000:1000
+ENTRYPOINT ["/df_engine"]