From e6487bc9fa36262a3954d1118ecdf5d68f0e0a6b Mon Sep 17 00:00:00 2001 From: Sahil Batra Date: Wed, 1 Jul 2026 18:12:52 -0400 Subject: [PATCH 1/2] Fix stackdriver-exporter: use CLI flags and mount GCP credentials The exporter v0.19+ requires --monitoring.metrics-prefixes CLI flags instead of the STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES env var (which was silently ignored, causing "metrics_prefixes must have at least one entry" error). Also mount Application Default Credentials for GCP authentication. Co-Authored-By: Claude Opus 4.6 --- observability/docker-compose.observability.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/observability/docker-compose.observability.yml b/observability/docker-compose.observability.yml index 607b3be..f178bb6 100644 --- a/observability/docker-compose.observability.yml +++ b/observability/docker-compose.observability.yml @@ -38,8 +38,18 @@ services: stackdriver-exporter: image: prometheuscommunity/stackdriver-exporter:latest + command: + - --google.project-ids=${GCP_PROJECT_ID:-deeplearning-sahil} + - --monitoring.metrics-prefixes=dataflow.googleapis.com/job + - --monitoring.metrics-prefixes=custom.googleapis.com/dataflow + - --monitoring.metrics-prefixes=workload.googleapis.com + - --monitoring.metrics-prefixes=bigtable.googleapis.com/server + - --monitoring.metrics-prefixes=pubsub.googleapis.com + - --monitoring.metrics-prefixes=run.googleapis.com + - --monitoring.metrics-prefixes=bigquery.googleapis.com/storage + volumes: + - $HOME/.config/gcloud/application_default_credentials.json:/credentials/adc.json:ro environment: - - STACKDRIVER_EXPORTER_GOOGLE_PROJECT_ID=${GCP_PROJECT_ID:-deeplearning-sahil} - - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,custom.googleapis.com/dataflow,workload.googleapis.com,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com,bigquery.googleapis.com/storage + - GOOGLE_APPLICATION_CREDENTIALS=/credentials/adc.json ports: - "9255:9255" From 2d86e2f12416a1d0815d412a14d7235ade818b88 Mon Sep 17 00:00:00 2001 From: Sahil Batra Date: Wed, 1 Jul 2026 18:13:18 -0400 Subject: [PATCH 2/2] Update observability plan docker-compose snippet to match actual config Co-Authored-By: Claude Opus 4.6 --- docs/observability_plan.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/observability_plan.md b/docs/observability_plan.md index 165e733..c81dbc7 100644 --- a/docs/observability_plan.md +++ b/docs/observability_plan.md @@ -702,16 +702,24 @@ services: - ./observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml # Bridge ALL GCP metrics into Prometheus (production monitoring path) + # NOTE: v0.19+ requires CLI flags, not env vars for metric prefixes stackdriver-exporter: image: prometheuscommunity/stackdriver-exporter:latest + command: + - --google.project-ids=${GCP_PROJECT_ID:-deeplearning-sahil} + - --monitoring.metrics-prefixes=dataflow.googleapis.com/job + - --monitoring.metrics-prefixes=custom.googleapis.com/dataflow + - --monitoring.metrics-prefixes=workload.googleapis.com + - --monitoring.metrics-prefixes=bigtable.googleapis.com/server + - --monitoring.metrics-prefixes=pubsub.googleapis.com + - --monitoring.metrics-prefixes=run.googleapis.com + - --monitoring.metrics-prefixes=bigquery.googleapis.com/storage + volumes: + - $HOME/.config/gcloud/application_default_credentials.json:/credentials/adc.json:ro environment: - - GOOGLE_APPLICATION_CREDENTIALS=/credentials/sa.json - - STACKDRIVER_EXPORTER_GOOGLE_PROJECT_ID=deeplearning-sahil - - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,custom.googleapis.com/dataflow,workload.googleapis.com,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com,bigquery.googleapis.com/storage + - GOOGLE_APPLICATION_CREDENTIALS=/credentials/adc.json ports: - "9255:9255" - volumes: - - ./deeplearning-sahil-e50332de6687.json:/credentials/sa.json ``` > **Prefix additions vs. original:**