diff --git a/Readme.md b/Readme.md
index 5440e4c..37bb08f 100644
--- a/Readme.md
+++ b/Readme.md
@@ -81,11 +81,11 @@ scripts/
 └── clean_reinstall.sh                  # Clean venv and reinstall
 docs/                                   # Design docs and plans
 observability/                          # Prometheus + Grafana monitoring stack
-├── otel-collector.yml                 # OTel Collector: receive OTLP, export to Prometheus
-├── prometheus.yml                     # Scrape config (OTel Collector + stackdriver-exporter)
+├── otel-collector.yml                 # OTel Collector: receive OTLP, export to Prometheus (local dev only)
+├── prometheus.yml                     # Scrape config (stackdriver-exporter + OTel Collector)
 ├── alert_rules.yml                    # Alert rules (error rates, latency, cost anomalies)
 ├── alertmanager.yml                   # Notification routing (Slack, PagerDuty)
-├── docker-compose.observability.yml   # Local dev stack (all observability services)
+├── docker-compose.observability.yml   # Observability stack (stackdriver-exporter bridges Cloud Monitoring)
 └── grafana/
     ├── provisioning/                  # Auto-configure datasources and dashboard loading
     └── dashboards/
@@ -303,15 +303,51 @@ Generate random Iris data and publish to Pub/Sub for testing streaming pipelines
 
 This can be run from any directory — the script resolves paths automatically.
 
-## Observability (Local Dev)
+## Observability
 
-The observability stack runs Prometheus, Grafana, OTel Collector, Alertmanager, and a stackdriver-exporter as Docker containers. It collects metrics from the FastAPI service (via OTel) and GCP platform metrics (via stackdriver-exporter).
+The observability stack monitors production Cloud Run and Dataflow services. All metrics flow through **Google Cloud Monitoring** and are bridged into a local Prometheus/Grafana stack via `stackdriver-exporter`. No direct network path from Cloud Run to local containers is needed.
+
+### Architecture
+
+```
+┌─── Google Cloud ───────────────────────────────────────────────┐
+│                                                                │
+│  FastAPI (Cloud Run) ──CloudMonitoringMetricsExporter──┐       │
+│  Dataflow (Beam)     ──auto-exported──────────────────┤       │
+│  GCP platform        ──auto-collected────────────────┐│       │
+│                                                      ││       │
+│                                              Cloud Monitoring  │
+│                                              (workload/,       │
+│                                               custom/dataflow, │
+│                                               dataflow/, run/, │
+│                                               pubsub/, etc.)   │
+└──────────────────────────────────────────────────┬─────────────┘
+                                                   │
+┌─── Local Docker Compose ─────────────────────────┼─────────────┐
+│                                                  ▼             │
+│                                     stackdriver-exporter       │
+│                                          :9255                 │
+│                                            │                   │
+│                                            ▼                   │
+│                    Prometheus :9090 ──▶ Grafana :3000           │
+│                         │                                      │
+│                         ▼                                      │
+│                    Alertmanager :9093                           │
+└────────────────────────────────────────────────────────────────┘
+```
+
+**FastAPI on Cloud Run** uses `CloudMonitoringMetricsExporter` by default — metrics land in Cloud Monitoring under `workload.googleapis.com/fastapi.*`. For local dev, set `OTEL_EXPORTER_OTLP_ENDPOINT` to fall back to OTLP push to a local OTel Collector (start it with `docker compose --profile local-dev up`).
 
 ### Start the Stack
 
 ```bash
 cd observability
+
+# Production monitoring (scrapes Cloud Monitoring via stackdriver-exporter)
 docker compose -f docker-compose.observability.yml up -d
+
+# Local dev (adds OTel Collector for locally-running FastAPI)
+docker compose -f docker-compose.observability.yml --profile local-dev up -d
 ```
 
 ### Access the UIs
@@ -329,10 +365,11 @@ Grafana auto-loads three dashboards on startup (no manual import needed):
 
 ### Verify It Works
 
-1. **Prometheus targets** — go to http://localhost:9090/targets and check that `otel-collector` and `stackdriver-exporter` targets are UP
+1. **Prometheus targets** — go to http://localhost:9090/targets and check that `stackdriver-exporter` is UP
 2. **Grafana datasource** — Prometheus is auto-configured as the default datasource; dashboards should show "No data" until metrics flow
-3. **FastAPI metrics** — start the FastAPI service with `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317` and send a `/predict` request; metrics appear in the Pipeline Health dashboard within ~15s
-4. **GCP metrics** — stackdriver-exporter requires GCP credentials; set `GCP_PROJECT_ID` env var and authenticate via `gcloud auth application-default login` or Workload Identity. Dataflow/Pub/Sub/Bigtable/Cloud Run metrics appear in the Cost Attribution dashboard
+3. **FastAPI metrics** — send a `/predict` request to the Cloud Run service; `workload_googleapis_com:fastapi_predictions_total` appears in Prometheus within ~2 minutes (Cloud Monitoring export interval + stackdriver-exporter scrape interval)
+4. **Beam metrics** — when Dataflow jobs are running, `custom_googleapis_com:dataflow_*` metrics (parse_success, prediction_latency, etc.) appear in Prometheus
+5. **GCP platform metrics** — stackdriver-exporter requires GCP credentials; set `GCP_PROJECT_ID` env var and authenticate via `gcloud auth application-default login` or Workload Identity. Dataflow/Pub/Sub/Bigtable/Cloud Run metrics appear in the Cost Attribution dashboard
 
 ### Stop the Stack
 
@@ -341,19 +378,6 @@ cd observability
 docker compose -f docker-compose.observability.yml down
 ```
 
-### Architecture
-
-```
-FastAPI (OTel SDK) ──OTLP gRPC──▶ OTel Collector ──scrape──▶ Prometheus ──▶ Grafana
-                                   (port 4317)      (port 8889)  (port 9090)  (port 3000)
-
-GCP Cloud Monitoring ◀──scrape── stackdriver-exporter ──scrape──▶ Prometheus
-(Dataflow, Pub/Sub,               (port 9255)
- Bigtable, Cloud Run)
-                                                        Prometheus ──▶ Alertmanager
-                                                                       (port 9093)
-```
-
 ### Alert Rules
 
 Alerts are defined in `observability/alert_rules.yml` and loaded by Prometheus:
@@ -503,7 +527,7 @@ The repository includes three GitHub Actions workflows:
 - **Data Validation**: Pydantic
 - **Data Processing**: Pandas, Polars, Dask
 - **Async HTTP**: aiohttp (micro-batch inference)
-- **Observability**: OpenTelemetry (instrumentation), Prometheus (metrics), Grafana (dashboards), Alertmanager (alerts), stackdriver-exporter (GCP metrics bridge)
+- **Observability**: OpenTelemetry (instrumentation), Cloud Monitoring (production metrics sink), Prometheus (metrics), Grafana (dashboards), Alertmanager (alerts), stackdriver-exporter (Cloud Monitoring → Prometheus bridge)
 - **Authentication**: Workload Identity, `google.auth.default()`
 - **Package Management**: uv, Hatchling
 - **CI/CD**: GitHub Actions (3 workflows: CI build + DAG sync, Dataflow inference deploy, Dataflow feature deploy)
@@ -575,7 +599,7 @@ Feature definitions live in `src/feature_store/` with a shared `FeatureConfig` c
 - **Scalable**: Dataflow auto-scales based on Pub/Sub message volume with Streaming Engine
 - **Reliable**: Blessed-model deployments, retry with backoff on online store reads, Pydantic message validation
 - **Consistent Features**: Single Feature Store serves training (offline/BQ), batch (offline/BQ), and real-time (online/Bigtable)
-- **Observable**: Prometheus + Grafana dashboards (pipeline health, dead letters, cost attribution), OTel instrumentation, Alertmanager alerts, structured JSON logging via Cloud Logging
+- **Observable**: All production metrics (FastAPI, Dataflow, GCP platform) flow through Cloud Monitoring → stackdriver-exporter → Prometheus → Grafana. OTel instrumentation, Alertmanager alerts, structured JSON logging via Cloud Logging
 
 ## Logging
 
diff --git a/docs/observability_plan.md b/docs/observability_plan.md
index 17fc727..165e733 100644
--- a/docs/observability_plan.md
+++ b/docs/observability_plan.md
@@ -41,37 +41,98 @@ Even though the backend is Prometheus, we instrument with **OpenTelemetry SDK**
 
 ### Decision
 
-Instrument with **OpenTelemetry SDK + auto-instrumentation**. Export metrics to **Prometheus** via the **OTel Collector's Prometheus exporter**. Visualize with **Grafana**. Re-evaluate SigNoz when Beam Python SDK gets OTel tracing support.
+Instrument with **OpenTelemetry SDK + auto-instrumentation**. Export metrics to **Google Cloud Monitoring** in production (via `CloudMonitoringMetricsExporter`), bridge into **Prometheus** via **stackdriver-exporter**. Visualize with **Grafana**. OTel Collector kept for local-only dev. Re-evaluate SigNoz when Beam Python SDK gets OTel tracing support.
 
 ---
 
 ## Architecture Overview
 
+### Production Architecture (Cloud Run + Dataflow → Cloud Monitoring → local Grafana)
+
+All production metrics flow through **one bridge**: Google Cloud Monitoring → stackdriver-exporter → Prometheus → Grafana. This avoids network boundary issues (Cloud Run can't reach a local/private OTel Collector).
+
 ```
-┌─────────────┐     ┌──────────────────┐     ┌─────────────┐
-│  Pub/Sub     │────▶│  Dataflow Jobs   │────▶│  BigQuery    │
-│  (messages)  │     │  (Beam workers)  │     │  (sink)      │
-└─────────────┘     └───────┬──────────┘     └─────────────┘
-                            │
-                    Beam Metrics → Cloud Monitoring
-                            │
-                            ▼
-┌─────────────┐     ┌──────────────────┐     ┌──────────────────┐     ┌─────────────┐
-│  FastAPI     │────▶│  OTel Collector  │────▶│  Prometheus      │────▶│  Grafana     │
-│  (OTel SDK) │     │  (receive +      │     │  (scrape + store)│     │  (dashboards)│
-└─────────────┘     │   export)        │     └───────┬──────────┘     └─────────────┘
-                    └──────┬───────────┘             │
-                           │                         ▼
-                    stackdriver-exporter      ┌──────────────────┐
-                    (GCP metrics bridge)      │  Alertmanager    │
-                                              │  (Slack/PD)      │
-                                              └──────────────────┘
+                     ┌──────────────────────────────────────────────────┐
+                     │               Google Cloud                       │
+                     │                                                  │
+  ┌───────────┐      │  ┌──────────────┐          ┌───────────┐        │
+  │ Pub/Sub   │─────────▶ Dataflow     │─────────▶│ BigQuery  │        │
+  └───────────┘      │  │ (Beam)       │          └───────────┘        │
+                     │  └──────┬───────┘                                │
+                     │         │ Beam Metrics                           │
+                     │         │ (auto-exported)                        │
+                     │         │                    ┌────────────────┐  │
+                     │  ┌──────────────┐            │                │  │
+                     │  │ FastAPI      │            │     Cloud      │  │
+                     │  │ (Cloud Run)  │            │   Monitoring   │  │
+                     │  └──────┬───────┘            │                │  │
+                     │         │ CloudMonitoring     │  ┌──────────┐ │  │
+                     │         │ MetricsExporter ───▶│  │workload/ │ │  │
+                     │         │                    │  │custom/df │ │  │
+                     │   Beam system metrics ──────▶│  │dataflow/ │ │  │
+                     │   GCP platform metrics ─────▶│  │run/pubsub│ │  │
+                     │   (auto-collected)           │  │bigtable/ │ │  │
+                     │                              │  └──────────┘ │  │
+                     │                              └───────┬────────┘  │
+                     └──────────────────────────────────────┼───────────┘
+                                                            │
+                               ALL metrics via one bridge   │
+                                                            │
+                     ┌──────────────────────────────────────┼───────────┐
+                     │        Local Docker Compose          │           │
+                     │                                      ▼           │
+                     │                       ┌────────────────────┐     │
+                     │                       │ stackdriver-       │     │
+                     │                       │ exporter :9255     │     │
+                     │                       │                    │     │
+                     │                       │ prefixes:          │     │
+                     │                       │  dataflow.../job   │     │
+                     │                       │  custom.../dataflow│     │
+                     │                       │  workload.google.. │     │
+                     │                       │  run.googleapis..  │     │
+                     │                       │  pubsub.google..   │     │
+                     │                       │  bigtable.google.. │     │
+                     │                       │  bigquery.google.. │     │
+                     │                       └────────┬───────────┘     │
+                     │                                │                 │
+                     │                                ▼                 │
+                     │                 ┌──────────────────────────┐     │
+                     │                 │   Prometheus :9090       │     │
+                     │                 └────────────┬─────────────┘     │
+                     │                              │                   │
+                     │                    ┌─────────┴─────────┐        │
+                     │                    ▼                   ▼        │
+                     │  ┌──────────────────────┐  ┌─────────────────┐  │
+                     │  │   Grafana :3000      │  │ Alertmanager    │  │
+                     │  │   (dashboards)       │  │ :9093           │  │
+                     │  └──────────────────────┘  └─────────────────┘  │
+                     └──────────────────────────────────────────────────┘
 ```
 
-**Collection patterns:**
-- **FastAPI service:** OTel SDK auto-instrumentation → OTel Collector → Prometheus (via Collector's Prometheus exporter)
-- **Dataflow workers:** Beam custom metrics → Cloud Monitoring → stackdriver-exporter → Prometheus
-- **GCP platform metrics:** stackdriver-exporter scrapes Cloud Monitoring for Dataflow/Pub/Sub/Cloud Run/Bigtable
+### Local Dev Architecture (optional — for testing without GCP)
+
+For local development, the OTel Collector is kept so a locally-running FastAPI can push metrics directly. This path is NOT used in production.
+
+```
+  ┌──────────────┐     ┌──────────────────┐     ┌──────────────────┐     ┌─────────────┐
+  │  FastAPI     │────▶│  OTel Collector  │────▶│  Prometheus      │────▶│  Grafana     │
+  │  (local)    │     │  :4317 → :8889   │     │  :9090           │     │  :3000       │
+  └──────────────┘     └──────────────────┘     └──────────────────┘     └─────────────┘
+```
+
+### Collection Patterns (production)
+
+| Source | Exporter | Cloud Monitoring Prefix | Notes |
+|---|---|---|---|
+| **FastAPI (Cloud Run)** | `CloudMonitoringMetricsExporter` | `workload.googleapis.com/fastapi.*` | Custom ML metrics (latency, batch size, predictions) |
+| **FastAPI (Cloud Run)** | Auto-collected by GCP | `run.googleapis.com/container/*` | Request count, latency, instance count |
+| **Dataflow (Beam custom)** | Auto-exported by Dataflow | `custom.googleapis.com/dataflow/*` | `parse_success`, `prediction_latency`, etc. |
+| **Dataflow (system)** | Auto-collected by GCP | `dataflow.googleapis.com/job/*` | Worker count, elapsed time, vCPUs |
+| **Pub/Sub** | Auto-collected by GCP | `pubsub.googleapis.com/*` | Backlog, publish rate, ack latency |
+| **Bigtable** | Auto-collected by GCP | `bigtable.googleapis.com/server/*` | Read/write latency, row count |
+| **BigQuery** | Auto-collected by GCP | `bigquery.googleapis.com/storage/*` | Bytes stored, streaming insert count |
+
+> **Why not OTel Collector in production?** The original plan had FastAPI push OTLP to a co-located OTel Collector. This works when both are on the same network (Docker, K8s), but fails when FastAPI is on Cloud Run — there's no OTel Collector to reach. Using `CloudMonitoringMetricsExporter` instead routes FastAPI metrics through the same Cloud Monitoring path that Beam and GCP platform metrics already use. One bridge (stackdriver-exporter), one scrape target, no network boundary issues.
 
 ---
 
@@ -104,28 +165,44 @@ OTel auto-instrumentation handles request latency, status codes, and trace conte
 automatically — no manual middleware needed. We add custom metrics only for
 ML-specific signals (prediction latency, batch size, model load time).
 
+The exporter is **environment-aware**: Cloud Run uses `CloudMonitoringMetricsExporter`
+(metrics land in Cloud Monitoring → stackdriver-exporter pulls them into Prometheus).
+Local dev uses `OTLPMetricExporter` (pushes directly to the OTel Collector in docker-compose).
+
 ```python
 # In fastapi_server.py
+import os
 from opentelemetry import metrics, trace
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
 import time
 
-# --- OTel setup (run once at startup, before app creation) ---
+resource = Resource.create({"service.name": "fastapi-inference"})
+
+# --- Environment-aware exporter ---
+# Cloud Run sets K_SERVICE automatically; use it to detect production.
+# Local dev: set OTEL_EXPORTER_OTLP_ENDPOINT to use OTel Collector instead.
+otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+
+if otel_endpoint:
+    # Local dev: push to OTel Collector (same Docker network)
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+    exporter = OTLPMetricExporter(endpoint=otel_endpoint, insecure=True)
+else:
+    # Production (Cloud Run): push to Google Cloud Monitoring
+    from opentelemetry.exporter.cloud_monitoring import CloudMonitoringMetricsExporter
+    exporter = CloudMonitoringMetricsExporter()
+
 metric_reader = PeriodicExportingMetricReader(
-    OTLPMetricExporter(endpoint="otel-collector:4317", insecure=True),
+    exporter,
     export_interval_millis=10_000,
 )
-metrics.set_meter_provider(MeterProvider(metric_readers=[metric_reader]))
-trace.set_tracer_provider(TracerProvider())
-
+metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[metric_reader]))
 meter = metrics.get_meter("fastapi-inference")
 
 # Auto-instruments all HTTP requests (latency, status, method, path)
-# Replaces the manual metrics_middleware and /metrics endpoint
 FastAPIInstrumentor.instrument_app(app)
 
 # Custom ML-specific metrics
@@ -142,13 +219,13 @@ batch_size_hist = meter.create_histogram(
     name="fastapi.predict.batch_size",
     description="Number of instances per /predict call",
 )
-model_load_time = meter.create_gauge(
+model_load_time = meter.create_histogram(
     name="fastapi.model.load_duration",
     description="Time taken to load the model at startup",
     unit="s",
 )
 
-# --- Usage in /predict endpoint ---
+# --- Usage in /predict endpoint (unchanged) ---
 @app.post("/predict", response_model=PredictionResponse)
 async def predict(request: PredictionRequest):
     if model is None:
@@ -175,10 +252,14 @@ async def predict(request: PredictionRequest):
         raise HTTPException(status_code=400, detail=f"Prediction failed: {str(e)}")
 ```
 
-> **No `/metrics` endpoint needed.** OTel SDK pushes metrics to the OTel Collector via OTLP gRPC.
-> The Collector then exposes them for Prometheus to scrape. This decouples the app from the
-> metrics backend — swap the Collector's exporter config to switch from Prometheus to SigNoz
-> without touching application code.
+> **Production path:** `CloudMonitoringMetricsExporter` pushes metrics to Cloud Monitoring
+> under the `workload.googleapis.com/` prefix. The stackdriver-exporter in docker-compose
+> pulls these into Prometheus. Metric names in Prometheus become
+> `workload_googleapis_com:fastapi_predict_duration`, etc.
+>
+> **Local dev path:** Setting `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317`
+> falls back to OTLP push to the local OTel Collector. Metric names in Prometheus are
+> `fastapi_fastapi_predict_duration` (via the Collector's `namespace: fastapi` config).
 
 ### Implementation — Dataflow (Beam Custom Metrics)
 
@@ -575,19 +656,25 @@ ORDER BY feature_fetch_latency_ms DESC;
 
 ## Infrastructure Setup
 
-### Deployment (Docker Compose for local dev, GKE for prod)
+### Deployment
+
+The docker-compose stack serves **production monitoring** — stackdriver-exporter bridges
+all Cloud Monitoring metrics into local Prometheus/Grafana. The OTel Collector is optional,
+only needed when running FastAPI locally for development.
 
 ```yaml
-# docker-compose.observability.yml (local development)
+# docker-compose.observability.yml
 services:
-  # Receives OTLP from FastAPI, exposes /metrics for Prometheus to scrape
+  # Optional: only needed for local dev (FastAPI running on host/docker)
+  # Not used for production monitoring — Cloud Run FastAPI pushes to Cloud Monitoring instead
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
     ports:
-      - "4317:4317"   # OTLP gRPC (FastAPI pushes here)
+      - "4317:4317"   # OTLP gRPC (local FastAPI pushes here)
       - "8889:8889"   # Prometheus exporter (Prometheus scrapes here)
     volumes:
       - ./observability/otel-collector.yml:/etc/otelcol-contrib/config.yaml
+    profiles: ["local-dev"]   # only starts with: docker compose --profile local-dev up
 
   prometheus:
     image: prom/prometheus:latest
@@ -614,19 +701,24 @@ services:
     volumes:
       - ./observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml
 
-  # Bridge GCP metrics into Prometheus
+  # Bridge ALL GCP metrics into Prometheus (production monitoring path)
   stackdriver-exporter:
     image: prometheuscommunity/stackdriver-exporter:latest
     environment:
       - GOOGLE_APPLICATION_CREDENTIALS=/credentials/sa.json
       - STACKDRIVER_EXPORTER_GOOGLE_PROJECT_ID=deeplearning-sahil
-      - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com
+      - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,custom.googleapis.com/dataflow,workload.googleapis.com,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com,bigquery.googleapis.com/storage
     ports:
       - "9255:9255"
     volumes:
       - ./deeplearning-sahil-e50332de6687.json:/credentials/sa.json
 ```
 
+> **Prefix additions vs. original:**
+> - `custom.googleapis.com/dataflow` — Beam custom metrics (`parse_success`, `prediction_latency_ms`, etc.)
+> - `workload.googleapis.com` — FastAPI OTel metrics pushed via `CloudMonitoringMetricsExporter`
+> - `bigquery.googleapis.com/storage` — was missing in original plan, added in Phase 5
+
 ### OTel Collector Config
 
 ```yaml
@@ -692,17 +784,21 @@ scrape_configs:
 
 ## Implementation Order
 
-### Phase 1: FastAPI Metrics (lowest effort, highest signal) -- DONE
+### Phase 1: FastAPI Metrics (lowest effort, highest signal) -- NEEDS FIX
 1. ~~Add OTel dependencies to `requirements.fastapi.txt`~~ (see Dependencies below)
 2. ~~Add OTel SDK setup + `FastAPIInstrumentor.instrument_app(app)` to `fastapi_server.py`~~
 3. ~~Add custom ML metrics: prediction latency, batch size, predictions total~~ (see Implementation above)
 4. ~~Deploy OTel Collector + Prometheus + Grafana (docker-compose locally)~~ — see `observability/`
 5. ~~Build Dashboard 1 (Pipeline Health)~~ — see `observability/grafana/dashboards/pipeline-health.json`
+6. **FIX:** Switch FastAPI exporter from `OTLPMetricExporter("otel-collector:4317")` to environment-aware exporter: `CloudMonitoringMetricsExporter` for production (Cloud Run), `OTLPMetricExporter` for local dev only. See updated Implementation section above.
+7. **FIX:** Add `opentelemetry-exporter-gcp-monitoring` to `requirements.fastapi.txt`
+8. **FIX:** Update Grafana dashboard queries from `fastapi_fastapi_*` to `workload_googleapis_com:fastapi_*` for production metric names
 
-### Phase 2: Beam / Dataflow Metrics -- DONE
+### Phase 2: Beam / Dataflow Metrics -- NEEDS FIX
 1. ~~Add `Metrics.counter()` and `Metrics.distribution()` to all DoFns~~
 2. ~~Deploy stackdriver-exporter to bridge Dataflow metrics → Prometheus~~
-3. Add feature fetch latency, error counters, write latency to dashboards (Phase 5 — Grafana panels)
+3. **FIX:** Add `custom.googleapis.com/dataflow` to stackdriver-exporter metric prefixes (currently only scrapes `dataflow.googleapis.com/job` system metrics, missing Beam custom counters)
+4. Add feature fetch latency, error counters, write latency to dashboards (Phase 5 — Grafana panels)
 
 ### Phase 3: Error Handling + Dead Letters -- DONE
 1. ~~Create `ml_dataset.dead_letters` BQ table~~ — run DDL manually in BQ console
@@ -737,13 +833,31 @@ Per-entity tracing and investigation deferred — aggregate Beam metrics (Phase
 opentelemetry-api>=1.25.0
 opentelemetry-sdk>=1.25.0
 opentelemetry-instrumentation-fastapi>=0.46b0
-opentelemetry-exporter-otlp-proto-grpc>=1.25.0
+opentelemetry-exporter-otlp-proto-grpc>=1.25.0       # local dev (OTel Collector)
+opentelemetry-exporter-gcp-monitoring>=1.9.0a0        # production (Cloud Monitoring)
 
 # pyproject.toml (add to beam/dataflow deps — Beam metrics are native, no OTel deps needed)
 # No additional deps for Phase 2 (Beam Metrics are built-in)
 # OTel deps only needed if/when Beam Python gets OTel tracing support (Phase 4+)
 ```
 
+## Grafana Metric Name Mapping
+
+Metrics coming through stackdriver-exporter have different Prometheus names than
+metrics coming through OTel Collector. Dashboards must use the production names.
+
+| Metric (OTel SDK name) | Via OTel Collector (local dev) | Via stackdriver-exporter (production) |
+|---|---|---|
+| `fastapi.predictions.total` | `fastapi_fastapi_predictions_total` | `workload_googleapis_com:fastapi_predictions_total` |
+| `fastapi.predict.duration` | `fastapi_fastapi_predict_duration` | `workload_googleapis_com:fastapi_predict_duration` |
+| `fastapi.predict.batch_size` | `fastapi_fastapi_predict_batch_size` | `workload_googleapis_com:fastapi_predict_batch_size` |
+| Beam: `parse_success` | N/A | `custom_googleapis_com:dataflow_ParsePubSubMessage_parse_success` |
+| Beam: `prediction_latency_ms` | N/A | `custom_googleapis_com:dataflow_BatchCallFastAPIService_prediction_latency_ms` |
+
+> **Dashboard strategy:** Build Grafana dashboards using the production (stackdriver-exporter)
+> metric names. For local dev with OTel Collector, either use a separate dashboard variant
+> or use Grafana variables to switch the metric prefix.
+
 ---
 
 ## File Structure
diff --git a/observability/docker-compose.observability.yml b/observability/docker-compose.observability.yml
index e01f9cc..607b3be 100644
--- a/observability/docker-compose.observability.yml
+++ b/observability/docker-compose.observability.yml
@@ -1,4 +1,7 @@
 services:
+  # Only needed for local dev (FastAPI running locally).
+  # Production monitoring uses stackdriver-exporter instead.
+  # Start with: docker compose --profile local-dev up
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
     ports:
@@ -6,6 +9,7 @@ services:
       - "8889:8889"
     volumes:
       - ./otel-collector.yml:/etc/otelcol-contrib/config.yaml
+    profiles: ["local-dev"]
 
   prometheus:
     image: prom/prometheus:latest
@@ -36,6 +40,6 @@ services:
     image: prometheuscommunity/stackdriver-exporter:latest
     environment:
       - STACKDRIVER_EXPORTER_GOOGLE_PROJECT_ID=${GCP_PROJECT_ID:-deeplearning-sahil}
-      - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com,bigquery.googleapis.com/storage
+      - STACKDRIVER_EXPORTER_MONITORING_METRICS_TYPE_PREFIXES=dataflow.googleapis.com/job,custom.googleapis.com/dataflow,workload.googleapis.com,bigtable.googleapis.com/server,pubsub.googleapis.com,run.googleapis.com,bigquery.googleapis.com/storage
     ports:
       - "9255:9255"
diff --git a/requirements.fastapi.txt b/requirements.fastapi.txt
index 12b44d7..6ae4c1c 100644
--- a/requirements.fastapi.txt
+++ b/requirements.fastapi.txt
@@ -12,4 +12,5 @@ requests>=2.31.0
 opentelemetry-api>=1.25.0
 opentelemetry-sdk>=1.25.0
 opentelemetry-instrumentation-fastapi>=0.46b0
-opentelemetry-exporter-otlp-proto-grpc>=1.25.0
\ No newline at end of file
+opentelemetry-exporter-otlp-proto-grpc>=1.25.0
+opentelemetry-exporter-gcp-monitoring>=1.9.0a0
\ No newline at end of file
diff --git a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/Dockerfile.fastapi b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/Dockerfile.fastapi
index 26e1427..83d05c8 100644
--- a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/Dockerfile.fastapi
+++ b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/Dockerfile.fastapi
@@ -29,7 +29,10 @@ USER appuser
 ENV MODEL_PATH=/app/models/model.joblib
 ENV PORT=8080
 ENV PYTHONPATH=/app
-ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+# When OTEL_EXPORTER_OTLP_ENDPOINT is unset, FastAPI uses CloudMonitoringMetricsExporter
+# (production path: metrics → Cloud Monitoring → stackdriver-exporter → Prometheus).
+# Set this env var only for local dev (e.g. docker-compose with OTel Collector).
+# ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
 
 EXPOSE 8080
 
diff --git a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/fastapi_server.py b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/fastapi_server.py
index 5b70f89..0f68d7c 100644
--- a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/fastapi_server.py
+++ b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/fastapi_server.py
@@ -13,8 +13,9 @@
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.cloud_monitoring import CloudMonitoringMetricsExporter
 
 from models.instance import Instance
 from models.prediction import Prediction
@@ -23,12 +24,19 @@
 logger = get_logger(__name__)
 logger.setLevel(os.getenv("LOG_LEVEL", "INFO"))
 
-# --- OTel metrics setup ---
-otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
+# --- OTel metrics setup (environment-aware exporter) ---
 resource = Resource.create({"service.name": "fastapi-inference"})
+otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+
+if otel_endpoint:
+    exporter = OTLPMetricExporter(endpoint=otel_endpoint, insecure=True)
+    logger.info(f"Using OTLP exporter → {otel_endpoint}")
+else:
+    exporter = CloudMonitoringMetricsExporter()
+    logger.info("Using Cloud Monitoring exporter")
 
 metric_reader = PeriodicExportingMetricReader(
-    OTLPMetricExporter(endpoint=otel_endpoint, insecure=True),
+    exporter,
     export_interval_millis=10_000,
 )
 metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[metric_reader]))
diff --git a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/requirements.fastapi.txt b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/requirements.fastapi.txt
index 12b44d7..6ae4c1c 100644
--- a/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/requirements.fastapi.txt
+++ b/src/ml_pipelines_kfp/iris_xgboost/pipelines/components/fastapi/requirements.fastapi.txt
@@ -12,4 +12,5 @@ requests>=2.31.0
 opentelemetry-api>=1.25.0
 opentelemetry-sdk>=1.25.0
 opentelemetry-instrumentation-fastapi>=0.46b0
-opentelemetry-exporter-otlp-proto-grpc>=1.25.0
\ No newline at end of file
+opentelemetry-exporter-otlp-proto-grpc>=1.25.0
+opentelemetry-exporter-gcp-monitoring>=1.9.0a0
\ No newline at end of file