From a8388e68807b6e6680570917305b6edee5d33650 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Fri, 13 Nov 2026 22:23:08 +0800 Subject: [PATCH] Map the gateway's retries, timeouts, resets and ejections The collector scraped Envoy's upstream retry, timeout, reset, ejection and response-class counters but filter/modelplane dropped them because nothing renamed them. Keep the inference gateway's copies (the cluster-gateways also report httproute///rule/0, so require gateway=inference-gateway) with namespace and service, and map them to modelplane_gateway_{retries,timeouts,upstream_resets,ejections, upstream_responses}_total. Also map the gateway listeners' downstream response classes to modelplane_gateway_requests_total. Timeouts and resets fold several Envoy counters onto one name, so they carry a kind label to keep the series apart. Fixes #536. Signed-off-by: ChethanUK --- docs/content/models/model-service.md | 6 +- docs/content/platform/telemetry.md | 14 +- .../function/collector.py | 56 ++- .../function/stacks/metrics.py | 41 ++- .../tests/test_collector.py | 345 +++++++++++++++++- .../compose-serving-stack/tests/test_fn.py | 65 +++- 6 files changed, 480 insertions(+), 47 deletions(-) diff --git a/docs/content/models/model-service.md b/docs/content/models/model-service.md index 8f1d336d3..377c2c0b8 100644 --- a/docs/content/models/model-service.md +++ b/docs/content/models/model-service.md @@ -185,8 +185,10 @@ Tune both from what the gateway measures, as first byte, and `modelplane_frontend_request_duration_seconds` how long whole requests take. The gateway's request duration buckets stop at 81.92 seconds, so for longer requests read the engine's `modelplane_request_duration_seconds` -instead. Modelplane doesn't yet export how often the gateway times out; see -[#536](https://github.com/modelplaneai/modelplane/issues/536). A restarting +instead. `modelplane_gateway_timeouts_total` counts the gateway's timeouts per +service, by `kind`: `request` for `request`, and `per_try_idle` for `idle`. A +stream cut short by `idle` still reaches the caller as a 200, so only +`per_try_idle` records it. A restarting gateway lets requests already in flight run for five minutes, so a restart can cut off a response allowed longer than that. diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index a92d2ec34..0c4a0444b 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -38,7 +38,13 @@ sent the request to, `/`. has a `service` label, and the same `namespace` label as an engine's series: the team's namespace as it's mirrored on the inference cluster, such as `mp-ml-team-51733`. The namespace in `gen_ai_original_model` is the team's own, -such as `ml-team`, so the two don't join on it. +such as `ml-team`, so the two don't join on it. The gateway's +`modelplane_gateway_*` counters of retries, timeouts, resets, ejections and +upstream responses have the same `namespace` and `service` labels. Every +deployment behind a service is one upstream to the gateway, so these count per +service, not per deployment. +`modelplane_gateway_requests_total` has neither, because it's counted per +listener, before the gateway picks a route. Each replica publishes its own series, so combine them in your query. Which combination is right follows from what the metric measures: @@ -83,6 +89,12 @@ Some examples of the available metrics: | `modelplane_frontend_tpot_seconds` | Time per output token, measured at the gateway. Streaming requests only. | | `modelplane_frontend_request_duration_seconds` | What the caller waited, end to end | | `modelplane_frontend_requests_active` | Requests open at the gateway, per `ModelService` | +| `modelplane_gateway_requests_total` | Requests the fleet gateway answered, by `status_class` (`1xx` to `5xx`), including 401, 404 and 429 responses no model served | +| `modelplane_gateway_upstream_responses_total` | Responses per `ModelService` by `status_class`, counted once per request by its final code, including the gateway's own 503 and 504 | +| `modelplane_gateway_retries_total` | Retries the gateway sent, per `ModelService` | +| `modelplane_gateway_timeouts_total` | Timeouts per `ModelService`, by `kind`: `request` counts requests, `per_try` and `per_try_idle` count attempts | +| `modelplane_gateway_upstream_resets_total` | Upstream resets per `ModelService`, by `kind`: `stream_reset` counts requests, `connection_closed` counts connections closed with requests in flight | +| `modelplane_gateway_ejections_total` | Times the gateway took an endpoint out of rotation after consecutive 5xx, per `ModelService` | | `modelplane_request_queue_seconds` | How long a request waited before the engine started | | `modelplane_requests_waiting` | Queue depth per engine | | `modelplane_kv_cache_utilization_ratio` | KV-cache occupancy, per replica | diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index ab55e97e1..d7ef9967c 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -68,6 +68,31 @@ _IN_FLIGHT = "envoy_cluster_upstream_rq_active" _FLEET_GATEWAY = "inference-gateway" +# Envoy's per-cluster counts of what happened to the requests it sent: retries, +# timeouts, resets, ejections and responses by class. Each cluster gateway +# reports its own httproute///rule/0 for the same requests, so +# these are kept and attributed exactly as the requests in flight are. +_UPSTREAM_SERIES = ( + _IN_FLIGHT, + "envoy_cluster_upstream_rq_retry", + "envoy_cluster_upstream_rq_timeout", + "envoy_cluster_upstream_rq_per_try_timeout", + "envoy_cluster_upstream_rq_per_try_idle_timeout", + "envoy_cluster_upstream_rq_rx_reset", + "envoy_cluster_upstream_cx_destroy_remote_with_active_rq", + "envoy_cluster_outlier_detection_ejections_enforced_total", + "envoy_cluster_upstream_rq_xx", +) +_UPSTREAM = f"(?:{'|'.join(_UPSTREAM_SERIES)})" + +# What the fleet gateway answered callers, per listener, which also counts the +# 401s, 404s and 429s that never reach a model. A cluster gateway counts the +# fleet gateway's forwarded requests a second time, and Envoy Gateway's own +# listeners, such as eg-stats-http, count scrapes and probes, so only the +# fleet gateway's Gateway listeners, named http- or https-, are kept. +_DOWNSTREAM = "envoy_http_downstream_rq_xx" +_LISTENER = r"https?-\d+" + # The cluster Envoy builds for a ModelService: compose-model-route's # AIGatewayRoute, named for the service in the team's mirrored namespace, and # rule 0, the one rule compose_route gives it. The namespace is the one an @@ -193,7 +218,10 @@ def _annotated_port() -> list[dict[str, Any]]: def _in_flight_relabels() -> list[dict[str, Any]]: - """Keep the fleet gateway's in-flight requests to each model, as namespace and service. + """Keep the fleet gateway's counts for each model, as namespace and service. + + Its requests in flight and the other _UPSTREAM series, and its _DOWNSTREAM + responses, which aren't per model and keep their listener. Prometheus relabeling rather than a transform statement, because it runs as the series are scraped, and the two drops are on labels a mapping can't @@ -207,29 +235,33 @@ def _in_flight_relabels() -> list[dict[str, Any]]: return [ { "source_labels": ["__name__", "envoy_cluster_name", "gateway"], - "regex": f"{_IN_FLIGHT};{_MODEL_ROUTE};{_FLEET_GATEWAY}", + "regex": f"{_UPSTREAM};{_MODEL_ROUTE};{_FLEET_GATEWAY}", "target_label": "__tmp_model", "replacement": "true", }, - {"source_labels": ["__name__", "__tmp_model"], "regex": f"{_IN_FLIGHT};", "action": "drop"}, + { + "source_labels": ["__name__", "envoy_http_conn_manager_prefix", "gateway"], + "regex": f"{_DOWNSTREAM};{_LISTENER};{_FLEET_GATEWAY}", + "target_label": "__tmp_model", + "replacement": "true", + }, + {"source_labels": ["__name__", "__tmp_model"], "regex": f"(?:{_UPSTREAM}|{_DOWNSTREAM});", "action": "drop"}, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": f"{_IN_FLIGHT};{_MODEL_ROUTE}", + "regex": f"{_UPSTREAM};{_MODEL_ROUTE}", "target_label": "namespace", "replacement": "$1", }, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": f"{_IN_FLIGHT};{_MODEL_ROUTE}", + "regex": f"{_UPSTREAM};{_MODEL_ROUTE}", "target_label": "service", "replacement": "$2", }, # An empty label is no label, so these leave the series with only the # identity just set. - *( - {"source_labels": ["__name__"], "regex": _IN_FLIGHT, "target_label": label, "replacement": ""} - for label in ("envoy_cluster_name", "__tmp_model") - ), + {"source_labels": ["__name__"], "regex": _UPSTREAM, "target_label": "envoy_cluster_name", "replacement": ""}, + {"target_label": "__tmp_model", "replacement": ""}, # The gateway is a target label, so it's on every series from the job, # and only the rules above read it. {"target_label": "gateway", "replacement": ""}, @@ -403,11 +435,13 @@ def _scrape_configs() -> list[dict[str, Any]]: *_annotated_port(), {"source_labels": ["__meta_kubernetes_namespace"], "target_label": "namespace"}, ], - # Another Envoy's count of requests in flight would otherwise leave + # Another Envoy's counts of requests would otherwise leave # under the gateway's name, with its own cluster names and none of # the gateway job's drops. An Istio sidecar publishes Envoy's # statistics, and annotates its pod for scraping. - "metric_relabel_configs": [{"source_labels": ["__name__"], "regex": _IN_FLIGHT, "action": "drop"}], + "metric_relabel_configs": [ + {"source_labels": ["__name__"], "regex": f"(?:{_UPSTREAM}|{_DOWNSTREAM})", "action": "drop"} + ], }, ] diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index ce2bfccdd..7d1f84362 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -59,8 +59,47 @@ # it, for minutes, is work no caller is waiting for. _GATEWAY_PROXY = { "envoy_cluster_upstream_rq_active": "modelplane_frontend_requests_active", + "envoy_cluster_upstream_rq_retry": "modelplane_gateway_retries_total", + "envoy_cluster_outlier_detection_ejections_enforced_total": "modelplane_gateway_ejections_total", } +# The fleet gateway's timeouts, resets and responses, kept per model like the +# requests in flight (see collector._UPSTREAM), and its responses to callers per +# listener (collector._DOWNSTREAM). Folded sources keep a label apart, or two +# counters for one service would land on one series and one would replace the +# other. A timeout's kind tells a request's timeout from an attempt's: Envoy +# counts per_try and per_try_idle once per attempt, timeout once per request. +# Envoy labels a response by the first digit of its code. +_RESPONSE_CLASSES = {str(n): f"{n}xx" for n in range(1, 6)} +_STATUS_CLASS = {"name": "status_class", "from": "envoy_response_code_class", "values": _RESPONSE_CLASSES} +_GATEWAY_PROXY_LABELLED: tuple[dict[str, object], ...] = ( + *( + {"from": src, "to": "modelplane_gateway_timeouts_total", "labels": [{"name": "kind", "value": kind}]} + for src, kind in ( + ("envoy_cluster_upstream_rq_timeout", "request"), + ("envoy_cluster_upstream_rq_per_try_timeout", "per_try"), + ("envoy_cluster_upstream_rq_per_try_idle_timeout", "per_try_idle"), + ) + ), + *( + {"from": src, "to": "modelplane_gateway_upstream_resets_total", "labels": [{"name": "kind", "value": kind}]} + for src, kind in ( + ("envoy_cluster_upstream_rq_rx_reset", "stream_reset"), + ("envoy_cluster_upstream_cx_destroy_remote_with_active_rq", "connection_closed"), + ) + ), + { + "from": "envoy_cluster_upstream_rq_xx", + "to": "modelplane_gateway_upstream_responses_total", + "labels": [_STATUS_CLASS], + }, + { + "from": "envoy_http_downstream_rq_xx", + "to": "modelplane_gateway_requests_total", + "labels": [_STATUS_CLASS, {"name": "listener", "from": "envoy_http_conn_manager_prefix"}], + }, +) + # The engines, which explain what the gateway measured. vLLM publishes these # names from v0.9.2. _VLLM = { @@ -264,7 +303,7 @@ def mappings() -> list[v1alpha1.MetricMapping]: """ return [ _mapping("modelplane-gateway", _GATEWAY), - _mapping("modelplane-gateway-proxy", _GATEWAY_PROXY), + _mapping("modelplane-gateway-proxy", _GATEWAY_PROXY, extra=_GATEWAY_PROXY_LABELLED), _mapping("modelplane-vllm", _VLLM, tokens=_VLLM_TOKENS, extra=(_VLLM_RESPONSES,)), _mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)), _mapping("modelplane-picker", _PICKER), diff --git a/functions/compose-serving-stack/tests/test_collector.py b/functions/compose-serving-stack/tests/test_collector.py index 3583f936d..89b5a564c 100644 --- a/functions/compose-serving-stack/tests/test_collector.py +++ b/functions/compose-serving-stack/tests/test_collector.py @@ -251,39 +251,40 @@ def _config( "metric_relabel_configs": [ { "source_labels": ["__name__", "envoy_cluster_name", "gateway"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0;inference-gateway", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0;inference-gateway", + "target_label": "__tmp_model", + "replacement": "true", + }, + { + "source_labels": ["__name__", "envoy_http_conn_manager_prefix", "gateway"], + "regex": "envoy_http_downstream_rq_xx;https?-\\d+;inference-gateway", "target_label": "__tmp_model", "replacement": "true", }, { "source_labels": ["__name__", "__tmp_model"], - "regex": "envoy_cluster_upstream_rq_active;", + "regex": "(?:(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)|envoy_http_downstream_rq_xx);", "action": "drop", }, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0", "target_label": "namespace", "replacement": "$1", }, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0", "target_label": "service", "replacement": "$2", }, { "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)", "target_label": "envoy_cluster_name", "replacement": "", }, - { - "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", - "target_label": "__tmp_model", - "replacement": "", - }, + {"target_label": "__tmp_model", "replacement": ""}, {"target_label": "gateway", "replacement": ""}, ], }, @@ -404,7 +405,7 @@ def _config( "metric_relabel_configs": [ { "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", + "regex": "(?:(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)|envoy_http_downstream_rq_xx)", "action": "drop", }, ], @@ -480,6 +481,27 @@ def _config( { "context": "datapoint", "statements": [ + 'set(datapoint.attributes["kind"], "request") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(datapoint.attributes["kind"], "per_try") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(datapoint.attributes["kind"], "per_try_idle") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(datapoint.attributes["kind"], "stream_reset") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(datapoint.attributes["kind"], "connection_closed") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["listener"], datapoint.attributes["envoy_http_conn_manager_prefix"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'delete_key(datapoint.attributes, "envoy_http_conn_manager_prefix") where metric.name == "envoy_http_downstream_rq_xx"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -504,6 +526,15 @@ def _config( 'set(metric.name, "modelplane_frontend_ttft_seconds") where metric.name == "gen_ai_server_time_to_first_token_seconds"', 'set(metric.name, "modelplane_frontend_tpot_seconds") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(metric.name, "modelplane_frontend_requests_active") where metric.name == "envoy_cluster_upstream_rq_active"', + 'set(metric.name, "modelplane_gateway_retries_total") where metric.name == "envoy_cluster_upstream_rq_retry"', + 'set(metric.name, "modelplane_gateway_ejections_total") where metric.name == "envoy_cluster_outlier_detection_ejections_enforced_total"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(metric.name, "modelplane_gateway_upstream_responses_total") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(metric.name, "modelplane_gateway_requests_total") where metric.name == "envoy_http_downstream_rq_xx"', 'set(metric.name, "modelplane_request_ttft_seconds") where metric.name == "vllm:time_to_first_token_seconds"', 'set(metric.name, "modelplane_request_duration_seconds") where metric.name == "vllm:e2e_request_latency_seconds"', 'set(metric.name, "modelplane_request_queue_seconds") where metric.name == "vllm:request_queue_time_seconds"', @@ -948,6 +979,260 @@ def test_in_flight(case: InFlightCase) -> None: assert got == case.want, case.reason +GATEWAY_COUNTERS_CASES = [ + InFlightCase( + name="Retries", + reason="The fleet gateway's retries for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_retry", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_retry", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="RequestTimeouts", + reason="The fleet gateway's request timeouts for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_timeout", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_timeout", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="PerTryTimeouts", + reason="The fleet gateway's per-try timeouts for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_per_try_timeout", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_per_try_timeout", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="PerTryIdleTimeouts", + reason="The fleet gateway's per-try idle timeouts for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_per_try_idle_timeout", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_per_try_idle_timeout", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="StreamResets", + reason="The fleet gateway's stream resets for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_rx_reset", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_rx_reset", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="ConnectionsClosed", + reason="The fleet gateway's connections closed with requests in flight for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_cx_destroy_remote_with_active_rq", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_cx_destroy_remote_with_active_rq", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="Ejections", + reason="The fleet gateway's ejections for a ModelService's route are kept, with only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_outlier_detection_ejections_enforced_total", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_outlier_detection_ejections_enforced_total", + "namespace": "mp-ml-team-51733", + "service": "qwen", + }, + ), + InFlightCase( + name="UpstreamResponses", + reason="The fleet gateway's upstream responses for a ModelService's route are kept with their response class, and only its namespace and service.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_xx", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen/rule/0", + "envoy_response_code_class": "5", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_cluster_upstream_rq_xx", + "namespace": "mp-ml-team-51733", + "service": "qwen", + "envoy_response_code_class": "5", + }, + ), + InFlightCase( + name="ClusterGatewayCounter", + reason="A cluster gateway's duplicate httproute///rule/0 cluster is dropped, so a sum counts each retry once.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_retry", + "envoy_cluster_name": "httproute/mp-ml-team-51733/qwen-34d80/rule/0", + "gateway": "cluster-gateway", + }, + want=None, + ), + InFlightCase( + name="NotAModelCounter", + reason="A counter for a route in modelplane-system, such as Grafana's, isn't a model's and is dropped.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_retry", + "envoy_cluster_name": "httproute/modelplane-system/grafana/rule/0", + "gateway": "inference-gateway", + }, + want=None, + ), + InFlightCase( + name="EnvoyClusterCounter", + reason="A counter for a cluster Envoy builds for itself is dropped.", + job="modelplane-gateway", + series={ + "__name__": "envoy_cluster_upstream_rq_retry", + "envoy_cluster_name": "xds_cluster", + "gateway": "inference-gateway", + }, + want=None, + ), + InFlightCase( + name="FleetDownstreamHTTP", + reason="The fleet gateway's responses on its HTTP listener are kept, without the gateway target label.", + job="modelplane-gateway", + series={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "http-10080", + "envoy_response_code_class": "4", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "http-10080", + "envoy_response_code_class": "4", + }, + ), + InFlightCase( + name="FleetDownstreamHTTPS", + reason="The fleet gateway's responses on its HTTPS listener are kept, without the gateway target label.", + job="modelplane-gateway", + series={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "https-10443", + "envoy_response_code_class": "4", + "gateway": "inference-gateway", + }, + want={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "https-10443", + "envoy_response_code_class": "4", + }, + ), + InFlightCase( + name="ClusterGatewayDownstream", + reason="A cluster gateway's responses are dropped, because they count the fleet gateway's forwarded requests a second time.", + job="modelplane-gateway", + series={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "http-10080", + "envoy_response_code_class": "4", + "gateway": "cluster-gateway", + }, + want=None, + ), + InFlightCase( + name="StatsListener", + reason="Responses on Envoy Gateway's own stats listener are scrapes, not model requests, and are dropped.", + job="modelplane-gateway", + series={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "eg-stats-http", + "envoy_response_code_class": "2", + "gateway": "inference-gateway", + }, + want=None, + ), + InFlightCase( + name="SubstrateClusterCounter", + reason="Another Envoy's retries, such as an Istio sidecar's, are dropped rather than mapped as the gateway's.", + job="modelplane-substrate", + series={ + "__name__": "envoy_cluster_upstream_rq_retry", + "envoy_cluster_name": "outbound|8080||reviews.default.svc.cluster.local", + "namespace": "default", + }, + want=None, + ), + InFlightCase( + name="SubstrateDownstream", + reason="Another Envoy's responses, such as an Istio sidecar's, are dropped rather than mapped as the gateway's.", + job="modelplane-substrate", + series={ + "__name__": "envoy_http_downstream_rq_xx", + "envoy_http_conn_manager_prefix": "inbound_0.0.0.0_8080", + "envoy_response_code_class": "2", + "namespace": "default", + }, + want=None, + ), +] + + +@pytest.mark.parametrize("case", GATEWAY_COUNTERS_CASES, ids=lambda case: case.name) +def test_gateway_counters(case: InFlightCase) -> None: + """Only the fleet gateway's retries, timeouts, ejections and responses are kept, labelled by model where they have one.""" + config = yaml.safe_load( + collector.config( + "prod-us-east", + list(stacks.BUILTIN_MAPPINGS), + [_sink(name="primary", type_="otlphttp", secret=None)], + {"oauth2client/acme": {"token_url": "https://issuer.acme.example/token"}}, + ) + ) + job = next(j for j in config["receivers"]["prometheus"]["config"]["scrape_configs"] if j["job_name"] == case.job) + got = _metric_relabel(rules=job["metric_relabel_configs"], labels=case.series) + assert got == case.want, case.reason + + STATEMENTS_CASES = [ # The built-in mappings, read from the stacks package rather than restated, # because what the collector compiles them to is what this case pins. @@ -977,6 +1262,27 @@ def test_in_flight(case: InFlightCase) -> None: 'convert_gauge_to_sum("cumulative", true) where metric.name == "gpu_energy_consumed"', ], [ + 'set(datapoint.attributes["kind"], "request") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(datapoint.attributes["kind"], "per_try") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(datapoint.attributes["kind"], "per_try_idle") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(datapoint.attributes["kind"], "stream_reset") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(datapoint.attributes["kind"], "connection_closed") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["listener"], datapoint.attributes["envoy_http_conn_manager_prefix"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'delete_key(datapoint.attributes, "envoy_http_conn_manager_prefix") where metric.name == "envoy_http_downstream_rq_xx"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -998,6 +1304,15 @@ def test_in_flight(case: InFlightCase) -> None: 'set(metric.name, "modelplane_frontend_ttft_seconds") where metric.name == "gen_ai_server_time_to_first_token_seconds"', 'set(metric.name, "modelplane_frontend_tpot_seconds") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(metric.name, "modelplane_frontend_requests_active") where metric.name == "envoy_cluster_upstream_rq_active"', + 'set(metric.name, "modelplane_gateway_retries_total") where metric.name == "envoy_cluster_upstream_rq_retry"', + 'set(metric.name, "modelplane_gateway_ejections_total") where metric.name == "envoy_cluster_outlier_detection_ejections_enforced_total"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(metric.name, "modelplane_gateway_upstream_responses_total") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(metric.name, "modelplane_gateway_requests_total") where metric.name == "envoy_http_downstream_rq_xx"', 'set(metric.name, "modelplane_request_ttft_seconds") where metric.name == "vllm:time_to_first_token_seconds"', 'set(metric.name, "modelplane_request_duration_seconds") where metric.name == "vllm:e2e_request_latency_seconds"', 'set(metric.name, "modelplane_request_queue_seconds") where metric.name == "vllm:request_queue_time_seconds"', @@ -1245,7 +1560,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="522e31c7cedc07c5", + config_hash="b432ac64502089aa", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -1290,7 +1605,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="1f03e323432223be", + config_hash="9e9c9afeae7deb9e", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -1357,7 +1672,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="b1539f5a25d85bfc", + config_hash="ac31d3b5c67018be", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}}, diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index d3e90092a..501e0ba4b 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -1496,39 +1496,40 @@ def _collector_config( "metric_relabel_configs": [ { "source_labels": ["__name__", "envoy_cluster_name", "gateway"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0;inference-gateway", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0;inference-gateway", + "target_label": "__tmp_model", + "replacement": "true", + }, + { + "source_labels": ["__name__", "envoy_http_conn_manager_prefix", "gateway"], + "regex": "envoy_http_downstream_rq_xx;https?-\\d+;inference-gateway", "target_label": "__tmp_model", "replacement": "true", }, { "source_labels": ["__name__", "__tmp_model"], - "regex": "envoy_cluster_upstream_rq_active;", + "regex": "(?:(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)|envoy_http_downstream_rq_xx);", "action": "drop", }, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0", "target_label": "namespace", "replacement": "$1", }, { "source_labels": ["__name__", "envoy_cluster_name"], - "regex": "envoy_cluster_upstream_rq_active;httproute/(mp-[^/]+)/([^/]+)/rule/0", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx);httproute/(mp-[^/]+)/([^/]+)/rule/0", "target_label": "service", "replacement": "$2", }, { "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", + "regex": "(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)", "target_label": "envoy_cluster_name", "replacement": "", }, - { - "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", - "target_label": "__tmp_model", - "replacement": "", - }, + {"target_label": "__tmp_model", "replacement": ""}, {"target_label": "gateway", "replacement": ""}, ], }, @@ -1638,7 +1639,7 @@ def _collector_config( "metric_relabel_configs": [ { "source_labels": ["__name__"], - "regex": "envoy_cluster_upstream_rq_active", + "regex": "(?:(?:envoy_cluster_upstream_rq_active|envoy_cluster_upstream_rq_retry|envoy_cluster_upstream_rq_timeout|envoy_cluster_upstream_rq_per_try_timeout|envoy_cluster_upstream_rq_per_try_idle_timeout|envoy_cluster_upstream_rq_rx_reset|envoy_cluster_upstream_cx_destroy_remote_with_active_rq|envoy_cluster_outlier_detection_ejections_enforced_total|envoy_cluster_upstream_rq_xx)|envoy_http_downstream_rq_xx)", "action": "drop", }, ], @@ -1694,6 +1695,27 @@ def _collector_config( { "context": "datapoint", "statements": [ + 'set(datapoint.attributes["kind"], "request") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(datapoint.attributes["kind"], "per_try") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(datapoint.attributes["kind"], "per_try_idle") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(datapoint.attributes["kind"], "stream_reset") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(datapoint.attributes["kind"], "connection_closed") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_cluster_upstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(datapoint.attributes["status_class"], datapoint.attributes["envoy_response_code_class"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["status_class"], "1xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "1"', + 'set(datapoint.attributes["status_class"], "2xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "2"', + 'set(datapoint.attributes["status_class"], "3xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "3"', + 'set(datapoint.attributes["status_class"], "4xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "4"', + 'set(datapoint.attributes["status_class"], "5xx") where metric.name == "envoy_http_downstream_rq_xx" and datapoint.attributes["envoy_response_code_class"] == "5"', + 'delete_key(datapoint.attributes, "envoy_response_code_class") where metric.name == "envoy_http_downstream_rq_xx"', + 'set(datapoint.attributes["listener"], datapoint.attributes["envoy_http_conn_manager_prefix"]) where metric.name == "envoy_http_downstream_rq_xx"', + 'delete_key(datapoint.attributes, "envoy_http_conn_manager_prefix") where metric.name == "envoy_http_downstream_rq_xx"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -1718,6 +1740,15 @@ def _collector_config( 'set(metric.name, "modelplane_frontend_ttft_seconds") where metric.name == "gen_ai_server_time_to_first_token_seconds"', 'set(metric.name, "modelplane_frontend_tpot_seconds") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(metric.name, "modelplane_frontend_requests_active") where metric.name == "envoy_cluster_upstream_rq_active"', + 'set(metric.name, "modelplane_gateway_retries_total") where metric.name == "envoy_cluster_upstream_rq_retry"', + 'set(metric.name, "modelplane_gateway_ejections_total") where metric.name == "envoy_cluster_outlier_detection_ejections_enforced_total"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_timeout"', + 'set(metric.name, "modelplane_gateway_timeouts_total") where metric.name == "envoy_cluster_upstream_rq_per_try_idle_timeout"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_rq_rx_reset"', + 'set(metric.name, "modelplane_gateway_upstream_resets_total") where metric.name == "envoy_cluster_upstream_cx_destroy_remote_with_active_rq"', + 'set(metric.name, "modelplane_gateway_upstream_responses_total") where metric.name == "envoy_cluster_upstream_rq_xx"', + 'set(metric.name, "modelplane_gateway_requests_total") where metric.name == "envoy_http_downstream_rq_xx"', 'set(metric.name, "modelplane_request_ttft_seconds") where metric.name == "vllm:time_to_first_token_seconds"', 'set(metric.name, "modelplane_request_duration_seconds") where metric.name == "vllm:e2e_request_latency_seconds"', 'set(metric.name, "modelplane_request_queue_seconds") where metric.name == "vllm:request_queue_time_seconds"', @@ -4746,7 +4777,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="be9842417e13d6c1", + config_hash="583ec4a226fbe8c2", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5065,7 +5096,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "collector": _collector( - config_hash="5d3c556be06d8ca8", + config_hash="fab7a9fe7c49f382", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -5352,7 +5383,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="0a72aaee56455e72", + config_hash="53a2a2b185eb20c8", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5615,7 +5646,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="57e31b8f680bd6d3", + config_hash="ec1ba720115337cc", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5906,7 +5937,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="be9842417e13d6c1", + config_hash="583ec4a226fbe8c2", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None,