From baba04d1414afe43e2ad809765d03fb56501c3bd Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Fri, 13 Nov 2026 22:19:54 +0800 Subject: [PATCH] Label the gateway's latency series by service and served model The modelplane_frontend_* latency series had no Modelplane label for the model a request was for, only Envoy AI Gateway's gen_ai_original_model and gen_ai_request_model. Grouping by deployment or service meant knowing the gateway's attribute names, and the telemetry docs' TTFT example used them. The modelplane-gateway mapping now renames gen_ai_original_model to service and gen_ai_request_model to served_model, through the labels field the other built-ins use. served_model stays whole: an external endpoint's model name, such as Qwen/Qwen3-8B, has the same shape as /, so no pattern can split it safely. The telemetry docs now use the new labels and show a deployment's share of a service's traffic. Fixes #534. Signed-off-by: ChethanUK --- docs/content/platform/telemetry.md | 30 ++++++++++++------- .../function/stacks/metrics.py | 15 +++++++++- .../tests/test_collector.py | 30 +++++++++++++++++-- .../compose-serving-stack/tests/test_fn.py | 22 ++++++++++---- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index a92d2ec34..da5d0b9bf 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -30,15 +30,17 @@ A series about a deployment also has `deployment`, `replica`, `namespace`, `engine`, and `role` labels. On a prefill/decode deployment, an engine's series also has a `phase` label, `Prefill` or `Decode`. -The gateway's `modelplane_frontend_*` series have no `deployment` label. They -carry the model instead: `gen_ai_original_model` is the model the caller sent, -`/`, and `gen_ai_request_model` is the deployment the gateway -sent the request to, `/`. -`modelplane_frontend_requests_active` is counted per `ModelService` instead. It -has a `service` label, and the same `namespace` label as an engine's series: -the team's namespace as it's mirrored on the inference cluster, such as -`mp-ml-team-51733`. The namespace in `gen_ai_original_model` is the team's own, -such as `ml-team`, so the two don't join on it. +The gateway's latency series, `modelplane_frontend_request_duration_seconds`, +`modelplane_frontend_ttft_seconds`, and `modelplane_frontend_tpot_seconds`, have +no `deployment` label. They name the model instead: `service` is the model the +caller sent, `/`, and `served_model` is the deployment the +gateway sent the request to, `/`. For an external +endpoint, `served_model` is the provider's model name. +`modelplane_frontend_requests_active` is counted per `ModelService` instead. Its +`service` label is the service's name alone, and it has the same `namespace` +label as an engine's series: the team's namespace as it's mirrored on the +inference cluster, such as `mp-ml-team-51733`. The namespace in the latency +series' `service` is the team's own, such as `ml-team`, so the two don't join on it. Each replica publishes its own series, so combine them in your query. Which combination is right follows from what the metric measures: @@ -274,7 +276,15 @@ fleet-wide p99 of time to first token, which requires streaming traffic: ```promql histogram_quantile(0.99, sum by (le) ( - rate(modelplane_frontend_ttft_seconds_bucket{gen_ai_request_model="ml-team/qwen3-8b"}[5m]))) + rate(modelplane_frontend_ttft_seconds_bucket{served_model="ml-team/qwen3-8b"}[5m]))) +``` + +Each deployment's share of a service's traffic, such as during a canary: + +```promql +sum by (served_model) (rate(modelplane_frontend_request_duration_seconds_count{service="ml-team/assistant"}[5m])) + / ignoring (served_model) group_left +sum (rate(modelplane_frontend_request_duration_seconds_count{service="ml-team/assistant"}[5m])) ``` Modelplane has no dashboards of its own. What it exports is counters and histogram buckets, and diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index ce2bfccdd..83b59628f 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -53,6 +53,17 @@ "gen_ai_server_time_per_output_token_seconds": "modelplane_frontend_tpot_seconds", } +# Which model each request was for, under Modelplane's names. The route matches +# on /, the name the caller sent, and each endpoint +# overrides it with its served model name, /, or the +# provider's model name for an external one. served_model stays whole rather +# than split into namespace and deployment, because an external name such as +# Qwen/Qwen3-8B has the same shape. +_GATEWAY_LABELS = [ + {"name": "service", "from": "gen_ai_original_model"}, + {"name": "served_model", "from": "gen_ai_request_model"}, +] + # The front door's own count of requests in flight to each model, which the # collector keeps for the fleet gateway's model routes only (see # collector._IN_FLIGHT). The engines' running and waiting requests well above @@ -237,6 +248,7 @@ def _mapping( tokens: tuple[str, str] | None = None, extra: tuple[dict[str, object], ...] = (), types: dict[str, str] | None = None, + labels: list[dict[str, str]] | None = None, ) -> v1alpha1.MetricMapping: units = units or {} types = types or {} @@ -244,6 +256,7 @@ def _mapping( {"from": src, "to": dst} | ({"fromUnit": units[src]} if src in units else {}) | ({"fromType": types[src]} if src in types else {}) + | ({"labels": labels} if labels else {}) for src, dst in pairs.items() ] if tokens: @@ -263,7 +276,7 @@ def mappings() -> list[v1alpha1.MetricMapping]: operator's do, and a built-in that breaks breaks the path everyone uses. """ return [ - _mapping("modelplane-gateway", _GATEWAY), + _mapping("modelplane-gateway", _GATEWAY, labels=_GATEWAY_LABELS), _mapping("modelplane-gateway-proxy", _GATEWAY_PROXY), _mapping("modelplane-vllm", _VLLM, tokens=_VLLM_TOKENS, extra=(_VLLM_RESPONSES,)), _mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)), diff --git a/functions/compose-serving-stack/tests/test_collector.py b/functions/compose-serving-stack/tests/test_collector.py index 3583f936d..3662bb433 100644 --- a/functions/compose-serving-stack/tests/test_collector.py +++ b/functions/compose-serving-stack/tests/test_collector.py @@ -480,6 +480,18 @@ def _config( { "context": "datapoint", "statements": [ + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -977,6 +989,18 @@ def test_in_flight(case: InFlightCase) -> None: 'convert_gauge_to_sum("cumulative", true) where metric.name == "gpu_energy_consumed"', ], [ + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -1245,7 +1269,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="522e31c7cedc07c5", + config_hash="fdd24284aba7958f", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -1290,7 +1314,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="1f03e323432223be", + config_hash="ceb2f66f9ded10e2", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -1357,7 +1381,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="b1539f5a25d85bfc", + config_hash="653cb26ec99e9671", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}}, diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index d3e90092a..fa61b35aa 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -1694,6 +1694,18 @@ def _collector_config( { "context": "datapoint", "statements": [ + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_request_duration_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_request_duration_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_to_first_token_seconds"', + 'set(datapoint.attributes["service"], datapoint.attributes["gen_ai_original_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_original_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'set(datapoint.attributes["served_model"], datapoint.attributes["gen_ai_request_model"]) where metric.name == "gen_ai_server_time_per_output_token_seconds"', + 'delete_key(datapoint.attributes, "gen_ai_request_model") where metric.name == "gen_ai_server_time_per_output_token_seconds"', 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', 'set(datapoint.attributes["direction"], "output") where metric.name == "vllm:request_generation_tokens_sum"', 'set(datapoint.attributes["reason"], datapoint.attributes["finished_reason"]) where metric.name == "vllm:request_success_total"', @@ -4746,7 +4758,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="be9842417e13d6c1", + config_hash="6a3bc6ff5fb2ef74", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5065,7 +5077,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "collector": _collector( - config_hash="5d3c556be06d8ca8", + config_hash="096cca2888bd9898", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -5352,7 +5364,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="0a72aaee56455e72", + config_hash="c350471b6a11500d", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5615,7 +5627,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="57e31b8f680bd6d3", + config_hash="92e391f460986abd", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5906,7 +5918,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="be9842417e13d6c1", + config_hash="6a3bc6ff5fb2ef74", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None,