Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/content/models/model-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,10 @@ Tune both from what the gateway measures, as
first byte, and `modelplane_frontend_request_duration_seconds` how long whole
requests take. The gateway's request duration buckets stop at 81.92 seconds, so
for longer requests read the engine's `modelplane_request_duration_seconds`
instead. Modelplane doesn't yet export how often the gateway times out; see
[#536](https://github.com/modelplaneai/modelplane/issues/536). A restarting
instead. `modelplane_gateway_timeouts_total` counts the gateway's timeouts per
service, by `kind`: `request` for `request`, and `per_try_idle` for `idle`. A
stream cut short by `idle` still reaches the caller as a 200, so only
`per_try_idle` records it. A restarting
gateway lets requests already in flight run for five minutes, so a restart can
cut off a response allowed longer than that.

Expand Down
14 changes: 13 additions & 1 deletion docs/content/platform/telemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@ sent the request to, `<namespace>/<deployment>`.
has a `service` label, and the same `namespace` label as an engine's series:
the team's namespace as it's mirrored on the inference cluster, such as
`mp-ml-team-51733`. The namespace in `gen_ai_original_model` is the team's own,
such as `ml-team`, so the two don't join on it.
such as `ml-team`, so the two don't join on it. The gateway's
`modelplane_gateway_*` counters of retries, timeouts, resets, ejections and
upstream responses have the same `namespace` and `service` labels. Every
deployment behind a service is one upstream to the gateway, so these count per
service, not per deployment.
`modelplane_gateway_requests_total` has neither, because it's counted per
listener, before the gateway picks a route.

Each replica publishes its own series, so combine them in your query. Which
combination is right follows from what the metric measures:
Expand Down Expand Up @@ -83,6 +89,12 @@ Some examples of the available metrics:
| `modelplane_frontend_tpot_seconds` | Time per output token, measured at the gateway. Streaming requests only. |
| `modelplane_frontend_request_duration_seconds` | What the caller waited, end to end |
| `modelplane_frontend_requests_active` | Requests open at the gateway, per `ModelService` |
| `modelplane_gateway_requests_total` | Requests the fleet gateway answered, by `status_class` (`1xx` to `5xx`), including 401, 404 and 429 responses no model served |
| `modelplane_gateway_upstream_responses_total` | Responses per `ModelService` by `status_class`, counted once per request by its final code, including the gateway's own 503 and 504 |
| `modelplane_gateway_retries_total` | Retries the gateway sent, per `ModelService` |
| `modelplane_gateway_timeouts_total` | Timeouts per `ModelService`, by `kind`: `request` counts requests, `per_try` and `per_try_idle` count attempts |
| `modelplane_gateway_upstream_resets_total` | Upstream resets per `ModelService`, by `kind`: `stream_reset` counts requests, `connection_closed` counts connections closed with requests in flight |
| `modelplane_gateway_ejections_total` | Times the gateway took an endpoint out of rotation after consecutive 5xx, per `ModelService` |
| `modelplane_request_queue_seconds` | How long a request waited before the engine started |
| `modelplane_requests_waiting` | Queue depth per engine |
| `modelplane_kv_cache_utilization_ratio` | KV-cache occupancy, per replica |
Expand Down
56 changes: 45 additions & 11 deletions functions/compose-serving-stack/function/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,31 @@
_IN_FLIGHT = "envoy_cluster_upstream_rq_active"
_FLEET_GATEWAY = "inference-gateway"

# Envoy's per-cluster counts of what happened to the requests it sent: retries,
# timeouts, resets, ejections and responses by class. Each cluster gateway
# reports its own httproute/<ns>/<deployment>/rule/0 for the same requests, so
# these are kept and attributed exactly as the requests in flight are.
_UPSTREAM_SERIES = (
_IN_FLIGHT,
"envoy_cluster_upstream_rq_retry",
"envoy_cluster_upstream_rq_timeout",
"envoy_cluster_upstream_rq_per_try_timeout",
"envoy_cluster_upstream_rq_per_try_idle_timeout",
"envoy_cluster_upstream_rq_rx_reset",
"envoy_cluster_upstream_cx_destroy_remote_with_active_rq",
"envoy_cluster_outlier_detection_ejections_enforced_total",
"envoy_cluster_upstream_rq_xx",
)
_UPSTREAM = f"(?:{'|'.join(_UPSTREAM_SERIES)})"

# What the fleet gateway answered callers, per listener, which also counts the
# 401s, 404s and 429s that never reach a model. A cluster gateway counts the
# fleet gateway's forwarded requests a second time, and Envoy Gateway's own
# listeners, such as eg-stats-http, count scrapes and probes, so only the
# fleet gateway's Gateway listeners, named http-<port> or https-<port>, are kept.
_DOWNSTREAM = "envoy_http_downstream_rq_xx"
_LISTENER = r"https?-\d+"

# The cluster Envoy builds for a ModelService: compose-model-route's
# AIGatewayRoute, named for the service in the team's mirrored namespace, and
# rule 0, the one rule compose_route gives it. The namespace is the one an
Expand Down Expand Up @@ -193,7 +218,10 @@ def _annotated_port() -> list[dict[str, Any]]:


def _in_flight_relabels() -> list[dict[str, Any]]:
"""Keep the fleet gateway's in-flight requests to each model, as namespace and service.
"""Keep the fleet gateway's counts for each model, as namespace and service.

Its requests in flight and the other _UPSTREAM series, and its _DOWNSTREAM
responses, which aren't per model and keep their listener.

Prometheus relabeling rather than a transform statement, because it runs
as the series are scraped, and the two drops are on labels a mapping can't
Expand All @@ -207,29 +235,33 @@ def _in_flight_relabels() -> list[dict[str, Any]]:
return [
{
"source_labels": ["__name__", "envoy_cluster_name", "gateway"],
"regex": f"{_IN_FLIGHT};{_MODEL_ROUTE};{_FLEET_GATEWAY}",
"regex": f"{_UPSTREAM};{_MODEL_ROUTE};{_FLEET_GATEWAY}",
"target_label": "__tmp_model",
"replacement": "true",
},
{"source_labels": ["__name__", "__tmp_model"], "regex": f"{_IN_FLIGHT};", "action": "drop"},
{
"source_labels": ["__name__", "envoy_http_conn_manager_prefix", "gateway"],
"regex": f"{_DOWNSTREAM};{_LISTENER};{_FLEET_GATEWAY}",
"target_label": "__tmp_model",
"replacement": "true",
},
{"source_labels": ["__name__", "__tmp_model"], "regex": f"(?:{_UPSTREAM}|{_DOWNSTREAM});", "action": "drop"},
{
"source_labels": ["__name__", "envoy_cluster_name"],
"regex": f"{_IN_FLIGHT};{_MODEL_ROUTE}",
"regex": f"{_UPSTREAM};{_MODEL_ROUTE}",
"target_label": "namespace",
"replacement": "$1",
},
{
"source_labels": ["__name__", "envoy_cluster_name"],
"regex": f"{_IN_FLIGHT};{_MODEL_ROUTE}",
"regex": f"{_UPSTREAM};{_MODEL_ROUTE}",
"target_label": "service",
"replacement": "$2",
},
# An empty label is no label, so these leave the series with only the
# identity just set.
*(
{"source_labels": ["__name__"], "regex": _IN_FLIGHT, "target_label": label, "replacement": ""}
for label in ("envoy_cluster_name", "__tmp_model")
),
{"source_labels": ["__name__"], "regex": _UPSTREAM, "target_label": "envoy_cluster_name", "replacement": ""},
{"target_label": "__tmp_model", "replacement": ""},
# The gateway is a target label, so it's on every series from the job,
# and only the rules above read it.
{"target_label": "gateway", "replacement": ""},
Expand Down Expand Up @@ -403,11 +435,13 @@ def _scrape_configs() -> list[dict[str, Any]]:
*_annotated_port(),
{"source_labels": ["__meta_kubernetes_namespace"], "target_label": "namespace"},
],
# Another Envoy's count of requests in flight would otherwise leave
# Another Envoy's counts of requests would otherwise leave
# under the gateway's name, with its own cluster names and none of
# the gateway job's drops. An Istio sidecar publishes Envoy's
# statistics, and annotates its pod for scraping.
"metric_relabel_configs": [{"source_labels": ["__name__"], "regex": _IN_FLIGHT, "action": "drop"}],
"metric_relabel_configs": [
{"source_labels": ["__name__"], "regex": f"(?:{_UPSTREAM}|{_DOWNSTREAM})", "action": "drop"}
],
},
]

Expand Down
41 changes: 40 additions & 1 deletion functions/compose-serving-stack/function/stacks/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,47 @@
# it, for minutes, is work no caller is waiting for.
_GATEWAY_PROXY = {
"envoy_cluster_upstream_rq_active": "modelplane_frontend_requests_active",
"envoy_cluster_upstream_rq_retry": "modelplane_gateway_retries_total",
"envoy_cluster_outlier_detection_ejections_enforced_total": "modelplane_gateway_ejections_total",
}

# The fleet gateway's timeouts, resets and responses, kept per model like the
# requests in flight (see collector._UPSTREAM), and its responses to callers per
# listener (collector._DOWNSTREAM). Folded sources keep a label apart, or two
# counters for one service would land on one series and one would replace the
# other. A timeout's kind tells a request's timeout from an attempt's: Envoy
# counts per_try and per_try_idle once per attempt, timeout once per request.
# Envoy labels a response by the first digit of its code.
_RESPONSE_CLASSES = {str(n): f"{n}xx" for n in range(1, 6)}
_STATUS_CLASS = {"name": "status_class", "from": "envoy_response_code_class", "values": _RESPONSE_CLASSES}
_GATEWAY_PROXY_LABELLED: tuple[dict[str, object], ...] = (
*(
{"from": src, "to": "modelplane_gateway_timeouts_total", "labels": [{"name": "kind", "value": kind}]}
for src, kind in (
("envoy_cluster_upstream_rq_timeout", "request"),
("envoy_cluster_upstream_rq_per_try_timeout", "per_try"),
("envoy_cluster_upstream_rq_per_try_idle_timeout", "per_try_idle"),
)
),
*(
{"from": src, "to": "modelplane_gateway_upstream_resets_total", "labels": [{"name": "kind", "value": kind}]}
for src, kind in (
("envoy_cluster_upstream_rq_rx_reset", "stream_reset"),
("envoy_cluster_upstream_cx_destroy_remote_with_active_rq", "connection_closed"),
)
),
{
"from": "envoy_cluster_upstream_rq_xx",
"to": "modelplane_gateway_upstream_responses_total",
"labels": [_STATUS_CLASS],
},
{
"from": "envoy_http_downstream_rq_xx",
"to": "modelplane_gateway_requests_total",
"labels": [_STATUS_CLASS, {"name": "listener", "from": "envoy_http_conn_manager_prefix"}],
},
)

# The engines, which explain what the gateway measured. vLLM publishes these
# names from v0.9.2.
_VLLM = {
Expand Down Expand Up @@ -264,7 +303,7 @@ def mappings() -> list[v1alpha1.MetricMapping]:
"""
return [
_mapping("modelplane-gateway", _GATEWAY),
_mapping("modelplane-gateway-proxy", _GATEWAY_PROXY),
_mapping("modelplane-gateway-proxy", _GATEWAY_PROXY, extra=_GATEWAY_PROXY_LABELLED),
_mapping("modelplane-vllm", _VLLM, tokens=_VLLM_TOKENS, extra=(_VLLM_RESPONSES,)),
_mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)),
_mapping("modelplane-picker", _PICKER),
Expand Down
Loading
Loading