From cf6d5476226f69916e4904bb7f3bb09df9aa5ffb Mon Sep 17 00:00:00 2001 From: Sihan Wang Date: Thu, 11 Jun 2026 18:07:01 +0000 Subject: [PATCH] [None][feat] Add per-request priority support to OpenAI chat/completions Expose the executor's per-request scheduling priority through the OpenAI frontend: accept 'priority' (float in [0,1], higher served first, engine default 0.5) on ChatCompletionRequest and CompletionRequest, and forward it to generate_async in openai_chat and openai_completion. Only honored when scheduler_config.waiting_queue_policy=priority; the default FCFS waiting queue ignores it. Validated on a 4xB300 canary (Kimi-K2.5-Turbo, 160-way saturation): HIGH(1.0) p50 TTFT 0.947s vs LOW(0.0) 2.857s (+66.8%), equal-priority control gap ~0. The mm_encoder and responses endpoints are intentionally not wired; deepapi only routes /v1/chat/completions and /v1/completions. Co-Authored-By: Claude Fable 5 Signed-off-by: Sihan Wang --- tensorrt_llm/serve/openai_protocol.py | 18 ++++++++++++++++++ tensorrt_llm/serve/openai_server.py | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 1c0918fe370e..de1d6ea29d47 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -435,6 +435,15 @@ class CompletionRequest(OpenAIBaseModel): "The time interval in milliseconds to create responses under the streaming mode. " "If not set, the engine-level default is used."), ) + priority: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Scheduling priority in [0.0, 1.0]; higher is served first. Only honored " + "when the engine runs with scheduler_config.waiting_queue_policy=priority. " + "If unset, the engine default (0.5) is used."), + ) # doc: end-completion-extra-params @@ -828,6 +837,15 @@ class ChatCompletionRequest(OpenAIBaseModel): "The time interval in milliseconds to create responses under the streaming mode. " "If not set, the engine-level default is used."), ) + priority: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Scheduling priority in [0.0, 1.0]; higher is served first. Only honored " + "when the engine runs with scheduler_config.waiting_queue_policy=priority. " + "If unset, the engine default (0.5) is used."), + ) agent_hierarchy: Optional[AgentHierarchy] = Field( default=None, description="Agent hierarchy ") diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index f1a9a012de29..c82464b06d4c 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -1597,6 +1597,7 @@ async def chat_stream_generator( cache_salt=request.cache_salt, trace_headers=trace_headers, scheduling_params=scheduling_params, + priority=request.priority if request.priority is not None else 0.5, ) if not self.postproc_worker_enabled: postproc_args.tokenizer = self.tokenizer @@ -1921,7 +1922,8 @@ async def generator_wrapper(generator: AsyncIterator[Any]): streaming=request.stream, lora_request=request.lora_request, disaggregated_params=disaggregated_params, - trace_headers=trace_headers + trace_headers=trace_headers, + priority=request.priority if request.priority is not None else 0.5, ) if not self.postproc_worker_enabled: postproc_args.tokenizer = self.tokenizer