From ce04ee029395a9a459756d5f408e8cfc6f07aec5 Mon Sep 17 00:00:00 2001 From: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 8 Jul 2026 05:23:29 -0700 Subject: [PATCH 1/4] [https://nvbugs/6388153][fix] Relay PP sample states synchronously on the executor thread Signed-off-by: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 55 ++++++++++++++----- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 6e3252b48688..639223778204 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -10,7 +10,7 @@ import traceback from contextlib import contextmanager from enum import IntEnum -from queue import Queue +from queue import Empty, Queue from typing import (TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Union) @@ -959,13 +959,11 @@ def on_detected(): if self.dist.pp_size > 1: self.event_loop = self._executor_loop_pp - # `TLLM_PP_ASYNC_BROADCAST_SAMPLE_STATE` controls whether to broadcast the sample state asynchronously. - # If true, the executor loop can broadcast and handle sample states asynchronously to achieve best perf. - # If false, the executor loop can only broadcast and handle each sample state in a pre-defined iteration. - # It is only for debugging purposes. - # Some tests can disable it to get a deterministic behavior. + # Sample-state relay mode. "1": a background thread relays them, + # overlapping with forward, but it needs the GIL and can be starved + # into a PP deadlock by GIL-holding native calls. "0": relay inline. self.pp_async_broadcast_sample_state = os.environ.get( - "TLLM_PP_ASYNC_BROADCAST_SAMPLE_STATE", "1") == "1" + "TLLM_PP_ASYNC_BROADCAST_SAMPLE_STATE", "0") == "1" else: self.event_loop = self._executor_loop if self.disable_overlap_scheduler else self._executor_loop_overlap if is_trace_enabled("TLLM_TRACE_EXECUTOR_LOOP"): @@ -1224,10 +1222,13 @@ def start_worker(self): with self.worker_lock: if not self.worker_started: if self.dist.pp_size > 1: - self.executed_batch_queue: Queue[BatchStatePP] = Queue( - maxsize=self.num_micro_batches) + # Both relay modes deposit relayed sample states here + # for the executor loop to handle. self.executed_batch_response_queue: Queue[ BatchStatePP] = Queue(maxsize=-1) + if self.dist.pp_size > 1 and self.pp_async_broadcast_sample_state: + self.executed_batch_queue: Queue[BatchStatePP] = Queue( + maxsize=self.num_micro_batches) # Duplicate the communicator on the main thread before the # PP event loop starts. MPI_Comm_dup is collective across # ranks, so doing it here avoids racing with the worker @@ -1464,7 +1465,7 @@ def shutdown(self): logger.error("Hang detected, shutting down immediately.") return self.worker_thread.join() - if self.dist.pp_size > 1: + if self.dist.pp_size > 1 and self.pp_async_broadcast_sample_state: self.executed_batch_queue.put(None) self.broadcast_sample_state_handler.join() # Signal non-rank-0 sleep/wakeup listener threads to exit. This runs @@ -2621,6 +2622,13 @@ def _executor_loop_pp(self): else: logger.debug(f"microbatch {microbatch_id} can be queued") + if not self.pp_async_broadcast_sample_state: + # Drain pending relay isends before forward: rendezvous + # sends need this rank to keep entering MPI, and a forward + # blocked in a native call would starve the receiving rank. + for mb in range(self.num_micro_batches): + self.wait_on_pp_send_handles(self.send_handles, mb) + self._add_inflight_ids(scheduled_batch) if self.kv_cache_transceiver: @@ -2753,7 +2761,13 @@ def _executor_loop_pp(self): offset) % self.num_micro_batches executed_batch = self.micro_batches[executed_microbatch_id] if executed_batch is not None: - self.executed_batch_queue.put(executed_batch) + if self.pp_async_broadcast_sample_state: + self.executed_batch_queue.put(executed_batch) + else: + # Relay inline on the executor thread; unlike the + # background thread, delivery cannot be starved by the + # GIL when a forward blocks inside a native call. + self._ring_broadcast_sample_state(executed_batch) self.unhandled_batch_counter += 1 self.micro_batches[executed_microbatch_id] = None @@ -2803,8 +2817,23 @@ def handle_executed_batches(executed_batch_num: int): dequeue_counter = 0 while dequeue_counter < executed_batch_num: with nvtx_range("get_executed_batch"): - executed_batch = self.executed_batch_response_queue.get( - ) + if self.pp_async_broadcast_sample_state: + executed_batch = self.executed_batch_response_queue.get( + ) + else: + # The inline relay deposits batches in the + # same iteration they are handled; an empty + # queue means the PP ranks diverged. + try: + executed_batch = self.executed_batch_response_queue.get_nowait( + ) + except Empty as e: + raise RuntimeError( + f"PP rank {self.dist.pp_rank} expected " + f"{executed_batch_num} relayed sample states " + f"this iteration but found only {dequeue_counter}; " + f"PP ranks diverged on the microbatch schedule." + ) from e self._handle_executed_batch(executed_batch) dequeue_counter += 1 else: From 6e71e648dd3f9eec8c436b6a0338a2665844789b Mon Sep 17 00:00:00 2001 From: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:05:46 -0700 Subject: [PATCH 2/4] Drop changes not required for correctness per review Signed-off-by: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 639223778204..58cf05a84346 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -10,7 +10,7 @@ import traceback from contextlib import contextmanager from enum import IntEnum -from queue import Empty, Queue +from queue import Queue from typing import (TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Union) @@ -1222,13 +1222,10 @@ def start_worker(self): with self.worker_lock: if not self.worker_started: if self.dist.pp_size > 1: - # Both relay modes deposit relayed sample states here - # for the executor loop to handle. - self.executed_batch_response_queue: Queue[ - BatchStatePP] = Queue(maxsize=-1) - if self.dist.pp_size > 1 and self.pp_async_broadcast_sample_state: self.executed_batch_queue: Queue[BatchStatePP] = Queue( maxsize=self.num_micro_batches) + self.executed_batch_response_queue: Queue[ + BatchStatePP] = Queue(maxsize=-1) # Duplicate the communicator on the main thread before the # PP event loop starts. MPI_Comm_dup is collective across # ranks, so doing it here avoids racing with the worker @@ -1465,7 +1462,7 @@ def shutdown(self): logger.error("Hang detected, shutting down immediately.") return self.worker_thread.join() - if self.dist.pp_size > 1 and self.pp_async_broadcast_sample_state: + if self.dist.pp_size > 1: self.executed_batch_queue.put(None) self.broadcast_sample_state_handler.join() # Signal non-rank-0 sleep/wakeup listener threads to exit. This runs @@ -2767,6 +2764,9 @@ def _executor_loop_pp(self): # Relay inline on the executor thread; unlike the # background thread, delivery cannot be starved by the # GIL when a forward blocks inside a native call. + # Requires the pre-forward drain above: the isend + # completes via rendezvous only while this rank keeps + # entering MPI calls. self._ring_broadcast_sample_state(executed_batch) self.unhandled_batch_counter += 1 self.micro_batches[executed_microbatch_id] = None @@ -2817,23 +2817,8 @@ def handle_executed_batches(executed_batch_num: int): dequeue_counter = 0 while dequeue_counter < executed_batch_num: with nvtx_range("get_executed_batch"): - if self.pp_async_broadcast_sample_state: - executed_batch = self.executed_batch_response_queue.get( - ) - else: - # The inline relay deposits batches in the - # same iteration they are handled; an empty - # queue means the PP ranks diverged. - try: - executed_batch = self.executed_batch_response_queue.get_nowait( - ) - except Empty as e: - raise RuntimeError( - f"PP rank {self.dist.pp_rank} expected " - f"{executed_batch_num} relayed sample states " - f"this iteration but found only {dequeue_counter}; " - f"PP ranks diverged on the microbatch schedule." - ) from e + executed_batch = self.executed_batch_response_queue.get( + ) self._handle_executed_batch(executed_batch) dequeue_counter += 1 else: From 5d9179df7fba32a3cc9a640b3c4832874de3432e Mon Sep 17 00:00:00 2001 From: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:01:56 -0700 Subject: [PATCH 3/4] [https://nvbugs/6388153][chore] Size the microbatch window by relay mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1024-slot minimum exists to absorb the async relay thread's delivery lag. The synchronous relay retires each microbatch within its own iteration, so pipeline-depth many slots suffice — and the pre-forward drain then scans pp_size handles instead of 1024. Signed-off-by: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 58cf05a84346..2fa5a5d0607b 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -737,9 +737,21 @@ def __init__( self.num_scheduled_requests: int = 0 self._configure_benchmark_req_queues_size() + # Sample-state relay mode. "1": a background thread relays them, + # overlapping with forward, but it needs the GIL and can be starved + # into a PP deadlock by GIL-holding native calls. "0": relay inline. + self.pp_async_broadcast_sample_state = os.environ.get( + "TLLM_PP_ASYNC_BROADCAST_SAMPLE_STATE", "0") == "1" + # list of requests in each PP micro batch - self.num_micro_batches = max(self.dist.pp_size, - self.MIN_ASYNC_MICRO_BATCH_NUM) + # The synchronous relay retires each microbatch within its own + # iteration, so pipeline-depth many slots suffice; the async relay + # thread needs a large window to absorb its delivery lag. + if self.pp_async_broadcast_sample_state: + self.num_micro_batches = max(self.dist.pp_size, + self.MIN_ASYNC_MICRO_BATCH_NUM) + else: + self.num_micro_batches = self.dist.pp_size self.micro_batches: List[BatchStatePP | None] = [None] * self.num_micro_batches self.send_handles = [None] * self.num_micro_batches @@ -959,11 +971,6 @@ def on_detected(): if self.dist.pp_size > 1: self.event_loop = self._executor_loop_pp - # Sample-state relay mode. "1": a background thread relays them, - # overlapping with forward, but it needs the GIL and can be starved - # into a PP deadlock by GIL-holding native calls. "0": relay inline. - self.pp_async_broadcast_sample_state = os.environ.get( - "TLLM_PP_ASYNC_BROADCAST_SAMPLE_STATE", "0") == "1" else: self.event_loop = self._executor_loop if self.disable_overlap_scheduler else self._executor_loop_overlap if is_trace_enabled("TLLM_TRACE_EXECUTOR_LOOP"): From 3de1064296ea3784ae3beb9b0b919ab6b6014a37 Mon Sep 17 00:00:00 2001 From: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> Date: Mon, 27 Jul 2026 01:30:01 -0700 Subject: [PATCH 4/4] Unwaive related tests Signed-off-by: WeiHaocheng <20514172+WeiHaocheng@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 647729a6c0da..977c50a9b240 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -32,8 +32,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mt accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428057) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6278337) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6388153) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6388153) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6428094) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428096) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) @@ -41,7 +39,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mt accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP] SKIP (https://nvbugs/6313993) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6388139) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6507095) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6388153) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) @@ -242,8 +239,6 @@ full:GB300/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accurac full:GB300/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-attn_dp_off-trtllm] SKIP (https://nvbugs/6329165) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6525057) -full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6388153) -full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6388153) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=WIDEEP] SKIP (https://nvbugs/6546609) full:GB300/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8] SKIP (https://nvbugs/6474894) full:GB300/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] SKIP (https://nvbugs/6475346) @@ -312,10 +307,8 @@ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False] SKIP (https://nvbugs/6313076) full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash SKIP (https://nvbugs/6273850) full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 SKIP (https://nvbugs/6273850) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] SKIP (https://nvbugs/6400067) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6473374) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) @@ -325,12 +318,8 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::Tes full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6473374) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6473374) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6473374) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6388153) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=0] SKIP (https://nvbugs/6551802) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0] SKIP (https://nvbugs/6551802) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6473373) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6313072)