From eea624fa70b6920766947e7acb6886ebb3a1591a Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:12:16 -0700 Subject: [PATCH] [https://nvbugs/6930163][fix] Shut down Inductor compile workers in KV pool rebalance test The logprobs=2 sampling params route the in-process PyExecutor through the torch.compile'd logprob fusions, which lazily start Inductor's compile-worker pool. Its daemon threads outlived the test and failed the integration thread-leak check. Shut the pool down inside the test body and join the threads it was told to stop; remove the nvbugs/6930163 waivers. Co-Authored-By: Claude Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- .../test_kv_pool_rebalance_accuracy.py | 37 ++++++++++++++++--- tests/integration/test_lists/waives.txt | 2 - 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py b/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py index 50abe9c76283..fac5da885a4b 100644 --- a/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py +++ b/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py @@ -23,7 +23,11 @@ tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py """ +import threading + import pytest +import torch._inductor.config as inductor_config +from torch._inductor.async_compile import shutdown_compile_workers from tensorrt_llm import LLM from tensorrt_llm.llmapi import KvCacheConfig, SamplingParams @@ -59,6 +63,22 @@ def _inject_pool_ratio_mismatch(llm: LLM, *, skew: float = 2.0) -> None: _introspection.force_rebalance_precondition(kv_cache_manager.impl, skew=skew) +def _shutdown_inductor_compile_workers() -> None: + """Stop Inductor's compile-worker pool and wait for its threads to exit. + + The sampler's logprob fusions are ``torch.compile``d, so the first request + with ``logprobs`` lazily starts Inductor's process-global compile pool. + With the PyExecutor in-process, that pool's daemon threads would otherwise + outlive the test and trip the integration suite's thread-leak check. + """ + shutdown_compile_workers() + # The pool's health monitor and quiesce timer only notice the shutdown at + # their next poll; the timer polls every quiesce_async_compile_time / 2. + for thread in threading.enumerate(): + if thread.name.startswith("InductorSubproc") or thread.name == "subproc_worker_timer": + thread.join(timeout=inductor_config.quiesce_async_compile_time) + + # --------------------------------------------------------------------------- # # Test # --------------------------------------------------------------------------- # @@ -232,13 +252,18 @@ def test_rebalance_matches_baseline(self, disable_overlap, monkeypatch): # can reach .engine on the client side. monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") - baseline = _generate_tokens( - model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=False - ) + # pytest-threadleak checks before fixture teardown, so the compile pool + # has to be shut down inside the test body. + try: + baseline = _generate_tokens( + model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=False + ) - treated = _generate_tokens( - model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=True - ) + treated = _generate_tokens( + model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=True + ) + finally: + _shutdown_inductor_compile_workers() assert len(baseline) == len(treated) == len(_PROMPTS) for i, (b, t) in enumerate(zip(baseline, treated)): diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2579531d1737..791f428f6d6c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -1,7 +1,5 @@ accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) accuracy/test_kimi3.py::TestKimiK3DSpark::test_gsm8k_tep8 SKIP (https://nvbugs/6766812) -accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] SKIP (https://nvbugs/6930163) -accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] SKIP (https://nvbugs/6930163) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6402058) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774)