From 1f172602f238d50781b820aa4cf8b066fb4c0f9d Mon Sep 17 00:00:00 2001 From: leslie-fang25 Date: Mon, 13 Jul 2026 09:54:46 +0000 Subject: [PATCH 1/2] [https://nvbugs/6165866][test] Skip quant=None and e128 W4A16_MXFP4 TRTLLM MoE cases on Blackwell Signed-off-by: leslie-fang25 --- .../_torch/modules/moe/moe_test_utils.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/unittest/_torch/modules/moe/moe_test_utils.py b/tests/unittest/_torch/modules/moe/moe_test_utils.py index 3512aef6a550..59aeb7fb8333 100644 --- a/tests/unittest/_torch/modules/moe/moe_test_utils.py +++ b/tests/unittest/_torch/modules/moe/moe_test_utils.py @@ -304,6 +304,44 @@ def should_skip_trtllm( "see the comment above for reproduction." ) + # [nvbug 6165866] The same uninitialized-trtllm-gen-scratch IMA cascade + # described above also poisons the whole `-k "TRTLLM"` pytest process on + # Blackwell (both SM100/B200 and SM103/B300) through two broader case + # families. They are skipped here (in addition to the specific SM103 case + # above) so the single-process run stays clean until the kernel is fixed. + # See https://nvbugspro.nvidia.com/bug/6165866 + # + # (1) quant=None (BF16): the flashinfer BF16 GEMM + # (`bmm_Bfloat16_..._sm100f`, trtllm_batched_gemm_runner.cu:305) is the + # cascade victim once a block-scale case has run earlier in the same + # process. Minimal repro (reproduces on both SM100 and SM103): + # e60_k4_h2048_i1408-seq=1-quant=None, + # e60_k4_h2048_i1408-seq=1-quant=FP8_BLOCK_SCALES, + # e60_k4_h2048_i1408-seq=1-quant=W4A8_MXFP4_FP8, + # then e8_k1_h512_i512-seq=1-quant=None -> IMA on the last case. + # + # (2) e128 W4A16_MXFP4 (all seq / plain+gptoss): the four e128 shape + # (num_experts=128, top_k=4, hidden=intermediate=2880) W4A16_MXFP4 + # cases poison each other (the SM103 gptoss-seq8 case above is one of + # them; on SM100 the first observed FAIL is e128-seq1-W4A16_MXFP4). + is_e128_w4a16_mxfp4 = ( + quant_algo == QuantAlgo.W4A16_MXFP4 + and model_config is not None + and model_config.num_experts == 128 + and model_config.top_k == 4 + and model_config.hidden_size == 2880 + and model_config.intermediate_size == 2880 + ) + if get_sm_version() in (100, 103) and (quant_algo is None or is_e128_w4a16_mxfp4): + return ( + "[nvbug 6165866] TRTLLM-Gen MoE on Blackwell (SM100/SM103) hits a " + "CUDA illegal memory access that cascades across the whole pytest " + "process; the quant=None (flashinfer BF16 GEMM victim) and e128 " + "W4A16_MXFP4 (self-cascading tileN=32 tactic) families are skipped " + "until the kernel is fixed. " + "See https://nvbugspro.nvidia.com/bug/6165866" + ) + # Routing method compatibility check (used by test_moe_module.py) # TRTLLMGen C++ routing kernel (runner.cu) implements: # - DeepSeekV3 (nGroup<=1: SigmoidBias+ScaledSumNormalize; nGroup>1: full DeepSeek kernel) From 4b644c987c8753dc07b47d8dba01c6153ccfbc79 Mon Sep 17 00:00:00 2001 From: leslie-fang25 Date: Mon, 13 Jul 2026 11:03:05 +0000 Subject: [PATCH 2/2] [https://nvbugs/6165866][test] Unwaive test_moe_backend -k "TRTLLM" on Blackwell The blanket B300/DGX_B200 waives are superseded by the in-process cascade skips added in should_skip_trtllm (quant=None and e128 W4A16_MXFP4); the remaining TRTLLM MoE cases now run in CI. Signed-off-by: leslie-fang25 --- tests/integration/test_lists/waives.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 8d1c6e2b1433..d2052bbf8cdf 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -229,8 +229,6 @@ full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_a full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype SKIP (https://nvbugs/6316983) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6322073) full:B300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) -full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) -full:DGX_B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[MIN_LATENCY] SKIP (https://nvbugs/6403920) full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[ONESHOT] SKIP (https://nvbugs/6403920) full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[TWOSHOT] SKIP (https://nvbugs/6403920)