From 4bfca236860b323767a2973e27b04ec7ecd351e5 Mon Sep 17 00:00:00 2001
From: bri-prism <288398250+bri-prism@users.noreply.github.com>
Date: Fri, 30 Oct 2026 11:35:30 -0700
Subject: [PATCH] cuda: keep the extended tall mat-vec types off fused gate+up
 launches

The 8-rows-per-CTA geometry for Q8_0/Q4_0/Q2_0/Q4_K/Q6_K helps plain one-column
mat-vecs but costs time on fused gate+up pairs. Leave those on the default
geometry; PQ2_0 keeps its existing behavior. Output is unchanged.
---
 ggml/src/ggml-cuda/mmvq.cu | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 27797a6e1d28..5198385b6bf6 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -1649,7 +1649,9 @@ static void mul_mat_vec_q_switch_ncols_dst(
     // GGML_CUDA_MMVQ_TALL_EXT=0 keeps it PQ2_0-only.
     const auto should_use_tall = [&] {
         static const bool ext = !getenv("GGML_CUDA_MMVQ_TALL_EXT") || atoi(getenv("GGML_CUDA_MMVQ_TALL_EXT")) != 0;
-        return (type == GGML_TYPE_PQ2_0 || (ext && mmvq_tall_ext_type(type))) && table_id == MMVQ_PARAMETERS_GENERIC && GGML_CUDA_CC_IS_NVIDIA(cc) &&
+        // The extended types are left on the default geometry when a gate matrix is fused (gate+up pair): there 8 rows per CTA ran slower.
+        const bool ext_ok = ext && mmvq_tall_ext_type(type) && fusion.gate == nullptr;
+        return (type == GGML_TYPE_PQ2_0 || ext_ok) && table_id == MMVQ_PARAMETERS_GENERIC && GGML_CUDA_CC_IS_NVIDIA(cc) &&
                cc >= GGML_CUDA_CC_HOPPER && cc < GGML_CUDA_CC_BLACKWELL && !has_ids && !y_soa && nrows_x >= 4096 &&
                nrows_x % MMVQ_TALL_ROWS_PER_BLOCK == 0; // whole CTAs: rows past the end would be read, not only skipped
     };