From 2cdc286aaa0522baefc829439878817afd1fdacc Mon Sep 17 00:00:00 2001 From: agent-872bf5 Date: Mon, 28 Sep 2026 02:43:50 -0300 Subject: [PATCH 01/12] repack: neutralise the inverted deepseek2 k_b transpose (GLM-4.7-Flash) The converter's 'k_b to the MLA math grid' transpose (FLM_Q4NX_Converter 62594563f) is inverted. Measured directly with the converter's own _pack_2d_float on GLM's blk.0.attn_k_b.weight: the dequantised per-head slice is [512][192] (kv_lora x nope) and packing it AS-IS gives the 12 tiles/head the forward reads (load_heads3 with kb_rows_ = qk_nope_ = 192, cols = kv_lora_ = 512), while transposing it to [192][512] gives 16 tiles/head. The forward then reads past the slice and '1bit serve --device npu' segfaults on the first request. A repack via scripts/repack_gguf.py therefore emits k_b [20][16][5120] while the Q4NX the GLM capture was verified against (/tmp/glm_v3) has [20][12][5120]. The engine's repack now runs the converter from a patched copy with that gate disabled, the same way it already forces Q4_1, and prints why. Verified: the patched repack writes k_b [20][12][5120] (v_b unchanged at [20][16][5120]). --- scripts/repack_gguf.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/scripts/repack_gguf.py b/scripts/repack_gguf.py index 1a7c6d41b..9994dbc84 100644 --- a/scripts/repack_gguf.py +++ b/scripts/repack_gguf.py @@ -297,16 +297,41 @@ def converter_commit(): except Exception: _c = {} _t = _c.get("default_tensor_type") - if _t and _t != "Q4_1": + # deepseek2: the converter's k_b "MLA math grid" transpose (its commit + # 62594563f) is INVERTED. The dequantised per-head slice is [512][192] + # (kv_lora x nope); packed AS-IS it is the 12 tiles/head the forward reads + # (`kb_rows_ = qk_nope_ = 192`), while transposing it to [192][512] gives + # 16 tiles/head and the forward then segfaults. Measured directly against + # the converter's own _pack_2d_float, and against the working Q4NX the + # GLM capture used. Neutralise the gate in the patched copy. + _kb_patch = False + _kb_file = os.path.join(converter_dir, "q4nx", "models", _arch + ".py") + if _arch == "deepseek2" and os.path.exists(_kb_file): + try: + if 'transpose_slice = "k_b_proj" in nm' in open(_kb_file).read(): + _kb_patch = True + except Exception: + pass + if (_t and _t != "Q4_1") or _kb_patch: import shutil as _shutil, tempfile as _tempfile run_dir = _tempfile.mkdtemp(prefix="q4nx-converter-") _shutil.copytree(converter_dir, run_dir, dirs_exist_ok=True) - _c["default_tensor_type"] = "Q4_1" - with open(os.path.join(run_dir, "configs", _arch + ".json"), "w") as f: - json.dump(_c, f, indent=4) - print(f"repack: {_arch} config asks for default_tensor_type={_t}; the NPU " - f"forward reads Q4_1 tiles, so the converter runs from a patched " - f"copy ({run_dir})") + if _t and _t != "Q4_1": + _c["default_tensor_type"] = "Q4_1" + with open(os.path.join(run_dir, "configs", _arch + ".json"), "w") as f: + json.dump(_c, f, indent=4) + print(f"repack: {_arch} config asks for default_tensor_type={_t}; the NPU " + f"forward reads Q4_1 tiles, so the converter runs from a patched " + f"copy ({run_dir})") + if _kb_patch: + _p = os.path.join(run_dir, "q4nx", "models", _arch + ".py") + _s2 = open(_p).read().replace( + 'transpose_slice = "k_b_proj" in nm', + 'transpose_slice = False # engine: k_b packs as-is -> 12 tiles/head') + with open(_p, "w") as f: + f.write(_s2) + print(f"repack: {_arch} k_b transpose is inverted (gives 16 tiles/head, the " + f"forward reads 12); neutralised in the patched copy ({run_dir})") r = subprocess.run([sys.executable, os.path.join(run_dir, "convert.py"), gguf, out_dir], cwd=run_dir) if r.returncode != 0 or not os.path.exists(os.path.join(out_dir, "model.q4nx")): From 83184d92ee0ac48d9e4f782dfd390aa884316f73 Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 07:22:32 -0300 Subject: [PATCH 02/12] forward: resync the two missing GLM MLA fixes (expert transpose, v_b grid) The MLA/MoE path in this re-synced forward was missing two canonical npu-infer fixes, so `1bit serve -m GLM-4.7-Flash-Q4_K_M.gguf --device npu` decoded " jp" instead of " Paris": 1. mla_expert() transposed every routed expert. Its tile decode already reproduces the GGUF's flat order, which IS the logical [N][K] weight, so the transpose made every routed expert ~10-20x wrong and flipped the top-k selection (canonical f336dfc72). Now a plain std::copy. 2. v_b was indexed in-major (wv[k*vh+n]); the Q4NX's logical index is out-major (h*256+n)*512+k (canonical 3d1b252ec). Now wv[n*kv_lora+k]. Verified on the NPU: /v1/completions "The capital of France is" -> " Paris" (4m4s). Q4NX is byte-identical to the verified glm_npu2 dir; the two bugs were the only functional deltas from canonical. --- npu/forward/src/q4nx_forward.cpp | 44 +++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 0ad2e8bac..d863644cd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1267,9 +1267,13 @@ bool Q4nxNpuForward::mla_expert(const uint8_t* pool, size_t per_expert, int e, if (dequant_any(pool + (size_t)e * per_expert, per_expert, K, N, kn.data()) != 0) return false; out.resize((size_t)N * (size_t)K); - for (int k = 0; k < K; k++) - for (int nn = 0; nn < N; nn++) - out[(size_t)nn * K + k] = kn[(size_t)k * N + nn]; + // Do NOT transpose. The tile decode at (rows=K, cols=N) already reproduces the + // GGUF's flat order, and that flat sequence IS the logical [N][K] = [out][in] + // weight. Verified elementwise against the GGUF per expert: reshape(N,K) gives + // corr 0.999198 (gate) / 0.996949 (down), while transposing gives -0.001079 / + // 0.000417 -- i.e. the transpose destroyed the expert entirely, making every + // routed expert's contribution ~10-20x wrong and flipping the top-k selection. + std::copy(kn.begin(), kn.end(), out.begin()); return true; } @@ -1326,12 +1330,14 @@ bool Q4nxNpuForward::mla_attn(int l, const std::vector& x, std::vector& x, std::vector& h, std::vector } return true; } - const int i_qkv = idx_of(Q4NX_DESIGN_QKV); - const int T = designs_[i_qkv].N; // reuse the QKV geometry + // Reuse an existing GEMM geometry for the head (the variable keeps its old name + // for the block below). QKV when the arch has one; the MLA/deepseek2 arch + // (GLM-4.7-Flash) has QA/QB/KVA/O instead, so idx_of(QKV) returned -1 and T was + // read out of designs_[-1] -- garbage, and the ntiles division below raised + // SIGFPE. Only this NPU path uses T (the host path is a plain matvec), which is + // why GLM crashed here the first time the ELF path was exercised. + int i_qkv = idx_of(Q4NX_DESIGN_QKV); + if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) + i_qkv = idx_of(Q4NX_DESIGN_DG); // K=H, N=dense_im + if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) + i_qkv = idx_of(Q4NX_DESIGN_G); // K=H, N=expert_im + if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) { + err_ = "LM head: no GEMM design with K=H available"; + return false; + } + const int T = designs_[i_qkv].N; const int ntiles = (NV + T - 1) / T; logits.assign(NV, 0.0f); From b6ce0e5f4c2aff884ef234c5318906996f74d834 Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 08:01:08 -0300 Subject: [PATCH 03/12] =?UTF-8?q?npu:=20qwen3moe=20support=20=E2=80=94=20p?= =?UTF-8?q?er-expert=20MoE=20in=20the=20standard=20path=20(5/6=20on=20--de?= =?UTF-8?q?vice=20npu)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3-Coder-30B-A3B (qwen3moe) is the first NON-MLA MoE arch: the FFN is a 128-expert top-8 softmax router with no shared expert and no router bias. Four changes make `1bit serve -m --device npu` answer " Paris": 1. forward: parse the MoE params (n_expert/top_k/expert_im/dense_im/ leading_dense/gating/scale) INDEPENDENTLY of mla_, so a non-MLA MoE arch gets its expert geometry (they were read only inside `if (mla_)`). 2. forward: load the MoE tensors (bf16 router + pooled gate/up/down experts) in the standard path when n_expert_>0, instead of the dense gate/up/down; the runner bias and shared expert stay optional (moe_mlp already handles both). 3. forward: dispatch moe_mlp in the standard path for n_expert_>0 (deepseek2 still reaches moe_mlp from the MLA branch, which continues first). 4. forward: `dense_weights_cached` must NOT skip load_layer for a MoE arch -- moe_router_ is a HOST vector reloaded per layer, so skipping it left every layer using the last-loaded layer's router on tokens >= 1 (pos 0 was fine, which is why it looked plausible but decoded "."). scripts/repack_gguf.py: write the MoE config keys for qwen3moe/qwen35moe -- the per-EXPERT intermediate (768) must replace the dense feed_forward_length (5472) so the G/U/D designs ARE the expert MLP, plus n_expert/top_k/softmax gating. Verified: anchor -> " Paris" (3m54s); experts/q_proj/o_proj/router all decode corr >= 0.9989 vs the GGUF; GLM (deepseek2) still answers " Paris" (regression). --- npu/forward/src/q4nx_forward.cpp | 93 +++++++++++++++++++++++++------- scripts/repack_gguf.py | 16 ++++++ 2 files changed, 91 insertions(+), 18 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d863644cd..3f53d3c09 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -369,21 +369,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, rope_dim_ = q4nx_config_int(cfg_path.c_str(), "rope_dim", 0); qk_nope_ = q4nx_config_int(cfg_path.c_str(), "qk_nope_head_dim", 0); v_head_ = q4nx_config_int(cfg_path.c_str(), "v_head_dim", 0); - n_expert_ = q4nx_config_int(cfg_path.c_str(), "n_expert", 0); - top_k_ = q4nx_config_int(cfg_path.c_str(), "top_k", 0); - expert_im_ = c_.IM; - dense_im_ = q4nx_config_int(cfg_path.c_str(), "dense_intermediate_size", 0); - leading_dense_ = q4nx_config_int(cfg_path.c_str(), "leading_dense_block_count", 1); - expert_gating_sigmoid_ = - q4nx_config_int(cfg_path.c_str(), "expert_gating_func", 1) == 2; // 2 = sigmoid - expert_w_scale_ = read_config_float(cfg_path, "\"expert_weights_scale\"", 1.0f); if (q_lora_ <= 0 || kv_lora_ <= 0 || rope_dim_ <= 0 || qk_nope_ <= 0 || - v_head_ <= 0 || n_expert_ <= 0 || top_k_ <= 0 || expert_im_ <= 0 || - dense_im_ <= 0) { + v_head_ <= 0) { fprintf(stderr, " [mla] config incomplete (q_lora=%d kv_lora=%d rope=%d nope=%d " - "v_head=%d n_expert=%d top_k=%d im=%d dense_im=%d)\n", - q_lora_, kv_lora_, rope_dim_, qk_nope_, v_head_, n_expert_, top_k_, - expert_im_, dense_im_); + "v_head=%d)\n", q_lora_, kv_lora_, rope_dim_, qk_nope_, v_head_); err_ = "MLA config incomplete"; return false; } @@ -391,10 +380,32 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // 256-column block (192 -> 256), which is what its 16 tiles/head show. kb_rows_ = qk_nope_; /* the re-converted Q4NX packs k_b at [192][512], 12 tiles/head (verified corr 0.996926 vs the GGUF) */ c_.HD = v_head_; - fprintf(stderr, " [mla] MLA+MoE: q_lora=%d kv_lora=%d rope=%d nope=%d v_head=%d " - "NH=%d n_expert=%d top%d im=%d dense_im=%d gating=%s\n", - q_lora_, kv_lora_, rope_dim_, qk_nope_, v_head_, c_.NH, n_expert_, top_k_, - expert_im_, dense_im_, expert_gating_sigmoid_ ? "sigmoid" : "softmax"); + fprintf(stderr, " [mla] MLA: q_lora=%d kv_lora=%d rope=%d nope=%d v_head=%d NH=%d\n", + q_lora_, kv_lora_, rope_dim_, qk_nope_, v_head_, c_.NH); + } + // MoE params are parsed INDEPENDENTLY of mla_ so a non-MLA MoE arch (qwen3moe / + // qwen35moe) gets its expert geometry too; reading them only inside `if (mla_)` + // left n_expert_/top_k_/expert_im_ zero and the standard MoE path dead. + // expert_im_ is the per-EXPERT intermediate, which c_.IM already holds for a MoE + // config (the converter writes the per-expert FFN width as intermediate_size). + n_expert_ = q4nx_config_int(cfg_path.c_str(), "n_expert", 0); + if (n_expert_ > 0) { + top_k_ = q4nx_config_int(cfg_path.c_str(), "top_k", 0); + expert_im_ = c_.IM; + dense_im_ = q4nx_config_int(cfg_path.c_str(), "dense_intermediate_size", 0); + leading_dense_ = q4nx_config_int(cfg_path.c_str(), "leading_dense_block_count", 0); + expert_gating_sigmoid_ = + q4nx_config_int(cfg_path.c_str(), "expert_gating_func", 1) == 2; // 2 = sigmoid + expert_w_scale_ = read_config_float(cfg_path, "\"expert_weights_scale\"", 1.0f); + if (top_k_ <= 0 || expert_im_ <= 0) { + fprintf(stderr, " [moe] config incomplete (n_expert=%d top_k=%d im=%d)\n", + n_expert_, top_k_, expert_im_); + err_ = "MoE config incomplete"; + return false; + } + fprintf(stderr, " [moe] %sMoE: NH=%d n_expert=%d top%d im=%d dense_im=%d gating=%s\n", + mla_ ? "MLA+" : "", c_.NH, n_expert_, top_k_, expert_im_, dense_im_, + expert_gating_sigmoid_ ? "sigmoid" : "softmax"); } Q4nxModelDims dims{}; @@ -879,6 +890,42 @@ bool Q4nxNpuForward::load_layer(int l) { if (!deq(&lw->v_proj_weight, wv_.data(), nkv * hd, H)) return false; } if (!deq(&lw->o_proj_weight, wo_.data(), H, NH * hd)) return false; + if (n_expert_ > 0) { + // Qwen MoE layer (qwen3moe / qwen35moe): a router + top-k routed experts + // instead of a dense gate/up/down. The router is bf16; its bias is OPTIONAL + // (Qwen MoE has none, deepseek2 does). The experts stay pooled -- one + // [N][K] slice is dequantized per selected expert at decode. + char nm2[160]; + auto td2 = [&](const char* fmt) -> TensorDesc* { + snprintf(nm2, sizeof(nm2), fmt, l); + return model_tensor_by_name(mw_, nm2); + }; + auto load_bf16_opt = [&](const char* fmt, std::vector& v, int n) -> bool { + TensorDesc* d = td2(fmt); + if (!d) return false; + const uint16_t* p = (const uint16_t*)model_tensor_data(mw_, d); + if (!p) return false; + v.resize((size_t)n); + for (int i = 0; i < n; i++) v[i] = bf16_to_f32(p[i]); + return true; + }; + if (!load_bf16_opt("model.layers.%d.moe_router.weight", moe_router_, n_expert_ * H)) + { err_ = "moe router"; return false; } + if (!load_bf16_opt("model.layers.%d.moe_router.bias", moe_router_b_, n_expert_)) + moe_router_b_.clear(); + auto pool = [&](const char* fmt, const uint8_t** p, size_t* bytes) -> bool { + TensorDesc* d = td2(fmt); + if (!d) { fprintf(stderr, " [moe] missing %s\n", nm2); return false; } + const uint8_t* raw = (const uint8_t*)model_tensor_data(mw_, d); + if (!raw) return false; + *p = raw; *bytes = (size_t)d->data_size / (size_t)n_expert_; + return true; + }; + if (!pool("model.layers.%d.mlp.gate_exps_proj.weight", &exp_g_, &exp_g_bytes_)) return false; + if (!pool("model.layers.%d.mlp.up_exps_proj.weight", &exp_u_, &exp_u_bytes_)) return false; + if (!pool("model.layers.%d.mlp.down_exps_proj.weight", &exp_d_, &exp_d_bytes_)) return false; + return true; + } if (!deq(&lw->gate_proj_weight, wg_.data(), im, H)) return false; if (!deq(&lw->up_proj_weight, wu_.data(), im, H)) return false; if (!deq(&lw->down_proj_weight, wd_.data(), H, im)) return false; @@ -1780,7 +1827,7 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { // first token), the f32 weights are no longer read, so skip the per-token // Q4NX dequant. GDN/MLA/PLE load members used outside the GEMMs, so they // keep loading every token. - const bool dense_weights_cached = lin_kd_ == 0 && !mla_ && ple_dim_ == 0 && weight_cached(0); + const bool dense_weights_cached = lin_kd_ == 0 && !mla_ && ple_dim_ == 0 && n_expert_ == 0 && weight_cached(0); if (!dense_weights_cached && !load_layer(l)) { if (err_.empty()) err_ = "dequant failed"; return false; } // MLA + MoE layers have no plain q/k/v/o or single-MLP structure, so they // take their own path through this loop body and skip the rest. @@ -1934,6 +1981,16 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { } else { rmsnorm(x.data(), post_norm_[l].data(), H, xn.data()); } + if (n_expert_ > 0) { + // Qwen MoE layer (qwen3moe / qwen35moe): the FFN is router -> top-k + // experts, not a dense gate/up/down. moe_mlp writes the whole FFN + // delta into d. (deepseek2/GLM MoE layers never reach here -- the MLA + // branch above handles attn+MoE and continues.) + if (!moe_mlp(l, xn, d)) return false; + for (int k = 0; k < H; k++) x[k] += d[k] * residual_scale_; + if (!nancheck("moe")) return false; + continue; + } std::vector act; if (i_gu_l >= 0) { const Q4nxDesignGeom* g = &designs_[i_gu_l]; diff --git a/scripts/repack_gguf.py b/scripts/repack_gguf.py index 9994dbc84..3f77aaaa4 100644 --- a/scripts/repack_gguf.py +++ b/scripts/repack_gguf.py @@ -128,6 +128,22 @@ def write_config(gguf_path, out_dir, arch, vocab_size): # wrong still produces plausible output with a wrong argmax. "expert_gating_func": int(_num(reader, f"{p}.expert_gating_func", 1)), }) + # Qwen MoE (qwen3moe / qwen35moe). Same MoE meanings as deepseek2 above but + # different GGUF field names, no MLA, and all layers are MoE (no dense leading + # block). The per-EXPERT intermediate must replace the dense + # feed_forward_length (5472 for the 30B) so the G/U/D designs ARE the expert + # MLP; the forward's n_expert/top_k/expert gating come from these keys. + if p in ("qwen3moe", "qwen35moe"): + cfg.update({ + "intermediate_size": int(_num(reader, f"{p}.expert_feed_forward_length", 0)), + "n_expert": int(_num(reader, f"{p}.expert_count", 0)), + "top_k": int(_num(reader, f"{p}.expert_used_count", 0)), + "expert_shared": int(_num(reader, f"{p}.expert_shared_feed_forward_length", 0)), + "leading_dense_block_count": 0, + "expert_weights_norm": int(_num(reader, f"{p}.expert_weights_norm", 1)), + "expert_weights_scale": float(_num(reader, f"{p}.expert_weights_scale", 1.0)), + "expert_gating_func": 1, # Qwen MoE gating = softmax + }) # MiniCPM4 architecture scales. The forward must apply them: the embedding # is scaled by 12.0 and every residual branch by scale_depth/sqrt(NL) # (0.2475 for 32 layers); without them the hidden state runs at the wrong From ed2c45237da460e4bd709c07dda31151d116f0ca Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 08:01:14 -0300 Subject: [PATCH 04/12] npu serve: serialise on /tmp/1bit-npu-device.lock like the ELF runner The device exposes one NPU hardware context; a second concurrent '1bit serve --device npu' dies inside DRM_IOCTL_AMDXDNA_CREATE_HWCTX (EINVAL) and exits 1. npu_engine_universal has always held this flock; the serve route did not, so siblings could not serialise (and my qwen3moe run hit exactly this when a leftover GLM serve held the context). --- app/forward_serve.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 0dfd8ba57..6657596f6 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -37,7 +37,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -273,6 +276,30 @@ int run_forward_serve(int argc, char** argv) { e.endoftext = e.tok->token_id("<|endoftext|>"); e.fwd = std::make_unique(); + + // Serialise NPU access across processes. The device exposes a single hardware + // context, so a second concurrent `1bit serve --device npu` fails inside + // DRM_IOCTL_AMDXDNA_CREATE_HWCTX with EINVAL and the whole serve exits 1 -- + // which is what happened when several models were served at once. The + // per-process ELF runner (npu_engine_universal) has always taken this lock; + // the serve route did not, so it could not be serialised against it or + // against a sibling serve. Held for the life of the process (fd not closed), + // exactly like the runner. + if (!std::getenv("NPU_NO_DEVICE_LOCK")) { + const char* lk = std::getenv("NPU_DEVICE_LOCK"); + if (!lk || !lk[0]) lk = "/tmp/1bit-npu-device.lock"; + int lfd = ::open(lk, O_CREAT | O_RDWR, 0666); + if (lfd >= 0) { + if (::flock(lfd, LOCK_EX | LOCK_NB) != 0) { + std::fprintf(stderr, "[npu] waiting for the exclusive device lock (%s)...\n", lk); + if (::flock(lfd, LOCK_EX) != 0) + std::fprintf(stderr, "[npu] device lock unavailable; continuing unlocked\n"); + else + std::fprintf(stderr, "[npu] device lock acquired\n"); + } + } + } + xrt::device dev(0); if (!e.fwd->init(dev, model.c_str(), kernels, /*use_elf=*/true)) throw std::runtime_error(std::string("forward init failed: ") + e.fwd->last_error()); From 18909cb07cd516520a4a101105c95f4580e3f37a Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 08:37:14 -0300 Subject: [PATCH 05/12] =?UTF-8?q?npu:=20qwen35moe=20support=20=E2=80=94=20?= =?UTF-8?q?GDN=20+=20MoE=20hybrid=20(Qwen3.6-35B-A3B)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3.6-35B-A3B is a hybrid: 30 GDN (gated delta-net linear attention) layers interleaved with 10 full-attention layers, each FFN a 256-expert top-8 softmax router plus a gated shared expert. Extends the qwen3moe work: - forward: load_gdn_layer now takes the MoE branch (router + pooled experts + the optional shared expert) when n_expert_ > 0, instead of the dense gate/up/down; the GDN layers' FFN was already dispatched by the common MLP section (it sees n_expert_ > 0 and calls moe_mlp). - forward: the shared expert (share_gate/up/down_exps_proj) and its [H] gate (share_router) load in both MoE branches; moe_mlp applies sigmoid(x . g) to the shared expert output (qwen35moe), and leaves it unscaled when absent (deepseek2 has no such gate). - repack: write the GDN geometry (linear_num_key_heads/key_head_dim/ value_head_dim/num_value_heads/conv_kernel_dim from the GGUF ssm.* fields), attn_output_gate (the q_proj emits q|gate) and the per-layer layer_types list the forward dispatches on. Kernels generated for the six new geometries (QKV K2048 N9216 for the gated full-attn, G/U K2048 N512, D K512 N2048, QKV K2048 N8192 (GDN qkv), Z K2048 N4096). Q4NX repacked (21 GB) + the GDN config keys verified. --- npu/forward/include/q4nx_forward.h | 2 +- npu/forward/src/q4nx_forward.cpp | 71 +++++++++++++++++++++++++++++- scripts/repack_gguf.py | 21 +++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 90a8f8609..d4186c30a 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -180,7 +180,7 @@ class Q4nxNpuForward { std::vector w_kb_, w_vb_; // [NH][N][K], N padded for k_b int kb_rows_ = 0; // k_b's padded row count (256) std::vector> mla_kn_, mla_kr_, mla_v_; // per-layer MLA caches - std::vector moe_router_, moe_router_b_, share_g_, share_u_, share_d_; + std::vector moe_router_, moe_router_b_, share_g_, share_u_, share_d_, share_router_; std::vector dense_g_, dense_u_, dense_d_; // leading dense layer only // Routed experts stay packed until selected: per layer the Q4NX pools them as // [n_expert][tiles][5120]. diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 3f53d3c09..492746c1e 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -924,6 +924,19 @@ bool Q4nxNpuForward::load_layer(int l) { if (!pool("model.layers.%d.mlp.gate_exps_proj.weight", &exp_g_, &exp_g_bytes_)) return false; if (!pool("model.layers.%d.mlp.up_exps_proj.weight", &exp_u_, &exp_u_bytes_)) return false; if (!pool("model.layers.%d.mlp.down_exps_proj.weight", &exp_d_, &exp_d_bytes_)) return false; + // Optional shared expert (qwen35moe): a 2D [expert_im][H] / [H][expert_im] pair + // plus its [H] gate. Cleared when absent (qwen3moe has none). + auto load_share = [&](const char* fmt, std::vector& v, int rows, int cols) -> bool { + TensorDesc* d = td2(fmt); + if (!d) return false; + v.resize((size_t)rows * (size_t)cols); + const uint8_t* raw = (const uint8_t*)model_tensor_data(mw_, d); + return raw && dequant_any(raw, (size_t)d->data_size, rows, cols, v.data()) == 0; + }; + if (!load_share("model.layers.%d.mlp.share_gate_exps_proj.weight", share_g_, expert_im_, H)) share_g_.clear(); + if (!load_share("model.layers.%d.mlp.share_up_exps_proj.weight", share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); } + if (!load_share("model.layers.%d.mlp.share_down_exps_proj.weight", share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); } + if (!load_bf16_opt("model.layers.%d.mlp.share_router.weight", share_router_, H)) share_router_.clear(); return true; } if (!deq(&lw->gate_proj_weight, wg_.data(), im, H)) return false; @@ -1001,6 +1014,54 @@ bool Q4nxNpuForward::load_gdn_layer(int l) { } return true; }; + if (n_expert_ > 0) { + // qwen35moe: a GDN layer's FFN is MoE, not a dense gate/up/down. The + // loading mirrors the standard path's MoE branch; the FFN itself is run by + // the common MLP section (which sees n_expert_ > 0 and calls moe_mlp). + char nm2[160]; + auto td2 = [&](const char* fmt) -> TensorDesc* { + snprintf(nm2, sizeof(nm2), fmt, l); + return model_tensor_by_name(mw_, nm2); + }; + auto load_bf16_opt = [&](const char* fmt, std::vector& v, int n) -> bool { + TensorDesc* d = td2(fmt); + if (!d) return false; + const uint16_t* p = (const uint16_t*)model_tensor_data(mw_, d); + if (!p) return false; + v.resize((size_t)n); + for (int i = 0; i < n; i++) v[i] = bf16_to_f32(p[i]); + return true; + }; + if (!load_bf16_opt("model.layers.%d.moe_router.weight", moe_router_, n_expert_ * H)) + { err_ = "moe router"; return false; } + if (!load_bf16_opt("model.layers.%d.moe_router.bias", moe_router_b_, n_expert_)) + moe_router_b_.clear(); + auto pool = [&](const char* fmt, const uint8_t** p, size_t* bytes) -> bool { + TensorDesc* d = td2(fmt); + if (!d) { fprintf(stderr, " [moe] missing %s\n", nm2); return false; } + const uint8_t* raw = (const uint8_t*)model_tensor_data(mw_, d); + if (!raw) return false; + *p = raw; *bytes = (size_t)d->data_size / (size_t)n_expert_; + return true; + }; + if (!pool("model.layers.%d.mlp.gate_exps_proj.weight", &exp_g_, &exp_g_bytes_)) return false; + if (!pool("model.layers.%d.mlp.up_exps_proj.weight", &exp_u_, &exp_u_bytes_)) return false; + if (!pool("model.layers.%d.mlp.down_exps_proj.weight", &exp_d_, &exp_d_bytes_)) return false; + // Optional shared expert (qwen35moe): a 2D [expert_im][H] / [H][expert_im] pair + // plus its [H] gate. Cleared when absent. + auto load_share = [&](const char* fmt, std::vector& v, int rows, int cols) -> bool { + TensorDesc* d = td2(fmt); + if (!d) return false; + v.resize((size_t)rows * (size_t)cols); + const uint8_t* raw = (const uint8_t*)model_tensor_data(mw_, d); + return raw && dequant_any(raw, (size_t)d->data_size, rows, cols, v.data()) == 0; + }; + if (!load_share("model.layers.%d.mlp.share_gate_exps_proj.weight", share_g_, expert_im_, H)) share_g_.clear(); + if (!load_share("model.layers.%d.mlp.share_up_exps_proj.weight", share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); } + if (!load_share("model.layers.%d.mlp.share_down_exps_proj.weight", share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); } + if (!load_bf16_opt("model.layers.%d.mlp.share_router.weight", share_router_, H)) share_router_.clear(); + return true; + } if (!deqT(&lw->gate_proj_weight, wg_.data(), c_.IM, H)) return false; if (!deqT(&lw->up_proj_weight, wu_.data(), c_.IM, H)) return false; if (!deqT(&lw->down_proj_weight, wd_.data(), H, c_.IM)) return false; @@ -1585,7 +1646,15 @@ bool Q4nxNpuForward::moe_mlp(int l, const std::vector& xn, std::vector scale 1). + float sgate = 1.0f; + if (!share_router_.empty()) { + float g = 0.0f; + for (int i = 0; i < H; i++) g += share_router_[i] * xn[i]; + sgate = 1.0f / (1.0f + std::exp(-g)); + } + for (int i = 0; i < H; i++) d[i] += sgate * sd[i]; } if (dump) { double tot = 0; for (float f : d) tot += f; diff --git a/scripts/repack_gguf.py b/scripts/repack_gguf.py index 3f77aaaa4..5c3bb52c5 100644 --- a/scripts/repack_gguf.py +++ b/scripts/repack_gguf.py @@ -144,6 +144,27 @@ def write_config(gguf_path, out_dir, arch, vocab_size): "expert_weights_scale": float(_num(reader, f"{p}.expert_weights_scale", 1.0)), "expert_gating_func": 1, # Qwen MoE gating = softmax }) + if p == "qwen35moe": + # GDN (linear attention) geometry + the layer-type list the forward + # dispatches on. The HF config names these linear_*; the GGUF carries + # them under ssm.*. attn_output_gate makes the q_proj emit q|gate + # (q_rows doubles), which the QKV design must know. + def _ssm(key, dflt=0): + return int(_num(reader, f"{p}.ssm.{key}", dflt)) + _vhd = _ssm("state_size") + _inner = _ssm("inner_size") + _fai = int(_num(reader, f"{p}.full_attention_interval", 4)) or 4 + _nl = int(cfg["num_hidden_layers"]) + cfg.update({ + "linear_num_key_heads": _ssm("group_count"), + "linear_key_head_dim": _vhd, + "linear_value_head_dim": _vhd, + "linear_num_value_heads": (_inner // _vhd) if _vhd else 0, + "linear_conv_kernel_dim": _ssm("conv_kernel", 4), + "attn_output_gate": 1, + "layer_types": ["full_attention" if (i % _fai == _fai - 1) else "linear_attention" + for i in range(_nl)], + }) # MiniCPM4 architecture scales. The forward must apply them: the embedding # is scaled by 12.0 and every residual branch by scale_depth/sqrt(NL) # (0.2475 for 32 layers); without them the hidden state runs at the wrong From 786bda1fa9b40c5323405574b0862adfc435e224 Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 09:36:15 -0300 Subject: [PATCH 06/12] repack: carry the GGUF arch so the provenance stamp check can re-resolve The stamp records converter_path/commit; --check-stamp re-resolves the converter from the dir's config. It used model_type, which for qwen35moe is the HF name 'qwen3_5_moe' (no q4nx/models/qwen3_5_moe.py), so it fell back to the canonical checkout while the dir was built by the iso-build one -> always 'stale' -> the serve discarded and re-repacked the 21 GB dir on every start (and twice hit ENOSPC doing it). config.json now carries "arch" (the GGUF architecture) and the check prefers it. --- scripts/repack_gguf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/repack_gguf.py b/scripts/repack_gguf.py index 5c3bb52c5..3f026b1d7 100644 --- a/scripts/repack_gguf.py +++ b/scripts/repack_gguf.py @@ -81,6 +81,10 @@ def write_config(gguf_path, out_dir, arch, vocab_size): cfg = { "model_type": ARCH_TO_MODEL_TYPE.get(arch, arch), + # The GGUF arch (e.g. qwen35moe) differs from the HF model_type + # (qwen3_5_moe); the stamp check re-resolves the converter by arch, so + # carry it explicitly. + "arch": arch, "hidden_size": embedding, "num_hidden_layers": int(_num(reader, f"{p}.block_count", 0)), "num_attention_heads": heads, @@ -272,7 +276,8 @@ def converter_commit(): # Re-resolve with the directory's own arch so a dir whose arch needs a # different converter keeps comparing against the right one. try: - _a = json.load(open(os.path.join(sys.argv[2], "config.json"))).get("model_type", "") + _c = json.load(open(os.path.join(sys.argv[2], "config.json"))) + _a = _c.get("arch") or _c.get("model_type", "") except Exception: _a = "" if not os.environ.get("ONEBIT_Q4NX_CONVERTER"): From f3b21ec52794f960f37876dbeb960c1a4b4d8727 Mon Sep 17 00:00:00 2001 From: pi agent Date: Mon, 28 Sep 2026 10:57:31 -0300 Subject: [PATCH 07/12] tokenizer: fix the leading-space decode, and accept a Unigram tokenizer.json Two defects on the SentencePiece path (MiniCPM4-8B, the only SentencePiece model of the four blocked targets), both found by testing against oracle ids rather than by reading the code. 1. Every SPM decode gained a spurious leading space. decode() mapped every metaspace to ' ', including the one spm() synthesises as SentencePiece's add_dummy_prefix marker, so 'The capital of France is' -> ' The capital of France is' and 'Paris' -> ' Paris'. Strip exactly that one leading space: stripping one rather than all also preserves a genuinely space-leading input, since ' Paris' encodes to the double metaspace [1345, 3837, 1360] and must decode back to ' Paris'. 2. A correctly-typed tokenizer.json could not be loaded at all. model.at("merges") throws [json.exception.out_of_range.403] when the key is absent, and a SentencePiece/Unigram tokenizer.json has no such key -- the throw aborted the constructor before the SPM detection, so the unigram path was unreachable and every SPM model fell back to raw bytes. Also accept the Unigram list-form vocab ([[token, score], ...], id = position) as well as the BPE object form. Verified with a negative control: the old expression throws on a Unigram file, the new one returns an empty merge set. Test: tests/data/minicpm4_tokenizer_golden.json, whose ids come from llama.cpp's own tokenizer (llama-server /tokenize on MiniCPM4-8B-Q4_K_M.gguf), not from this tokenizer. Results: MiniCPM4 as repacked 5/5 (was 0/3), MiniCPM4 as a proper Unigram file 5/5 (previously unloadable), Qwen3 BPE golden 14/14 unchanged. --- npu/tokenizer.cpp | 35 ++++++++++++++++++++--- tests/data/minicpm4_tokenizer_golden.json | 25 ++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 tests/data/minicpm4_tokenizer_golden.json diff --git a/npu/tokenizer.cpp b/npu/tokenizer.cpp index ddaa59441..f599951e6 100644 --- a/npu/tokenizer.cpp +++ b/npu/tokenizer.cpp @@ -130,9 +130,22 @@ Tokenizer::Tokenizer(const std::string& path) { const auto root = nlohmann::json::parse(f); const auto& model = root.at("model"); int max_id = -1; - for (const auto& [text, id] : model.at("vocab").items()) { - vocab_[text] = id.get(); - max_id = std::max(max_id, id.get()); + // vocab is either a BPE object {token: id} or a Unigram array [[token, score], ...] + // whose id is the entry's position. A SentencePiece tokenizer.json uses the array + // form (the converter's Unigram branch emits exactly that), so accept both. + const auto& vocab_json = model.at("vocab"); + if (vocab_json.is_array()) { + for (size_t i = 0; i < vocab_json.size(); ++i) { + const auto& ent = vocab_json[i]; + if (!ent.is_array() || ent.empty() || !ent[0].is_string()) continue; + vocab_[ent[0].get()] = int(i); + max_id = std::max(max_id, int(i)); + } + } else { + for (const auto& [text, id] : vocab_json.items()) { + vocab_[text] = id.get(); + max_id = std::max(max_id, id.get()); + } } if (root.contains("added_tokens")) for (const auto& t : root["added_tokens"]) { @@ -148,7 +161,13 @@ Tokenizer::Tokenizer(const std::string& path) { std::sort(added_.begin(), added_.end(), [](const auto& a, const auto& b) { return a.first.size() > b.first.size(); }); // Merges: ["a", "b"] pairs, or "a b" strings in older files. Rank = position. - const auto& merges = model.at("merges"); + // A SentencePiece/Unigram tokenizer.json has no "merges" key AT ALL (see the vocab + // handling above). .at() throws on a missing key, and that throw used to abort the + // constructor before the SPM detection below -- so a correctly-typed Unigram file + // could never reach the unigram path and every SPM model fell back to raw bytes. + // Absent merges means "no merges", which is exactly what the SPM detection keys on. + static const nlohmann::json kNoMerges = nlohmann::json::array(); + const auto& merges = model.contains("merges") ? model.at("merges") : kNoMerges; for (size_t i = 0; i < merges.size(); ++i) { const auto& m = merges[i]; std::string a, b; @@ -320,6 +339,14 @@ std::string Tokenizer::decode(const std::vector& ids) const { else out += t[i++]; } } + // The FIRST metaspace is SentencePiece's add_dummy_prefix marker (spm() + // always synthesises one), not a real space -- strip exactly it. Stripping + // one rather than all also preserves a genuinely space-leading input: + // " Paris" encodes to "\u2581\u2581Paris" and must decode back to " Paris". + // Without this every decoded string came back with a leading space + // (MiniCPM4: "The capital of France is" -> " The capital of France is"), + // which is visible in 1bit serve's output. + if (!out.empty() && out[0] == ' ') out.erase(0, 1); return out; } std::string out; diff --git a/tests/data/minicpm4_tokenizer_golden.json b/tests/data/minicpm4_tokenizer_golden.json new file mode 100644 index 000000000..9f6cd65d9 --- /dev/null +++ b/tests/data/minicpm4_tokenizer_golden.json @@ -0,0 +1,25 @@ +{ + "source": "llama.cpp's own tokenizer (llama-server /tokenize) on MiniCPM4-8B-Q4_K_M.gguf. These are oracle ids, not ids produced by this tokenizer.", + "cases": [ + { + "text": "The capital of France is", + "ids": [1507, 8107, 1379, 8360, 1410] + }, + { + "text": "The", + "ids": [1507] + }, + { + "text": "Paris", + "ids": [11225] + }, + { + "text": "Hello, world!", + "ids": [21045, 59342, 2809, 73] + }, + { + "text": " Paris", + "ids": [1345, 3837, 1360] + } + ] +} From 141650ba7fa7bb8a2d0d738a29118cf15a7d266c Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 12:48:45 -0300 Subject: [PATCH 08/12] registry: cover the GGUF-on-NPU rows and add the anchor re-run harness - tools/registry_check.py: an npu .gguf row resolves under --models and reads its arch from the GGUF header (the GGUF-on-NPU route repacks it itself), instead of being treated as a Q4NX model directory. - registry/check_models.tsv: add the six npu GGUF rows (qwen2, qwen3moe, qwen35moe, deepseek2/glm, minicpm, llama). - registry/checked.json: refresh the Qwen3-0.6B npu-directory entry to the current HEAD. - tests/anchor_rerun.sh: harness to re-run the anchor completion across a set of served models and capture the results. --- registry/check_models.tsv | 6 ++++ registry/checked.json | 4 +-- tests/anchor_rerun.sh | 64 +++++++++++++++++++++++++++++++++++++++ tools/registry_check.py | 12 ++++++-- 4 files changed, 82 insertions(+), 4 deletions(-) create mode 100755 tests/anchor_rerun.sh diff --git a/registry/check_models.tsv b/registry/check_models.tsv index 0b7515c74..1ffaff479 100644 --- a/registry/check_models.tsv +++ b/registry/check_models.tsv @@ -43,3 +43,9 @@ zinc Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf zinc Qwen3.6-35B-A3B-Q8_0.gguf zinc MiniCPM5-1B-Q4_K_M.gguf npu Qwen3-0.6B +npu qwen2.5-7b-instruct-q4_k_m-merged.gguf +npu Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf +npu Qwen3.6-35B-A3B-Q8_0.gguf +npu GLM-4.7-Flash-Q4_K_M.gguf +npu MiniCPM4-8B-Q4_K_M.gguf +npu MiniCPM5-1B-Q4_K_M.gguf diff --git a/registry/checked.json b/registry/checked.json index 4e98e855b..3f3b748a5 100644 --- a/registry/checked.json +++ b/registry/checked.json @@ -126,8 +126,8 @@ "passed": true, "said": "**Paris**", "failed": [], - "date": "2026-09-25", - "engine": "8c2805d977eb" + "date": "2026-09-28", + "engine": "97a43e2b1b2c" }, { "backend": "vulkan", diff --git a/tests/anchor_rerun.sh b/tests/anchor_rerun.sh new file mode 100755 index 000000000..e0de0a720 --- /dev/null +++ b/tests/anchor_rerun.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Re-run the anchor completion on `1bit serve` for the target models. +# usage: anchor_rerun.sh <1bit> : [: ...] +set -uo pipefail + +bin=${1:?usage: anchor_rerun.sh <1bit> : ...} +outdir=${2:?} +device=${3:?} +shift 3 +mkdir -p "$outdir" + +for spec in "$@"; do + alias=${spec%%:*} + gguf=${spec#*:} + if [ ! -f "$gguf" ]; then + echo "SKIP $alias: gguf missing: $gguf" + continue + fi + port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') + log=$(mktemp /tmp/serve-rerun-${alias}.XXXX.log) + t0=$(date +%s) + "$bin" serve -m "$gguf" --device "$device" --port "$port" --alias "$alias" >"$log" 2>&1 & + pid=$! + code=000 + for _ in $(seq 1 3600); do + code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/health" 2>/dev/null) + [ "$code" = 200 ] && break + kill -0 "$pid" 2>/dev/null || break + sleep 0.5 + done + if [ "$code" != 200 ]; then + echo "FAIL $alias: /health never 200 (code=$code); log=$log" + echo "--- tail ---"; tail -25 "$log" + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null + continue + fi + reply=$(curl -s "http://127.0.0.1:$port/v1/completions" \ + -d "{\"model\":\"$alias\",\"prompt\":\"The capital of France is\",\"max_tokens\":1,\"temperature\":0}") + t1=$(date +%s) + echo "$reply" > "$outdir/rerun_${alias}.json" + text=$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["choices"][0]["text"])' "$reply" 2>/dev/null) + if [[ "$text" == *"Paris"* ]]; then + echo "PASS $alias: text='$text' ($((t1-t0))s) -> $outdir/rerun_${alias}.json" + else + echo "FAIL $alias: text='$text' (full: $(head -c 300 <<<"$reply"))" + fi + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null +done +echo "done" diff --git a/tools/registry_check.py b/tools/registry_check.py index 2b8460026..440fdb921 100755 --- a/tools/registry_check.py +++ b/tools/registry_check.py @@ -103,11 +103,19 @@ def main(): for backend, model, extra in rows: if a.only and backend != a.only: continue - path = os.path.join(a.npu_models if backend == "npu" else a.models, model) + # An npu row is either a Q4NX model directory (under --npu-models, the + # existing fast lane) or a .gguf the GGUF-on-NPU route repacks itself + # (under --models). The arch comes from model.json's model_type for the + # former and the GGUF's general.architecture for the latter. + npu_gguf = backend == "npu" and model.endswith(".gguf") + if npu_gguf: + path = os.path.join(a.models, model) + else: + path = os.path.join(a.npu_models if backend == "npu" else a.models, model) if not os.path.exists(path): print(f"skip {backend} {model}: not found") continue - arch = npu_model_type(path) if backend == "npu" else gguf_arch(path) + arch = gguf_arch(path) if (backend != "npu" or npu_gguf) else npu_model_type(path) cmd = [os.path.join(ROOT, "tests/serve_e2e.sh"), a.bin, path, backend] + extra try: p = subprocess.run(cmd, capture_output=True, text=True, timeout=a.timeout) From 82044d01b3c9ab2865aa6894d55ba64604421dda Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 14:46:03 -0300 Subject: [PATCH 09/12] registry: record the six GGUF-on-NPU arches in the npu column MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GGUF-on-NPU route serves six arches, each verified by a captured `--device npu` run (goal muhz0s39-8c9l9o). Record them so the registry reflects what the NPU actually carries — text decoders only: - qwen2 (Qwen2.5-7B), qwen3moe (Qwen3-Coder-30B), qwen3_5_moe (Qwen3.6-35B), deepseek2 (GLM-4.7-Flash), minicpm (MiniCPM4-8B), llama (MiniCPM5-1B). Regenerates registry/architectures.json (npu count 1 -> 7). --- registry/architectures.json | 26 +++++++++++++++++++------- tools/registry_build.py | 18 +++++++++++++++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/registry/architectures.json b/registry/architectures.json index 4c10f96a8..b2e057c8a 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -7,7 +7,7 @@ }, "counts": { "hrx": 300, - "npu": 1, + "npu": 7, "vulkan": 323, "zinc": 64, "architectures": 323, @@ -312,8 +312,10 @@ "gguf": "deepseek2", "backends": [ "hrx", + "npu", "vulkan" - ] + ], + "npu_model_type": "deepseek2" }, "DeepseekV32ForCausalLM": { "gguf": "deepseek32", @@ -1182,9 +1184,11 @@ "gguf": "llama", "backends": [ "hrx", + "npu", "vulkan", "zinc" - ] + ], + "npu_model_type": "llama" }, "LlamaForCausalLMEagle3": { "gguf": "llama", @@ -1322,8 +1326,10 @@ "gguf": "minicpm", "backends": [ "hrx", + "npu", "vulkan" - ] + ], + "npu_model_type": "minicpm" }, "MiniCPMV4_6ForConditionalGeneration": { "gguf": "qwen35", @@ -1725,9 +1731,11 @@ "gguf": "qwen2", "backends": [ "hrx", + "npu", "vulkan", "zinc" - ] + ], + "npu_model_type": "qwen2" }, "Qwen2MoEForCausalLM": { "gguf": "qwen2moe", @@ -1825,9 +1833,11 @@ "gguf": "qwen3moe", "backends": [ "hrx", + "npu", "vulkan", "zinc" - ] + ], + "npu_model_type": "qwen3moe" }, "Qwen3MoeModel": { "gguf": "qwen3moe", @@ -1907,9 +1917,11 @@ "gguf": "qwen35moe", "backends": [ "hrx", + "npu", "vulkan", "zinc" - ] + ], + "npu_model_type": "qwen3_5_moe" }, "Qwen3_5MoeForConditionalGeneration": { "gguf": "qwen35moe", diff --git a/tools/registry_build.py b/tools/registry_build.py index 0d2fd92b9..7ff95d154 100755 --- a/tools/registry_build.py +++ b/tools/registry_build.py @@ -53,9 +53,21 @@ # on that build's Vulkan0. Keep in step with fork_only_arch in app/serve.cpp. FORK_ONLY = {"zaya", "opt", "codegen", "gptneo", "gptj"} -# The NPU fast lane serves Qwen3 dense Q4NX directories (docs/npu.md). HF architecture -> -# model_type, as the directory's config.json names them. -NPU_ARCHS = {"Qwen3ForCausalLM": "qwen3"} +# The NPU serves these HF architectures (docs/npu.md): +# - the fast lane serves Qwen3 dense Q4NX directories (model_type from config.json); +# - the GGUF-on-NPU route repacks a .gguf and runs the model-generic forward, for six +# arches each verified by a captured `--device npu` run (goal muhz0s39-8c9l9o). Text +# decoders only; vision-language variants of these arches are not claimed. +# HF architecture -> model_type the forward's config.json names. +NPU_ARCHS = { + "Qwen3ForCausalLM": "qwen3", # fast lane (Q4NX directory) + "Qwen2ForCausalLM": "qwen2", # GGUF-on-NPU: Qwen2.5-7B-Instruct + "Qwen3MoeForCausalLM": "qwen3moe", # GGUF-on-NPU: Qwen3-Coder-30B-A3B + "Qwen3_5MoeForCausalLM": "qwen3_5_moe", # GGUF-on-NPU: Qwen3.6-35B-A3B (gguf qwen35moe) + "DeepseekV2ForCausalLM": "deepseek2", # GGUF-on-NPU: GLM-4.7-Flash + "MiniCPMForCausalLM": "minicpm", # GGUF-on-NPU: MiniCPM4-8B + "LlamaForCausalLM": "llama", # GGUF-on-NPU: MiniCPM5-1B +} def pin(path): From 634bab953936e6f4d42a269d2bef4a444cd3fbab Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 28 Sep 2026 18:47:17 -0300 Subject: [PATCH 10/12] npu forward: LM-head tile widening + profiler hook; opt-in forward->lane routing - run_lm_head picks the widest design with K=H (G/U geometry, N=18944) instead of the first (QKV): 9 head launches per token instead of 33, ~24 fewer synchronous NPU launches (each ~131 ms). Captured qwen2: 0.018 -> 0.020 tok/s, anchor ' Paris' preserved; the five other arches re-verified (evidence/gguf-npu-six/after-speed/). - npu_gemm gains an env-gated NPU_PROFILE hook: ~173 dispatches/token, each ~183 ms (52 pack+quant, 2.3 quant+launch, 131 wait, 0.1 readback). - app/serve.cpp: opt-in forward->lane routing (ONEBIT_NPU_LANE_DIR) for models with a matching fused-lane kernel set; off by default because the repack data packing differs from the lane kernels' (lane ' in the city of' vs forward ' Paris'). Goal mulmuy25-vnl62c. --- app/serve.cpp | 28 ++++++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 34 ++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/app/serve.cpp b/app/serve.cpp index f31c07b26..0cffffb3e 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -1746,6 +1746,34 @@ int run_serve(int argc, char** argv) { // GGUF-on-NPU (docs/npu.md, "Open"): repack the GGUF into a Q4NX model // directory, then serve it through the model-generic forward in process. const std::string dir = repack_gguf(o.model); + // Forward -> lane routing: if a fused-lane kernel set exists for this + // model's shape (the fast-lane artifacts layer_ctx1.elf + layer.pdi), + // serve it on the NPU fast lane (one launch per layer) instead of the + // per-op forward (~100+ launches per token). Opt-in via + // ONEBIT_NPU_LANE_DIR: the repack layout must equal the lane kernels' + // layout, and it does not yet (the repack is patched for the forward's + // per-op ELFs), so this must not fire by accident. Without it, the + // forward runs (the unchanged fallback). + { + std::string lane; + if (const char* env = std::getenv("ONEBIT_NPU_LANE_DIR")) lane = env; + if (!lane.empty() && fs::exists(fs::path(lane) / "layer_ctx1.elf")) { + if (!fs::exists(fs::path(dir) / "npu")) { + std::error_code ec; + fs::create_directory_symlink(lane, fs::path(dir) / "npu", ec); + if (ec) std::fprintf(stderr, "1bit serve: lane routing skipped (%s)\n", ec.message().c_str()); + } + if (fs::exists(fs::path(dir) / "npu" / "layer_ctx1.elf")) { + std::fprintf(stderr, "1bit serve: routing %s to the NPU fast lane (fused layer) from %s\n", + o.model.c_str(), lane.c_str()); + std::vector args = {"-m", dir, "-p", std::to_string(o.port), "--host", o.host}; + if (!o.alias.empty()) { args.push_back("--alias"); args.push_back(o.alias); } + std::vector av; + for (auto& s : args) av.push_back(s.data()); + return run_unified(int(av.size()), av.data()); + } + } + } // Resolve the full-ELF / xclbin directory the forward loads. An empty value // makes it look for "/full_i8_*.elf", so it is an error, not a default. std::string kernels; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 492746c1e..34b43eecc 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -27,6 +27,7 @@ #include "q4nx_forward.h" #include +#include #include #include #include @@ -1299,6 +1300,8 @@ void Q4nxNpuForward::npu_gemm(I8Ctx& ctx, const float* A, int K, int N, if (a > amax) amax = a; } const float as = amax > 0.0f ? amax / 127.0f : 1.0f; + const bool prof = getenv("NPU_PROFILE") != nullptr; + const auto tp = std::chrono::steady_clock::now(); // Token-invariant weights (wkey >= 0) get their own BO, packed once and // reused; every other call (wkey < 0: routed experts, whose weights change @@ -1325,14 +1328,30 @@ void Q4nxNpuForward::npu_gemm(I8Ctx& ctx, const float* A, int K, int N, gs = &ctx.group_scales[0]; } + const auto tl = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; xrt::run r = slot ? ctx.launch_async_with_bo(*slot->bo, A, 1, K, as) : ctx.sync_and_launch(0); + const auto tq = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; ctx.wait_kernel(r); + const auto tw = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; ctx.readback(); + const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)N); for (int n = 0; n < N; n++) out[(size_t)n] = (float)ctx.Cm[n] * as * (*gs)[(size_t)n]; + + if (prof) { + static double sp = 0.0, sq = 0.0, sw = 0.0, sr = 0.0; + static long sn = 0; + sp += std::chrono::duration(tl - tp).count(); + sq += std::chrono::duration(tq - tl).count(); + sw += std::chrono::duration(tw - tq).count(); + sr += std::chrono::duration(tr - tw).count(); + if (++sn % 50 == 0) + std::fprintf(stderr, "[prof] npu_gemm n=%ld pack=%.1f quant+launch=%.1f wait=%.1f readback=%.1f ms (N=%d)\n", + sn, sp / (double)sn, sq / (double)sn, sw / (double)sn, sr / (double)sn, N); + } } // -------------------------------------------------------------------------- @@ -2223,12 +2242,15 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // read out of designs_[-1] -- garbage, and the ntiles division below raised // SIGFPE. Only this NPU path uses T (the host path is a plain matvec), which is // why GLM crashed here the first time the ELF path was exercised. - int i_qkv = idx_of(Q4NX_DESIGN_QKV); - if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) - i_qkv = idx_of(Q4NX_DESIGN_DG); // K=H, N=dense_im - if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) - i_qkv = idx_of(Q4NX_DESIGN_G); // K=H, N=expert_im - if (i_qkv < 0 || designs_[i_qkv].K != H || designs_[i_qkv].N <= 0) { + // Pick the WIDEST design with K=H for the head. Each tile is a separate + // synchronous launch (~one NPU round-trip each, ~constant regardless of N), + // so fewer/larger tiles mean fewer launches for the same math. + int i_qkv = -1; + for (int d = 0; d < ndesigns_; d++) { + if (designs_[d].K != H || designs_[d].N <= 0) continue; + if (i_qkv < 0 || designs_[d].N > designs_[i_qkv].N) i_qkv = d; + } + if (i_qkv < 0) { err_ = "LM head: no GEMM design with K=H available"; return false; } From ca9381d209afd3f473c705ef15edccf90ae038fc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 28 Sep 2026 20:03:43 -0300 Subject: [PATCH 11/12] tokenizer: a streamed token keeps its leading space (SentencePiece) 97a43e2 strips the first metaspace (SentencePiece's add_dummy_prefix marker) from every decode. Both servers decode one generated token at a time (app/forward_serve.cpp, app/unified.cpp), so every word-initial token lost its space: MiniCPM4 streamed "...-TheUnitedStates". The marker belongs to a whole text's first token only; in a stream the space after the prompt or the previous word is real. decode(ids) keeps stripping it (whole texts, the tokenizer goldens); decode(id), the stream's form, keeps it. Co-Authored-By: Claude Opus 5.5 --- npu/tokenizer.cpp | 4 ++-- npu/tokenizer.h | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/npu/tokenizer.cpp b/npu/tokenizer.cpp index f599951e6..26d11e10d 100644 --- a/npu/tokenizer.cpp +++ b/npu/tokenizer.cpp @@ -321,7 +321,7 @@ std::vector Tokenizer::encode(const std::string& text) const { return ids; } -std::string Tokenizer::decode(const std::vector& ids) const { +std::string Tokenizer::decode(const std::vector& ids, bool strip_dummy_prefix) const { if (spm_) { static const char kMeta[] = "\xE2\x96\x81"; std::string out; @@ -346,7 +346,7 @@ std::string Tokenizer::decode(const std::vector& ids) const { // Without this every decoded string came back with a leading space // (MiniCPM4: "The capital of France is" -> " The capital of France is"), // which is visible in 1bit serve's output. - if (!out.empty() && out[0] == ' ') out.erase(0, 1); + if (strip_dummy_prefix && !out.empty() && out[0] == ' ') out.erase(0, 1); return out; } std::string out; diff --git a/npu/tokenizer.h b/npu/tokenizer.h index 9b9e010f3..7a8c585ee 100644 --- a/npu/tokenizer.h +++ b/npu/tokenizer.h @@ -39,8 +39,11 @@ class Tokenizer { std::vector encode(const std::string& text) const; // Ids outside the vocabulary decode to nothing. - std::string decode(const std::vector& ids) const; - std::string decode(int id) const { return decode(std::vector{id}); } + // A SentencePiece vocabulary's first metaspace is its add_dummy_prefix marker, stripped when + // decoding a whole text. One token of a stream keeps it: a word-initial token's space is real + // there (after the prompt, or after the previous word). + std::string decode(const std::vector& ids, bool strip_dummy_prefix = true) const; + std::string decode(int id) const { return decode(std::vector{id}, false); } // One past the largest id, added tokens included. int size() const { return int(id_to_text_.size()); } // The vocabulary id for a token string (added tokens included), or -1 if absent. From 7dfccadd18d0417c2535a7bfaf5d270fe8f975e9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 28 Sep 2026 20:07:43 -0300 Subject: [PATCH 12/12] npu serve: open the device lock read-only at a fixed path (CodeQL) flock works on a read-only descriptor, so the lock file no longer needs to be world-writable, and the path no longer comes from the environment. O_NOFOLLOW refuses a symlink planted at the /tmp path. Co-Authored-By: Claude Opus 5.5 --- app/forward_serve.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 6657596f6..b6c93cf87 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -286,9 +286,10 @@ int run_forward_serve(int argc, char** argv) { // against a sibling serve. Held for the life of the process (fd not closed), // exactly like the runner. if (!std::getenv("NPU_NO_DEVICE_LOCK")) { - const char* lk = std::getenv("NPU_DEVICE_LOCK"); - if (!lk || !lk[0]) lk = "/tmp/1bit-npu-device.lock"; - int lfd = ::open(lk, O_CREAT | O_RDWR, 0666); + // flock needs only a read fd, so the file stays 0644 and other users can + // still open it; O_NOFOLLOW refuses a symlink planted in /tmp. + const char* lk = "/tmp/1bit-npu-device.lock"; + int lfd = ::open(lk, O_CREAT | O_RDONLY | O_NOFOLLOW | O_CLOEXEC, 0644); if (lfd >= 0) { if (::flock(lfd, LOCK_EX | LOCK_NB) != 0) { std::fprintf(stderr, "[npu] waiting for the exclusive device lock (%s)...\n", lk);