diff --git a/docs/npu.md b/docs/npu.md index 0ab1ebf9b..801de1c98 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -363,6 +363,17 @@ is otherwise served a chat prompt with no BOS: measured on ZR1-1.5B, the forward 5-token prompt where llama.cpp built 6, and the reply opened with the end-of-turn token. `tools/safetensors_to_q4nx.py` carries the same two keys from `tokenizer_config.json`. +**A GGUF that declares a RoPE scaling block is refused.** This route has no scaling rule — the +packed lane computes `theta^(-i/half)` and nothing else — and a container written from a scaled +GGUF would carry `rope_theta` alone, be served plain RoPE, stay fluent and be wrong past the +model's original window, with nothing in any log to say so ([engine#1087](https://github.com/1bit-MONSTER/engine/issues/1087)). +So `pack` exits non-zero naming the type (`*.rope.scaling.type` = `yarn`/`linear`/`longrope`/…) +before it creates the output directory. Two rules *are* implemented and are accepted on the +forward route, which reads the block from `config.json`: `rope_type: "llama3"` +(Llama-3.1/3.2/3.3) and LongRoPE's `short_factor` (MiniCPM4). The same refusal covers a +declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` / +`forward-serve` load time rather than at request time. + | GGUF type | To q4_1 | Error | |---|---|---| | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero | diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 9fe56abe2..36d666b7c 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -37,6 +37,8 @@ */ #pragma once +#include + #ifdef __cplusplus extern "C" { #endif @@ -69,6 +71,76 @@ static inline double q4nx_rope_llama3_inv_freq(double inv, const Q4nxRopeLlama3* return (1.0 - smooth) * inv / s->factor + smooth * inv; } +/* The scaling a config.json DECLARES, and whether this forward implements it (engine#1087). + * + * A declared block that is not applied is worse than a failed load: the forward runs plain + * theta^(-i/half), the reply stays fluent and is wrong past the model's original window, and no line + * in the log says the model was served something other than it asked for. Two rules are implemented + * — `llama3` (above) and LongRoPE's `short_factor` (read_rope_short_factor in q4nx_forward.cpp) — so + * every other declared type is refused at load BY NAME instead. + * + * q4nx_rope_declared_type() writes "" when the config declares no scaling block at all (the case for + * every model that has no rope_scaling key), and the declared type otherwise. + */ +static inline void q4nx_rope_declared_type(const char* json, char* out, int cap) { + if (!out || cap <= 0) return; + out[0] = '\0'; + if (!json) return; + const char* rs = strstr(json, "\"rope_scaling\""); + if (!rs) return; + const char* v = strchr(rs, ':'); + if (!v) return; + for (++v; *v == ' ' || *v == '\t' || *v == '\n' || *v == '\r'; ++v) { + } + if (*v != '{') return; /* null (or a non-object) declares nothing */ + /* The window is the block's OWN closing brace, not a byte count. A fixed window is wrong in both + * directions: a block that serialises its factor arrays before the type key can push "rope_type" + * past it (which would read as NO block, i.e. served plain — the failure this reader exists to + * stop), and a block that has no type at all can reach a NEIGHBOUR's "type" inside it (which would + * refuse a model that asked for nothing). Braces are matched through strings, so a "}" inside a + * string cannot end the block early. */ + int depth = 0, instr = 0, esc = 0; + const char* end = 0; + for (const char* q = v; *q; ++q) { + const char ch = *q; + if (instr) { + if (esc) esc = 0; + else if (ch == '\\') esc = 1; + else if (ch == '"') instr = 0; + continue; + } + if (ch == '"') instr = 1; + else if (ch == '{' || ch == '[') ++depth; + else if ((ch == '}' || ch == ']') && --depth == 0) { end = q; break; } + } + if (!end) return; /* unterminated block: nothing trustworthy to refuse */ + const char* p = strstr(v, "\"rope_type\""); /* current HF spelling */ + if (!p || p > end) p = strstr(v, "\"type\""); /* pre-4.45 spelling: {"type": "yarn"} */ + if (!p || p > end) return; + const char* c = strchr(p, ':'); + if (!c || c > end) return; + const char* q = strchr(c, '"'); + if (!q || q > end) return; + const char* e = strchr(q + 1, '"'); + if (!e || e > end) return; + int n = (int)(e - q - 1); + if (n > cap - 1) n = cap - 1; + memcpy(out, q + 1, (size_t)n); + out[n] = '\0'; +} + +/* 1 when the declared type is one this forward applies, 0 when it must be refused. `llama3` needs + * usable numbers (a declared-but-unusable block is not the same thing as no block) and `longrope` + * needs the short_factor the forward reads. */ +static inline int q4nx_rope_type_supported(const char* type, const Q4nxRopeLlama3* llama3, + int has_short_factor) { + if (!type || !type[0]) return 1; /* no block declared: plain rope */ + if (!strcmp(type, "default") || !strcmp(type, "none")) return 1; + if (!strcmp(type, "llama3")) return llama3 && llama3->on; + if (!strcmp(type, "longrope")) return has_short_factor ? 1 : 0; + return 0; /* yarn, linear, dynamic, mrope, ... */ +} + #ifdef __cplusplus } #endif diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index da9258a4f..8831189b8 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -544,6 +544,27 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, } rope_scale_ = read_rope_short_factor(cfg_path); // LongRoPE (MiniCPM4) rope_llama3_ = read_rope_llama3(cfg_path); // rope_type "llama3" (engine#1048) + { + // A declared scaling this forward does not apply is refused BY NAME (engine#1087): serving it + // would run plain rope, stay fluent and be wrong past the original window, with nothing in the + // log to say so. "" (no rope_scaling block) is every model that never declared one. + std::ifstream f(cfg_path); + std::string t((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + char declared[64]; + q4nx_rope_declared_type(t.c_str(), declared, (int)sizeof declared); + if (!q4nx_rope_type_supported(declared, &rope_llama3_, rope_scale_.empty() ? 0 : 1)) { + if (!std::strcmp(declared, "llama3")) + err_ = "config.json declares rope_type \"llama3\" but its numbers are not usable (factor > 0, " + "original_max_position_embeddings > 0, high_freq_factor > low_freq_factor), so the " + "scaling cannot be applied; serving it would run plain RoPE"; + else + err_ = std::string("config.json declares rope scaling \"") + declared + + "\", which this forward does not implement (only llama3 and longrope short_factor " + "are), so it would be served plain RoPE and reply wrongly past the model's " + "original window (engine#1048, #1087)"; + return false; + } + } embed_scale_ = read_config_float(cfg_path, "\"embedding_scale\"", 1.0f); residual_scale_ = read_config_float(cfg_path, "\"residual_scale\"", 1.0f); logit_scale_ = read_config_float(cfg_path, "\"logit_scale\"", 1.0f); diff --git a/tests/gguf_to_q4nx_test.py b/tests/gguf_to_q4nx_test.py index a851077d2..7b4cbc036 100755 --- a/tests/gguf_to_q4nx_test.py +++ b/tests/gguf_to_q4nx_test.py @@ -85,7 +85,7 @@ def q4_k_blocks(rng, N, K): return b.reshape(N, nb * 144) -def synthetic(path, rng): +def synthetic(path, rng, rope_scaling=None): H, IM, NH, NKV, HD, V = 256, 512, 2, 1, 128, 512 w = gguf.GGUFWriter(path, "qwen3") w.add_block_count(1) @@ -97,6 +97,10 @@ def synthetic(path, rng): w.add_key_length(HD) w.add_value_length(HD) w.add_rope_freq_base(1e6) + if rope_scaling: + # what llama.cpp writes for a model whose config.json carries rope_scaling + w.add_rope_scaling_type(getattr(gguf.RopeScalingType, rope_scaling.upper())) + w.add_rope_scaling_factor(4.0) w.add_layer_norm_rms_eps(1e-6) w.add_eos_token_id(1) # The two flags app/forward_serve.cpp builds the prompt from; add_bos_token is the one a @@ -446,6 +450,21 @@ def dense_3d_pack_check(gguf_path, out, ok): return ok and good and not left +def rope_scaling_check(gguf_path, out, kind, ok): + """A declared rope scaling this route cannot apply is refused by name, not silently dropped.""" + tool = os.path.join(HERE, "..", "tools", "gguf_to_q4nx.py") + r = subprocess.run([sys.executable, tool, "pack", gguf_path, out], capture_output=True, text=True) + msg = r.stdout + r.stderr + good = r.returncode != 0 and "rope.scaling.type" in msg and kind in msg + print(f"{'ok ' if good else 'FAIL'} packing a {kind}-scaled GGUF exits non-zero and names the " + f"type (rc {r.returncode})") + if not good: + print(msg[-800:]) + left = os.path.exists(os.path.join(out, "model.q4nx")) + print(f"{'ok ' if not left else 'FAIL'} no model.q4nx was written for it") + return ok and good and not left + + def main(): ok = golden_tile() ok = golden_tile8() and ok @@ -462,6 +481,11 @@ def main(): print(r.stderr, end="") print(f"FAIL {cmd[0]} exited {r.returncode}") ok = False + yarn = os.path.join(d, "yarn.gguf") + for kind in ("yarn", "linear", "longrope"): + scaled = os.path.join(d, f"{kind}.gguf") + synthetic(scaled, rng, rope_scaling=kind) + ok = rope_scaling_check(scaled, os.path.join(d, f"{kind}-out"), kind, ok) mout = os.path.join(d, "moe") moe_synthetic(os.path.join(d, "tiny-moe.gguf"), rng) for cmd in (["pack", os.path.join(d, "tiny-moe.gguf"), mout], diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index cc9f52df8..1a20c9dd0 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -31,6 +31,7 @@ #include #include +#include static int failures = 0; @@ -97,6 +98,105 @@ static int test_table(const char* tag, int half, double theta, double factor, do return 0; } +/* The declared scaling type, and whether it is one this forward applies (engine#1087). A type it does + * not implement must be refused by name: served plain it stays fluent and is wrong past the original + * window. The reader is exercised on the config shapes that exist in the wild, including the two + * spellings of the key and a config whose "model_type" must not be mistaken for a scaling type. */ +static void test_declared_type(void) { + struct { + const char* json; + const char* type; + int llama3_on; + int has_short; + int supported; + } cases[] = { + /* no rope_scaling at all: plain rope, and "model_type" is not a scaling type */ + {"{\"model_type\": \"qwen3\", \"rope_theta\": 1000000.0}", "", 0, 0, 1}, + /* the pack-imported Llama-3.2 config, verbatim shape */ + {"{\"rope_theta\": 500000.0, \"rope_scaling\": {\"factor\": 32.0, \"high_freq_factor\": 4.0, " + "\"low_freq_factor\": 1.0, \"original_max_position_embeddings\": 8192, \"rope_type\": \"llama3\"}}", + "llama3", 1, 0, 1}, + /* declared llama3 whose numbers are unusable: not the same thing as no block */ + {"{\"rope_scaling\": {\"factor\": 0.0, \"high_freq_factor\": 4.0, \"low_freq_factor\": 1.0, " + "\"original_max_position_embeddings\": 8192, \"rope_type\": \"llama3\"}}", + "llama3", 0, 0, 0}, + /* YaRN, both spellings: refused */ + {"{\"rope_scaling\": {\"factor\": 4.0, \"original_max_position_embeddings\": 32768, " + "\"rope_type\": \"yarn\"}}", + "yarn", 0, 0, 0}, + {"{\"rope_scaling\": {\"type\": \"yarn\", \"factor\": 4.0}}", "yarn", 0, 0, 0}, + /* dynamic NTK and linear: refused */ + {"{\"rope_scaling\": {\"type\": \"dynamic\", \"factor\": 2.0}}", "dynamic", 0, 0, 0}, + {"{\"rope_scaling\": {\"type\": \"linear\", \"factor\": 8.0}}", "linear", 0, 0, 0}, + /* explicit no-scaling spellings: supported */ + {"{\"rope_scaling\": null}", "", 0, 0, 1}, + /* null followed by another block that does carry a "type": still nothing declared */ + {"{\"rope_scaling\": null, \"other\": {\"type\": \"yarn\"}}", "", 0, 0, 1}, + {"{\"rope_scaling\": {\"type\": \"none\"}}", "none", 0, 0, 1}, + {"{\"rope_scaling\": {\"rope_type\": \"default\"}}", "default", 0, 0, 1}, + /* LongRoPE: only with the short_factor the forward reads */ + {"{\"rope_scaling\": {\"rope_type\": \"longrope\", \"short_factor\": [1.0, 1.1]}}", "longrope", 0, + 1, 1}, + {"{\"rope_scaling\": {\"rope_type\": \"longrope\"}}", "longrope", 0, 0, 0}, + }; + for (size_t i = 0; i < sizeof cases / sizeof cases[0]; i++) { + char type[64]; + q4nx_rope_declared_type(cases[i].json, type, (int)sizeof type); + Q4nxRopeLlama3 s; + s.on = cases[i].llama3_on; + s.factor = 32.0; + s.high_freq_factor = 4.0; + s.low_freq_factor = 1.0; + s.original_max_position = 8192.0; + const int got = q4nx_rope_type_supported(type, &s, cases[i].has_short); + char buf[192]; + snprintf(buf, sizeof buf, "declared type \"%s\" reads as \"%s\"", cases[i].type, type); + check(!strcmp(type, cases[i].type), buf); + snprintf(buf, sizeof buf, "declared \"%s\" supported %d == %d", cases[i].type, got, + cases[i].supported); + check(got == cases[i].supported, buf); + } + /* The review's two edges (PR #1090): the window is the block's own closing brace, not a byte count. + * (a) a type that sits past a fixed window must still be found — reading it as "no block" would + * serve the model plain, which is the failure this reader exists to stop (it fails OPEN); + * (b) a block with no type must not take a neighbour's "type" — that refuses a model that asked + * for nothing. */ + { + char big[1600]; + char pad[1000]; + memset(pad, 'x', sizeof pad - 1); + pad[sizeof pad - 1] = '\0'; + snprintf(big, sizeof big, "{\"rope_scaling\": {\"pad\": \"%s\", \"rope_type\": \"yarn\"}}", pad); + char type[64]; + q4nx_rope_declared_type(big, type, (int)sizeof type); + check(!strcmp(type, "yarn"), "a type 1,000 bytes into the block is still read (fails closed)"); + check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0) == 0, + "... and is refused"); + } + { + char type[64]; + q4nx_rope_declared_type("{\"rope_scaling\": {\"factor\": 8.0}, " + "\"quantization_config\": {\"type\": \"fp8\"}}", type, (int)sizeof type); + check(type[0] == '\0', "a block with no type does not take a neighbour's type"); + check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0) == 1, "... and is not refused"); + /* nested arrays/objects before the type, and a '}' inside a string, both keep the block open */ + q4nx_rope_declared_type("{\"rope_scaling\": {\"mrope_section\": [1, 2], \"rope_type\": \"yarn\"}}", + type, (int)sizeof type); + check(!strcmp(type, "yarn"), "a nested array before the type does not end the block"); + q4nx_rope_declared_type("{\"rope_scaling\": {\"note\": \"} not the end\", \"rope_type\": \"yarn\"}}", + type, (int)sizeof type); + check(!strcmp(type, "yarn"), "a '}' inside a string does not end the block"); + q4nx_rope_declared_type("{\"rope_scaling\": {\"rope_type\": \"yarn\"", type, (int)sizeof type); + check(type[0] == '\0', "an unterminated block declares nothing rather than guessing"); + } + + /* no config text at all: nothing declared, nothing to refuse */ + char none[8]; + q4nx_rope_declared_type((const char*)0, none, (int)sizeof none); + check(none[0] == '\0', "null config declares no type"); + check(q4nx_rope_type_supported(none, (const Q4nxRopeLlama3*)0, 0) == 1, "no type is supported"); +} + int main(void) { /* Llama-3.2-1B: head_dim 64, theta 500000, factor 32, high 4, low 1, original_max 8192 */ static const double t1b[32] = { @@ -167,6 +267,8 @@ int main(void) { check(q4nx_rope_llama3_inv_freq(1e-5, &s) == 1e-5, "original_max 0 is refused"); } + test_declared_type(); + if (failures == 0) printf("all llama3 rope checks passed\n"); return failures == 0 ? 0 : 1; } diff --git a/tools/gguf_to_q4nx.py b/tools/gguf_to_q4nx.py index 2cb8a61e1..6d8ab1a0c 100755 --- a/tools/gguf_to_q4nx.py +++ b/tools/gguf_to_q4nx.py @@ -172,11 +172,32 @@ def field(r, key, default=None): return default if f is None else f.contents() +def refuse_unsupported_rope(r): + """A scaling this route cannot apply is refused before anything is written (engine#1087). + + A declared block that is dropped is worse than a failed conversion: the container would carry + rope_theta and nothing else, be served plain RoPE, stay fluent and be wrong past the model's + original window, with no line anywhere saying so. This route has no scaling rule at all (the packed + lane computes theta^(-i/half) and nothing else), so every declared type is refused by name. The + forward implements llama3 and longrope short_factor, but only for a container whose config.json + carries the block, which is a different route (1bit npu-pack import, or a checkpoint's own config). + """ + arch = field(r, "general.architecture") + scaling = field(r, f"{arch}.rope.scaling.type") + if scaling is not None and str(scaling) not in ("none", "default", ""): + raise SystemExit( + f"{arch}.rope.scaling.type is {str(scaling)!r}: this converter has no RoPE scaling rule, so the " + f"container would be served plain RoPE and reply wrongly past the model's original window. " + f"Convert from the checkpoint with its config.json (the forward route reads rope_scaling), or " + f"file the scaling type you need (engine#1087)") + + def config(r, vocab): arch = field(r, "general.architecture") if arch not in ("qwen3", "qwen2", "qwen3moe"): raise SystemExit(f"architecture {arch!r}: only qwen3, qwen2 and qwen3moe are supported") a = lambda k, d=None: field(r, f"{arch}.{k}", d) # noqa: E731 + refuse_unsupported_rope(r) heads = a("attention.head_count") tied = not any(t.name == "output.weight" for t in r.tensors) moe = arch == "qwen3moe" @@ -353,6 +374,7 @@ def moe_meta(r, items, table): def pack(args): r = gguf.GGUFReader(args.gguf) + refuse_unsupported_rope(r) # before the output directory exists: no half-written container os.makedirs(args.out, exist_ok=True) items = planned(r) # --dtype8 names tensors to store as Q8 chunks instead of q4_1. Selection is by Q4NX name,