From 4240f158db731303ff35471a3a627066fee7ed14 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:32:10 -0400 Subject: [PATCH] test-backend-ops: space TOPK_MOE's logits, so the ids comparison cannot turn on a last bit (engine #1259) `test_topk_moe` compares the expert ids it selected with the CPU reference's, as an exact index set. With random logits in [-1,1] and 256 experts over 6 routes, the 6th and 7th probabilities are within a few ULPs often enough to matter, and then the two producers legitimately disagree: the CPU gathers `probs` from one softmax and sorts it, the HRX path computes the same quantity in another place, and a last-bit difference flips their order. The ids differ, the weights differ by ~1e-7 (inside this test's nmse tolerance), so the report is "[VIEW] ids only, tiny ERR" - which is exactly what the row at pin a3e8d42c9 looked like: `TOPK_MOE(ne=[256,22,1,1],n_expert_used=6,with_norm=1,bias_probs=0,gating_func=3)`, `[VIEW] ERR = 2.5e-4`, ~2 full-suite runs in 9, and 12/12 clean when the case is run alone. Measured, not asserted (simulation of this test's own data, numpy float32, 4000 draws per point): exact tie at the k-th boundary, gating_func=3 0.05 % per case -> 0.16 % per full-suite run (cannot explain it) adjacent gap <= 4e-7 relative, gating_func=3 0.12 % per case -> 33 % per full-suite run (matches 2/9) adjacent gap <= 4e-7 relative, softmax 0.10 % per case -> 27 % per full-suite run So the first explanation I wrote for this row - "exact ties, and the reference's tie order is undefined because `ggml_compute_forward_argsort_f32` sorts with an unstable, value-only comparator" - is arithmetically too rare to be the cause (it stays true, it is just not what happens here), and this commit does not rest on it. The fill keeps every other tensor as the base class fills it and gives the logits exactly representable values 2^-9 apart, shuffled per row so the order is not monotone. Adjacent probabilities then differ by ~1e-3, three orders above the few-ULP window, so an id can only differ if a producer is genuinely wrong - which is what the comparison is for. Nothing is skipped, waived or tolerance-widened. Refs #1259 --- tests/test-backend-ops.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6064fa57759..82b2db89250 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -6082,6 +6082,7 @@ struct test_topk_moe : public test_case { const float scale_w; ggml_tensor * weights {}; ggml_tensor * selected_experts {}; + ggml_tensor * logits {}; test_topk_moe(std::array ne = { 10, 5, 1, 1 }, int n_expert_used = 1, @@ -6112,6 +6113,7 @@ struct test_topk_moe : public test_case { const int n_tokens = ne[1]; ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + this->logits = logits; ggml_tensor * probs = (gating_func == GATING_FUNC_SOFTMAX) ? ggml_soft_max(ctx, logits) : (gating_func == GATING_FUNC_SIGMOID) ? ggml_sigmoid(ctx, logits) : @@ -6156,6 +6158,32 @@ struct test_topk_moe : public test_case { return weights; } + void initialize_tensors(ggml_context * ctx) override { + // Base fill first, so every other tensor keeps the coverage it had. + test_case::initialize_tensors(ctx); + if (logits == nullptr) { + return; + } + // The ids comparison below asserts an *exact* index set, which with random logits can turn on a last bit: with 256 + // experts over 6 routes the 6th and 7th values are within a few ULPs in ~0.1 % of cases (~33 % of full-suite runs), + // and the CPU and HRX paths compute those probabilities in different places, so the order - and the id - flips while + // the gathered weight moves by ~1e-7, inside this test's tolerance. That is the shape of the row that flaked at pin + // a3e8d42c9 (engine #1259). Space the logits instead: exactly representable values 2^-9 apart, shuffled per row, so + // adjacent probabilities differ by ~1e-3 and an id can only differ if a producer is genuinely wrong. + const int64_t n = logits->ne[0]; + std::vector data(ggml_nelements(logits)); + std::vector row(n); + for (int64_t i = 0; i < n; ++i) { + row[i] = (float) (i - n / 2) * (1.0f / 512.0f); + } + std::mt19937 rng(0x1259u + (unsigned) n); + for (int64_t r = 0; r < ggml_nrows(logits); ++r) { + std::shuffle(row.begin(), row.end(), rng); + std::copy(row.begin(), row.end(), data.begin() + r * n); + } + ggml_backend_tensor_set(logits, data.data(), 0, ggml_nbytes(logits)); + } + // Verify two outputs std::vector fusion_test_nodes() override { return { selected_experts, weights }; }