Steps 3b-3c: NPU fast lane runtime on full ELFs, served through Lemonade

.github/workflows/ci.yml

@@ -25,10 +25,10 @@ jobs:
       - name: Copyright notice on every file
         run: python3 tools/copyright.py --check
       - name: Install build dependencies
-        run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev libzstd-dev libssl-dev libdrm-dev
+        run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev libzstd-dev libssl-dev libdrm-dev libpcre2-dev
       - name: Configure
         run: cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
       - name: Build
-        run: cmake --build build --target onebit npu_full_elf_test -j
+        run: cmake --build build --target onebit npu_full_elf_test npu_pack_test tokenizer_test -j
       - name: Test
         run: ctest --test-dir build --output-on-failure

CMakeLists.txt

@@ -53,9 +53,13 @@ if(ONEBIT_HRX)
 endif()
 
 # ── Step 3: NPU engine, full ELFs only (docs/npu.md) ──────────────────────
-# 3a: full-ELF generation. Pure C++, no XRT, so it builds and tests anywhere.
-add_library(onebit_npu STATIC npu/full_elf.cpp)
+# Host side (full-ELF generation, model reader, buffer packing): pure C++, no
+# XRT, so it builds and tests anywhere.
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
+add_library(onebit_npu STATIC npu/full_elf.cpp npu/model.cpp npu/pack.cpp npu/tokenizer.cpp)
 target_include_directories(onebit_npu PUBLIC npu)
+target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json PRIVATE PkgConfig::PCRE2)
 
 add_executable(npu_full_elf_test tests/npu_full_elf_test.cpp)
 target_link_libraries(npu_full_elf_test PRIVATE onebit_npu)
@@ -69,4 +73,41 @@ if(ONEBIT_NPU_CAPTURED AND ONEBIT_NPU_PDI AND ONEBIT_NPU_GOLDEN)
              COMMAND npu_full_elf_test ${ONEBIT_NPU_CAPTURED} ${ONEBIT_NPU_PDI} ${ONEBIT_NPU_GOLDEN})
 endif()
 
+add_executable(npu_pack_test tests/npu_pack_test.cpp)
+target_link_libraries(npu_pack_test PRIVATE onebit_npu)
+set(ONEBIT_NPU_MODEL "" CACHE PATH "Q4NX model directory (model.q4nx + config.json)")
+set(ONEBIT_NPU_BO_DUMP "" CACHE PATH "The reference lane's buffer dump (RT_DUMP_BOS)")
+add_executable(tokenizer_test tests/tokenizer_test.cpp)
+target_link_libraries(tokenizer_test PRIVATE onebit_npu)
+if(ONEBIT_NPU_MODEL)
+    add_test(NAME tokenizer_qwen3
+             COMMAND tokenizer_test ${ONEBIT_NPU_MODEL}/tokenizer.json ${CMAKE_SOURCE_DIR}/tests/data/qwen3_tokenizer_golden.json)
+endif()
+if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_BO_DUMP)
+    add_test(NAME npu_pack_model COMMAND npu_pack_test ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_BO_DUMP})
+endif()
+
+# The lane itself: needs XRT (the NPU runtime) and an XDNA NPU to run.
+option(ONEBIT_NPU "Build the NPU fast lane (needs XRT)" OFF)
+set(ONEBIT_XRT_ROOT "/opt/xilinx/xrt" CACHE PATH "XRT install prefix")
+if(ONEBIT_NPU)
+    if(NOT EXISTS "${ONEBIT_XRT_ROOT}/include/xrt/xrt_device.h")
+        message(FATAL_ERROR "ONEBIT_NPU needs XRT; none at ONEBIT_XRT_ROOT=${ONEBIT_XRT_ROOT}")
+    endif()
+    add_library(onebit_npu_lane STATIC npu/lane.cpp npu/generate.cpp)
+    target_include_directories(onebit_npu_lane SYSTEM PUBLIC ${ONEBIT_XRT_ROOT}/include)
+    target_link_libraries(onebit_npu_lane PUBLIC onebit_npu ${ONEBIT_XRT_ROOT}/lib/libxrt_coreutil.so)
+    target_sources(onebit PRIVATE app/unified.cpp)
+    target_link_libraries(onebit PRIVATE onebit_npu_lane)
+    target_compile_definitions(onebit PRIVATE ONEBIT_NPU)
+    set_target_properties(onebit PROPERTIES BUILD_RPATH ${ONEBIT_XRT_ROOT}/lib INSTALL_RPATH ${ONEBIT_XRT_ROOT}/lib)
+    set(ONEBIT_NPU_KERNELS "" CACHE PATH "Kernel directory (layer_ctx{1,2,17}.elf, lmhead.elf, layer.pdi)")
+    set(ONEBIT_NPU_REFERENCE "" CACHE PATH "Reference lane's logits for the anchor prompt")
+    if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_KERNELS AND ONEBIT_NPU_REFERENCE)
+        add_test(NAME npu_lane_e2e
+                 COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_lane_e2e.sh $<TARGET_FILE:onebit>
+                         ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_KERNELS} ${ONEBIT_NPU_REFERENCE})
+    endif()
+endif()
+
 add_test(NAME smoke_lemonade COMMAND ${CMAKE_SOURCE_DIR}/tests/smoke_lemonade.sh $<TARGET_FILE:onebit>)

app/main.cpp

@@ -16,6 +16,10 @@
 //
 //   1bit lemonade [lemond options]   Lemonade's server core, embedded in-process,
 //                                    with the engine's `onebit` backend.
+//   1bit unified -m <model dir>      one native model on the NPU fast lane behind
+//                                    OpenAI endpoints; what the onebit backend spawns.
+//   1bit npu-run [options]           token ids in, token ids out, on the NPU fast
+//                                    lane (docs/npu.md); for checks and benchmarks.
 //
 // Ported from 1bit-MONSTER tools/onebin.cpp and tools/unified_server.cpp
 // (run_embedded_lemonade). See docs/PORTING.md for what lands next.
@@ -29,8 +33,20 @@
 
 #include <nlohmann/json.hpp>
 
+#ifdef ONEBIT_NPU
+#include "generate.h"
+#include "model.h"
+#include "unified.h"
+#endif
+
+#include <chrono>
 #include <cstdio>
+#include <cstdlib>
+#include <filesystem>
 #include <cstring>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
 #include <memory>
 #include <string>
 #include <vector>
@@ -45,6 +61,10 @@ void usage(FILE* out) {
                  "\n"
                  "commands:\n"
                  "  lemonade [lemond options]   run Lemonade's server with the engine behind it\n"
+#ifdef ONEBIT_NPU
+                 "  unified -m <model dir>      serve one NPU model (OpenAI endpoints)\n"
+                 "  npu-run [options]           generate on the NPU fast lane (npu-run --help)\n"
+#endif
                  "  version                     print the version\n"
                  "  help                        show this help\n");
 }
@@ -69,6 +89,38 @@ void use_engine_hrx_build(nlohmann::json& config) {
 }
 #endif
 
+// Native models for the `onebit` backend: every NPU model directory
+// (docs/npu.md) directly under the roots in ONEBIT_MODEL_ROOTS (colon-separated,
+// default ~/models). Lemonade serves each through `1bit unified -m <dir>`. The
+// architecture registry (docs/PORTING.md step 5) replaces this scan.
+std::vector<lemon::ModelInfo> native_models() {
+    std::vector<lemon::ModelInfo> out;
+#ifdef ONEBIT_NPU
+    std::string roots;
+    if (const char* r = std::getenv("ONEBIT_MODEL_ROOTS"); r && *r) roots = r;
+    else if (const char* h = std::getenv("HOME"); h && *h) roots = std::string(h) + "/models";
+    std::istringstream in(roots);
+    for (std::string root; std::getline(in, root, ':');) {
+        std::error_code ec;
+        for (const auto& d : std::filesystem::directory_iterator(root, ec)) {
+            if (!d.is_directory() || !onebit::is_npu_model_dir(d.path().string())) continue;
+            lemon::ModelInfo mi;
+            mi.model_name = d.path().filename().string();
+            mi.recipe = "onebit";
+            mi.checkpoints["main"] = d.path().string();
+            mi.resolved_paths["main"] = d.path().string();
+            mi.downloaded = true;
+            mi.source = "engine";
+            mi.labels = {"chat", "npu"};
+            out.push_back(std::move(mi));
+        }
+    }
+    std::fprintf(stderr, "1bit: %zu NPU model(s) registered with the onebit backend (roots: %s)\n", out.size(),
+                 roots.c_str());
+#endif
+    return out;
+}
+
 // Hands argv to Lemonade's own CLI and runs its server in this process. The
 // engine's native models reach Lemonade through the `onebit` backend
 // (third_party/lemonade/src/cpp/include/lemon/backends/onebit/onebit.h); they
@@ -87,9 +139,7 @@ int run_lemonade(int argc, char** argv) {
     use_engine_hrx_build(config_json);
 #endif
 
-    // Native NPU artifacts are registered by the NPU engine port (docs/PORTING.md
-    // step 3); until then the onebit backend lists none.
-    lemon::backends::onebit::set_onebit_models({});
+    lemon::backends::onebit::set_onebit_models(native_models());
 
     if (cli_config.port != -1) config_json["port"] = cli_config.port;
     if (!cli_config.host.empty()) config_json["host"] = cli_config.host;
@@ -102,6 +152,60 @@ int run_lemonade(int argc, char** argv) {
     return 0;
 }
 
+#ifdef ONEBIT_NPU
+// Token ids in, token ids out. Prints one id per line on stdout and the timing
+// on stderr; --dump-logits writes each step's logits as float32
+// (decode_logits_idx<N>.bin, N = decode step), for comparison with a reference.
+int run_npu(int argc, char** argv) {
+    std::string model_dir, kernel_dir, ids, dump;
+    onebit::npu::GenerateOptions opt;
+    opt.max_tokens = 24;
+    for (int i = 0; i < argc; ++i) {
+        const std::string a = argv[i];
+        auto next = [&]() -> std::string {
+            if (i + 1 >= argc) throw std::runtime_error(a + " needs a value");
+            return argv[++i];
+        };
+        if (a == "--model") model_dir = next();
+        else if (a == "--kernels") kernel_dir = next();
+        else if (a == "--ids") ids = next();
+        else if (a == "-n") opt.max_tokens = std::stoi(next());
+        else if (a == "--repetition-penalty") opt.repetition_penalty = std::stof(next());
+        else if (a == "--no-eos") opt.stop_at_eos = false;
+        else if (a == "--dump-logits") dump = next();
+        else if (a == "--help" || a == "-h") {
+            std::printf("usage: 1bit npu-run --model <dir> --kernels <dir> --ids \"<id> <id> ...\"\n"
+                        "                    [-n 24] [--repetition-penalty 1.1] [--no-eos] [--dump-logits <dir>]\n"
+                        "  --model    directory with model.q4nx and config.json\n"
+                        "  --kernels  directory with layer_ctx{1,2,17}.elf, lmhead.elf and layer.pdi\n");
+            return 0;
+        } else throw std::runtime_error("unknown option " + a);
+    }
+    if (model_dir.empty() || kernel_dir.empty() || ids.empty()) throw std::runtime_error("--model, --kernels and --ids are required");
+    std::vector<int> prompt;
+    std::istringstream in(ids);
+    for (int t; in >> t;) prompt.push_back(t);
+
+    const onebit::npu::Model model(model_dir);
+    auto t0 = std::chrono::steady_clock::now();
+    onebit::npu::Lane lane(model, kernel_dir);
+    const double load_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+    std::vector<float> lg;
+    if (!dump.empty())
+        opt.on_logits = [&](int step, onebit::npu::Lane& l) {
+            l.logits(lg);
+            std::ofstream(dump + "/decode_logits_idx" + std::to_string(step) + ".bin", std::ios::binary)
+                .write(reinterpret_cast<const char*>(lg.data()), std::streamsize(lg.size() * sizeof(float)));
+        };
+    const auto r = onebit::npu::generate(lane, model, prompt, opt);
+    for (int t : r.tokens) std::printf("%d\n", t);
+    std::fprintf(stderr, "load %.0f ms | prefill %zu tokens %.1f ms | decode %zu tokens %.1f ms/token (%.1f tok/s)%s\n",
+                 load_ms, prompt.size(), r.prefill_ms, r.tokens.size(), r.decode_ms / double(r.tokens.size()),
+                 1000.0 * double(r.tokens.size()) / r.decode_ms, r.stopped_at_eos ? " | stopped at EOS" : "");
+    return 0;
+}
+#endif
+
 }  // namespace
 
 int main(int argc, char** argv) {
@@ -116,6 +220,24 @@ int main(int argc, char** argv) {
         for (int i = 2; i < argc; ++i) args.push_back(argv[i]);
         return run_lemonade(static_cast<int>(args.size()), args.data());
     }
+#ifdef ONEBIT_NPU
+    if (cmd == "unified") {
+        try {
+            return onebit::run_unified(argc - 2, argv + 2);
+        } catch (const std::exception& e) {
+            std::fprintf(stderr, "1bit unified: %s\n", e.what());
+            return 1;
+        }
+    }
+    if (cmd == "npu-run") {
+        try {
+            return run_npu(argc - 2, argv + 2);
+        } catch (const std::exception& e) {
+            std::fprintf(stderr, "1bit npu-run: %s\n", e.what());
+            return 1;
+        }
+    }
+#endif
     if (cmd == "version" || cmd == "--version") {
         std::printf("1bit %s\n", kVersion);
         return 0;