diff --git a/.github/workflows/bump-hrx.yml b/.github/workflows/bump-hrx.yml index 2bb87c27d..c0488e998 100644 --- a/.github/workflows/bump-hrx.yml +++ b/.github/workflows/bump-hrx.yml @@ -152,6 +152,7 @@ jobs: git update-index --cacheinfo "160000,$REBASED,third_party/llama.cpp" git update-index --cacheinfo "160000,$AMD_HRX,third_party/hrx-system" git commit -q -m "Bump HRX: llama.cpp ${AMD_LLAMA:0:12}, hrx-system ${AMD_HRX:0:12} (AMD's tested pair)" + scripts/registry-regen.sh # the registry must record the new pin (ctest registry_pins) git push -q origin "$branch" gh pr create --base main --head "$branch" \ --title "Bump HRX: llama.cpp ${AMD_LLAMA:0:12}, hrx-system ${AMD_HRX:0:12}" \ diff --git a/.github/workflows/bump-llama-vulkan.yml b/.github/workflows/bump-llama-vulkan.yml index ee0852783..b1b225bcf 100644 --- a/.github/workflows/bump-llama-vulkan.yml +++ b/.github/workflows/bump-llama-vulkan.yml @@ -137,6 +137,7 @@ jobs: git switch -c "$branch" git update-index --cacheinfo "160000,$REBASED,third_party/llama.cpp-vulkan" git commit -q -m "Bump Vulkan llama.cpp: ggml-org/llama.cpp ${TAG} (${UPSTREAM:0:12})" + scripts/registry-regen.sh # the registry must record the new pin (ctest registry_pins) git push -q origin "$branch" gh pr create --base main --head "$branch" \ --title "Bump Vulkan llama.cpp: ggml-org/llama.cpp ${TAG}" \ diff --git a/.github/workflows/bump-zinc.yml b/.github/workflows/bump-zinc.yml index 79b02c3f0..12149af98 100644 --- a/.github/workflows/bump-zinc.yml +++ b/.github/workflows/bump-zinc.yml @@ -85,6 +85,7 @@ jobs: git switch -c "$branch" git update-index --cacheinfo "160000,$UPSTREAM,third_party/zinc" git commit -q -m "Bump ZINC: zolotukhin/zinc ${UPSTREAM:0:12}" + scripts/registry-regen.sh # the registry must record the new pin (ctest registry_pins) git push -q origin "$branch" gh pr create --base main --head "$branch" \ --title "Bump ZINC: zolotukhin/zinc ${UPSTREAM:0:12}" \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a8381b48..4f9d979cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -312,10 +312,13 @@ if(ONEBIT_LAYA_ROUTE_MODEL) add_test(NAME laya_route_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_route_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) endif() -# The registry against the reviewed not-an-alias list (docs/arch-gaps.md): no pins or network. +# The registry against the reviewed not-an-alias list (docs/arch-gaps.md) and the committed pins: +# no checkouts or network. find_package(Python3 COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) add_test(NAME registry_gaps COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/registry_build.py --check-gaps) + # a pin bump that did not regenerate the registry (only the gitlinks: no checkouts) + add_test(NAME registry_pins COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/registry_build.py --check-pins) endif() add_executable(npu_full_elf_test tests/npu_full_elf_test.cpp) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f011267af..817dfe2cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,9 @@ an RFC or issue. - **Pins only move forward.** CI fails a pull request whose submodule pin is behind, or has diverged from, the pin on `main` (`tools/check_pins.py`), because that drops commits the engine already shipped. A rollback on purpose carries the `pin rollback` label. +- **The registry follows the pins.** `registry/architectures.json` records the pins it was built + from, and ctest `registry_pins` fails when they differ from the committed ones. The bump + workflows regenerate it with `scripts/registry-regen.sh`; a hand-made pin change runs the same. This repository is a port of the working code in 1bit-MONSTER, our private development repository, without its history. diff --git a/docs/registry.md b/docs/registry.md index 245c32a2e..8da8ad6d2 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -38,7 +38,7 @@ hand: | `zinc` | ZINC's `parseArchitecture` accepts the GGUF architecture | | `npu` | the fast lane's model type: `qwen3` (the lane kernels are built for Qwen3-0.6B's shapes) | -At the current pins (llama.cpp `8dd75eb`, llama.cpp-vulkan `546800c`, ZINC `29bc350`): 323 HF architectures mapped, vulkan 323, hrx 300, zinc 64, npu 1. Mapped means a backend's code accepts the architecture; the census below reports how many were checked. +On 2026-09-26: 323 HF architectures mapped, vulkan 323, hrx 300, zinc 64, npu 1. Mapped means a backend's code accepts the architecture; the census below reports how many were checked. `registry/architectures.json` always holds the current counts and the pins they come from: every pin bump regenerates it (`scripts/registry-regen.sh`, run by the bump workflows), and CI's `registry_pins` test fails a pin that moved without it. **Reviewed gaps.** Some unmapped architectures only look like a supported family. [Architecture gaps](arch-gaps.md) records why each of them is not an alias. `registry/significant.json` lists those classes, and `tools/registry_build.py --check-gaps` (ctest `registry_gaps`) fails if one becomes mapped without a recorded reason. diff --git a/scripts/registry-regen.sh b/scripts/registry-regen.sh new file mode 100755 index 000000000..b2c2381ac --- /dev/null +++ b/scripts/registry-regen.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Regenerate registry/architectures.json for the pins in HEAD and fold it into HEAD. +# The bump workflows run this right after committing a new pin, because CI's registry_pins +# test fails a pin that moved without the registry. It fetches only the three trees the +# registry reads (llama.cpp, llama.cpp-vulkan, zinc), each at its pinned commit, depth 1. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +for s in third_party/llama.cpp third_party/llama.cpp-vulkan third_party/zinc; do + git submodule update --init --depth 1 -- "$s" +done +python3 tools/registry_build.py +if ! git diff --quiet -- registry/architectures.json; then + git add registry/architectures.json + git commit -q --amend --no-edit +fi +python3 tools/registry_build.py --check-pins diff --git a/tools/registry_build.py b/tools/registry_build.py index aa6066da5..e1ffcf1a9 100755 --- a/tools/registry_build.py +++ b/tools/registry_build.py @@ -15,7 +15,7 @@ # limitations under the License. """Build registry/architectures.json: which backends of this engine map each HF architecture. -usage: tools/registry_build.py [--check] [--out registry/architectures.json] +usage: tools/registry_build.py [--check | --check-gaps | --check-pins] [--out registry/architectures.json] Every entry is read from a pinned source, never typed in by hand: - HF architecture -> GGUF architecture: the `@ModelBase.register(...)` classes in llama.cpp's @@ -33,6 +33,8 @@ architecture was run and checked here is recorded separately, in registry/checked.json. --check exits 1 when the file on disk differs from what the pinned sources give. +--check-pins only compares the pins the file records with the committed gitlinks, so it needs no +checkouts (CI and ctest `registry_pins` run it): a pin bump must regenerate the registry. """ import argparse import ast @@ -204,7 +206,22 @@ def main(): ap.add_argument("--check", action="store_true") ap.add_argument("--check-gaps", action="store_true", help="only check the committed registry against registry/significant.json (no pins needed)") + ap.add_argument("--check-pins", action="store_true", + help="only check that the registry records the committed pins (no checkouts needed)") a = ap.parse_args() + if a.check_pins: + recorded = json.load(open(a.out))["sources"] + stale = [(name, recorded.get(name, ""), pin(path)) for name, path in + (("llama.cpp (vulkan)", UPSTREAM), ("llama.cpp (hrx)", HRX), ("zinc", ZINC)) + if recorded.get(name) != pin(path)] + for name, old, new in stale: + print(f"{a.out}: {name} recorded at {old[:9] or 'nothing'}, pinned at {new[:9]}") + if stale: + print("a pin moved without regenerating the registry: check out the submodules at their " + "pins (git submodule update --init) and run tools/registry_build.py") + return 1 + print(f"{a.out} records the committed pins") + return 0 if a.check_gaps: bad = report_gaps(json.load(open(a.out))) if not bad: