diff --git a/.gitattributes b/.gitattributes index 1ff0c423042..ef8281262d7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ # Set default behavior to automatically normalize line endings. ############################################################################### * text=auto +*.sh text eol=lf ############################################################################### # Set default behavior for command prompt diff. diff --git a/.github/workflows/buildtest.yml b/.github/workflows/buildtest.yml index 6ee6ad43231..0177aef560d 100644 --- a/.github/workflows/buildtest.yml +++ b/.github/workflows/buildtest.yml @@ -1,93 +1,79 @@ ---- -name: Run Tests +name: Compare saved builds on: pull_request: - branches: - - dev + branches: [dev, tests-branch] workflow_dispatch: + inputs: + base_ref: + description: Base commit or ref (empty uses this workflow's commit) + type: string + head_ref: + description: Candidate commit or ref (empty uses this workflow's commit) + type: string + strict: + description: Fail on stat differences as well as calculation errors + type: boolean + default: false +permissions: + contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: build-corpus-${{ github.ref }} cancel-in-progress: true jobs: - run_build_diff: + fixture_smoke: + name: Fixed fixture comparison runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - name: Checkout HEAD - uses: actions/checkout@v4 - - name: Fetch Dev branch - id: get-dev-ref - run: | - git fetch --depth=1 origin dev - echo "devref=$(git rev-parse origin/dev)" >> $GITHUB_OUTPUT - - name: Download Dev branch cache - id: download-dev-ref-cache - uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da - with: - name: cache-devref-${{ steps.get-dev-ref.outputs.devref }} - path: /tmp/cache/ - if_no_artifact_found: warn - search_artifacts: true - # Dev ref cache contains the build list and build xmls. Use that one to keep tests reproducible - - name: Update static builds list from cache - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'true' }} - run: cat /tmp/cache/builds.txt > spec/builds.txt - - name: Download latest build list - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }} - id: download-build-list - uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - name: builds.txt - path: /tmp/latestbuildlist/ - workflow: updatebuildlist.yml - if_no_artifact_found: warn - search_artifacts: true - - name: Update static builds list - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }} - run: cat /tmp/latestbuildlist/builds.txt > spec/builds.txt - - name: Download latest build xmls - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }} - uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - name: build-xmls - path: /tmp/cache/ - if_no_artifact_found: warn - search_artifacts: true - - name: Calculate build xmls and differences between them + python-version: '3.12' + - name: Compare every fixed fixture on base and candidate + env: + BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.sha }} + HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }} + STRICT_DIFF: ${{ inputs.strict }} run: | - mkdir /tmp/cache || true # Make sure /tmp/cache exists. Ignore exit code - chmod -R 777 /tmp/cache && docker compose run -v '/tmp/cache/:/cache' -e 'CACHEDIR=/cache' busted-diff | tee /tmp/dockerlog - - name: Generate artefact + args=() + if [ "$STRICT_DIFF" = true ]; then args+=(--strict); fi + python spec/RunBuildDiff.py --base "$BASE_REF" --head "$HEAD_REF" --fixtures-only --output "$RUNNER_TEMP/fixture-diff" "${args[@]}" + - name: Run importer and runner contract tests run: | - sed -n '/Runtime comparison for/,/Savefile Diff for/{/Savefile Diff for/!p;}' /tmp/dockerlog > /tmp/artefact - sed -n '/Savefile Diff for/, $p' /tmp/dockerlog >> /tmp/artefact - [ -s /tmp/artefact ] || rm /tmp/artefact - - name: Upload artefact - uses: actions/upload-artifact@v4 - with: - name: build-diff-output - path: /tmp/artefact - - name: Save used build list into cache - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }} - run: cp spec/builds.txt /tmp/cache/ - - name: Move xmls found in builds.txt to a new directory - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }} + python -m unittest discover -s tests -p 'test_update_build_corpus.py' -v + python -m unittest discover -s tests -p 'test_build_diff_contract.py' -v + - name: Test output comparison semantics run: | - mkdir new-build-xmls - while IFS= read -r line; do - FILENAME="/tmp/cache/${line//[^a-zA-Z0-9]/}.xml" - if [ -f "$FILENAME" ]; then - mv "$FILENAME" "./new-build-xmls/" - fi - done < "spec/builds.txt" - - name: Upload new build xmls - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }} - uses: actions/upload-artifact@v4 + docker build -t pob-corpus-contracts -f Dockerfile.test-builds . + docker run --rm --network none --mount "type=bind,source=$PWD,target=/workdir,readonly" -w /workdir pob-corpus-contracts luajit tests/test_diff_output.lua + corpus_comparison: + name: Complete rotating corpus comparison + if: vars.TEST_BUILD_CORPUS_ENABLED == 'true' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - name: build-xmls - path: './new-build-xmls/*' - - name: Upload dev ref cache - if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }} - uses: actions/upload-artifact@v4 + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - name: cache-devref-${{ steps.get-dev-ref.outputs.devref }} - path: /tmp/cache/ + python-version: '3.12' + - name: Pin saved corpus once + run: | + git fetch origin refs/heads/build-test-corpus + CORPUS_SHA=$(git rev-parse FETCH_HEAD) + echo "Pinned corpus commit: $CORPUS_SHA" + git worktree add --detach "$RUNNER_TEMP/corpus" "$CORPUS_SHA" + - name: Calculate all saved inputs without provider requests + env: + BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.sha }} + HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }} + STRICT_DIFF: ${{ inputs.strict }} + run: | + args=() + if [ "$STRICT_DIFF" = true ]; then args+=(--strict); fi + python spec/RunBuildDiff.py --base "$BASE_REF" --head "$HEAD_REF" --corpus "$RUNNER_TEMP/corpus" --output "$RUNNER_TEMP/corpus-diff" "${args[@]}" diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 43d280673ce..580bb969bcb 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -4,14 +4,20 @@ on: push: branches: - dev + - tests-branch pull_request: branches: - dev + - tests-branch +permissions: + contents: read jobs: run_unit_tests: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Run busted tests run: docker compose run --no-TTY busted-tests diff --git a/.github/workflows/updatebuildlist.yml b/.github/workflows/updatebuildlist.yml index ea841152b01..440428c5b95 100644 --- a/.github/workflows/updatebuildlist.yml +++ b/.github/workflows/updatebuildlist.yml @@ -1,37 +1,47 @@ ---- -name: Update build list for tests +name: Refresh monthly build corpus on: schedule: - cron: '20 4 * * *' workflow_dispatch: +permissions: + contents: write +concurrency: + group: monthly-build-corpus-writer + cancel-in-progress: false jobs: - update-builds-list: - runs-on: ubuntu-22.04 + refresh: + if: github.event_name == 'workflow_dispatch' || (vars.TEST_BUILD_CORPUS_ENABLED == 'true' && github.ref_name == github.event.repository.default_branch) + runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Checkout HEAD - uses: actions/checkout@v4 - - name: Install moreutils - run: sudo apt-get install -y moreutils - - name: Download latest build list - uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - name: builds.txt - if_no_artifact_found: warn - path: /tmp/latestbuildlist/ - - name: Update list - run: > - cat spec/builds.txt /tmp/latestbuildlist/builds.txt - <({ curl "https://pobarchives.com/api/builds?q=latest" & curl "https://pobarchives.com/api/builds?q=trending"; } - | jq -r '.builds[].build_info.build_link') - | tail -n 500 - | sort -u - | sponge builds.txt - - name: Print new builds list - run: cat builds.txt - - name: Save new build list - uses: actions/upload-artifact@v4 - with: - name: builds.txt - path: builds.txt - overwrite: true - retention-days: 3 + python-version: '3.12' + - name: Load saved corpus or bootstrap an empty branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + REMOTE_REF=$(git ls-remote origin refs/heads/build-test-corpus) + if [ -n "$REMOTE_REF" ]; then + git fetch origin refs/heads/build-test-corpus + git worktree add --detach "$RUNNER_TEMP/corpus" FETCH_HEAD + else + git worktree add --detach "$RUNNER_TEMP/corpus" HEAD + git -C "$RUNNER_TEMP/corpus" switch --orphan build-test-corpus + fi + - name: Validate and construct next corpus + run: python spec/UpdateBuildCorpus.py --url https://api.pob.codes/test-builds --prior "$RUNNER_TEMP/corpus" --output "$RUNNER_TEMP/next-corpus" + - name: Commit manifest and retained bytes together + run: | + git -C "$RUNNER_TEMP/corpus" rm -r --ignore-unmatch codes manifest.json + cp -a "$RUNNER_TEMP/next-corpus/." "$RUNNER_TEMP/corpus/" + git -C "$RUNNER_TEMP/corpus" add manifest.json codes + if git -C "$RUNNER_TEMP/corpus" diff --cached --quiet; then + echo "Monthly batch already applied; no corpus change." + exit 0 + fi + git -C "$RUNNER_TEMP/corpus" commit -m "Refresh monthly test-build corpus" + # Plain fast-forward push rejects a competing writer. The next daily run + # refetches/reapplies; never force-push or publish a partial manifest. + git -C "$RUNNER_TEMP/corpus" push origin HEAD:refs/heads/build-test-corpus diff --git a/.gitignore b/.gitignore index 855a9e1765f..43f6a495dab 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ Builds/ Settings.xml # Testing +__pycache__/ +*.pyc luajit/ spec/test_results.log spec/test_generation.log @@ -35,4 +37,4 @@ src/Export/ggpk/*.dll src/Data/TimelessJewelData/*.bin # Simplegraphic Debugging -runtime/imgui.ini \ No newline at end of file +runtime/imgui.ini diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 394aadee37d..59f5f52bd54 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,8 @@ # Contributing to Path of Building +For the monthly test-build feed, local calculation comparisons, and CI activation, +see [Build corpus CI](docs/build-corpus-ci.md). + # Table of contents 1. [Reporting bugs](#reporting-bugs) 2. [Requesting features](#requesting-features) diff --git a/Dockerfile.test-builds b/Dockerfile.test-builds new file mode 100644 index 00000000000..462c0ebc589 --- /dev/null +++ b/Dockerfile.test-builds @@ -0,0 +1,7 @@ +FROM ghcr.io/pathofbuildingcommunity/pathofbuilding-tests@sha256:171dc3da232b8c874882e4ae3b3aa4a6e130a6c9450a31904b312435a6bf5daf AS modern_luajit +FROM ghcr.io/paliak/busted-tests@sha256:0ce3f27d276dd6918d78ae11339e4135c445ea0e4fd31dbc88087ea25232ed90 +RUN luarocks install luautf8 0.1.6-1 +# Retain the existing test dependencies, but use upstream's GC64-capable LuaJIT. +COPY --from=modern_luajit /usr/local/bin/luajit-2.1.1784580905 /usr/local/bin/luajit-current +COPY --from=modern_luajit /usr/local/share/luajit-2.1 /usr/local/share/luajit-2.1 +RUN ln -sf /usr/local/bin/luajit-current /usr/local/bin/luajit diff --git a/Dockerfile.test-builds.dockerignore b/Dockerfile.test-builds.dockerignore new file mode 100644 index 00000000000..b16ab7e9648 --- /dev/null +++ b/Dockerfile.test-builds.dockerignore @@ -0,0 +1,2 @@ +** +!Dockerfile.test-builds diff --git a/docs/build-corpus-ci.md b/docs/build-corpus-ci.md new file mode 100644 index 00000000000..4ce589043d5 --- /dev/null +++ b/docs/build-corpus-ci.md @@ -0,0 +1,118 @@ +# Build corpus CI + +PoB Codes publishes up to 100 monthly builds at +`https://api.pob.codes/test-builds`. This repository retains up to 500 unique +encoded builds in a FIFO on `build-test-corpus`. Comparisons load one saved +corpus and calculate every input on both Git revisions, alongside every XML +fixture under `spec/TestBuilds`, including versioned subdirectories. + +The API supplies inputs. This repository owns retention and calculations. +There is no migration of old build lists or saved calculated-output baseline. +Existing Busted commands, fixture files, and legacy generation tools remain. + +## Run locally + +Requirements: Python 3.12+, Git, and Linux Docker containers. Both compared +revisions must support the builds' game/tree version. Current 3.29 inputs work +with PoB release v2.67.2 (`b32759ab0f31a1c8499a0d420cb0f0633d4fe478`). A 3.25 +runtime cannot calculate those inputs: this runner does not downgrade builds, +skip incompatible inputs, or silently substitute a different runtime. + +Run from the repository root. Use new output directories for each run: + +```sh +# A missing prior directory bootstraps an empty corpus. +python spec/UpdateBuildCorpus.py --url https://api.pob.codes/test-builds --prior ../corpus-prior --output ../corpus-next + +# Use actual, locally available Git refs or commit SHAs. +python spec/RunBuildDiff.py --base --head --corpus ../corpus-next --output ../comparison --strict + +# Fixed fixtures only; no corpus or API access required. +python spec/RunBuildDiff.py --base --head --fixtures-only --output ../fixture-comparison --strict + +# Refresh into a new directory; the prior corpus remains intact. +python spec/UpdateBuildCorpus.py --url https://api.pob.codes/test-builds --prior ../corpus-next --output ../corpus-later +``` + +Commit calculation edits before comparing: the runner archives committed +`src` and `runtime` from each ref, so uncommitted calculation edits are excluded. +It uses the harness in the current checkout identically for both revisions. +`--repo` can select another local Git checkout explicitly; the normal case +compares this checkout's own base/head. No source revisions are fetched implicitly. + +Outputs are saved in `base/` and `head/` beneath the result directory. Logs name +each input, both resolved SHAs, the corpus digest, and the Docker image ID. +Exit 0 means the requested comparison completed. With `--strict`, exit 1 means +all calculations completed but stats differ. Exit 2 means invalid arguments, +inputs, calculations, or comparison execution. Without `--strict`, differences +are reported in the log and exit 0; execution errors still fail. + +The runner builds `Dockerfile.test-builds` automatically. Its two source images +are digest-pinned; the existing dependencies use a GC64-capable LuaJIT to avoid +the older allocator's memory limit. `--image ` reuses an image; +both sides use the same inspected image ID. `--parallel 1` runs one container +at a time instead of two. `--extra-fixtures ` adds local XML inputs. + +Calculation containers have no network, read-only runtime/input mounts, two +CPUs, and 2 GiB memory. Batches contain at most 25 inputs, with a 30-second +per-input alarm and a 300-second container deadline. Missing player stats, +missing active-minion stats, import popups, incomplete output sets, and runtime +errors fail the run. A failed batch cancels queued batches; running containers +remain bounded by their deadlines. Reordered XML attributes do not create +false comparison failures. + +## Test the harness + +```sh +python -m unittest discover -s tests -p 'test_update_build_corpus.py' -v +python -m unittest discover -s tests -p 'test_build_diff_contract.py' -v +docker build -t pob-corpus-tests -f Dockerfile.test-builds . +docker run --rm --network none --mount "type=bind,source=$PWD,target=/workdir,readonly" -w /workdir pob-corpus-tests luajit tests/test_diff_output.lua +``` + +The last command uses POSIX shell syntax; in PowerShell, use an absolute path +for the mount's `source`. The existing `docker compose run --rm --no-TTY +busted-tests` command still runs the application's Busted suite. + +## Corpus contract and publication + +Schema 1 has `batchId`, UTC `period` (`YYYY-MM`), canonical millisecond UTC +`generatedAt`, `patchVersion`, `requestedCount: 100`, `count: 1..100`, and +`builds: [{code, sha256}]`. SHA-256 covers the exact UTF-8 code string. Limits: +150 KiB encoded per input, 4 MiB inflated XML, 16 MiB per API response. + +Validate the complete batch before writing the new corpus. Append new hashes +in batch order and evict the oldest beyond 500. Replays do nothing; a changed +payload under an accepted batch ID fails. Older/same-period replacement batches +are ignored. Duplicates in later months do not reorder retained builds. +`manifest.json` and `codes/.txt` are published together in one Git commit. + +Refreshes use ETags and bounded retries for transport errors, HTTP 429 and 5xx. +They honor `Retry-After`. Invalid responses never replace the prior corpus. +A normal fast-forward push rejects a competing writer; the next run refetches +and reapplies. The API needs no authentication. Calculation jobs never call it. + +## GitHub activation + +1. Put the CI changes on a branch with a compatible PoB runtime. For PR + comparisons, the target/base must also be compatible with the corpus. +2. Manually run **Refresh monthly build corpus** on that branch to bootstrap + `build-test-corpus`. Manual dispatch is allowed before setting a repository + variable; only a maintainer-triggered refresh has `contents: write`. +3. Manually run **Compare saved builds**. Optional `base_ref`/`head_ref` select + explicit revisions; empty inputs compare the dispatched commit to itself. + Enable `strict` for a zero-difference control. Manual runs require the saved + corpus branch and fail if it is absent. +4. Set `TEST_BUILD_CORPUS_ENABLED=true` to enable rotating comparisons on PRs. + Fixed fixtures and harness tests run without this variable. PR comparisons + always use the actual event's base/head SHAs and have read-only credentials. +5. For daily conditional polling, put the refresh workflow and helpers on the + default branch and enable the same variable. A manual run on a development + branch does not activate GitHub's schedule. + +The job checks the API daily but only retains a new batch once per month. +Provider outages do not affect comparisons using the saved corpus. PR stat +differences are advisory; import/calculation failures are errors in both modes. +Confirm hosted-runner permissions and performance before making the rotating +job required. FIFO retention at 500 is covered by tests; 500 distinct live +builds require accumulation over multiple monthly batches to benchmark. diff --git a/spec/CompareBuilds.sh b/spec/CompareBuilds.sh new file mode 100644 index 00000000000..d3d1d90988a --- /dev/null +++ b/spec/CompareBuilds.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +failed=0 +count=0 +changed=0 +for base in /outputs/base/*.build; do + if [ ! -f "$base" ]; then + echo "No calculated build pairs to compare" + exit 2 + fi + name=$(basename "$base") + status=0 + luajit /harness/DiffOutput.lua "/outputs/head/$name" "$base" || status=$? + if [ "$status" -ne 0 ]; then + printf 'Input %s comparison exit status: %s\n' "$name" "$status" + fi + if [ "$status" -gt 1 ]; then + failed=2 + elif [ "$status" -eq 1 ]; then + changed=$((changed + 1)) + if [ "${STRICT_DIFF:-0}" = 1 ] && [ "$failed" -eq 0 ]; then + failed=1 + fi + fi + count=$((count + 1)) +done +printf 'Compared %s input pairs; %s with differences\n' "$count" "$changed" +exit "$failed" diff --git a/spec/DiffOutput.lua b/spec/DiffOutput.lua index 5f3228f21c0..37538cff712 100644 --- a/spec/DiffOutput.lua +++ b/spec/DiffOutput.lua @@ -8,11 +8,15 @@ local function buildOutputMap(filecontent) local playerOutput = {} local minionOutput = {} for line in splitLines(filecontent) do - local key, val = line:match('PlayerStat stat="(.-)" value="(.-)"') + local playerTag = line:match('') + local key = playerTag and playerTag:match('stat="(.-)"') + local val = playerTag and playerTag:match('value="(.-)"') if key then playerOutput[key] = val else - local key,val = line:match('MinionStat stat="(.-)" value="(.-)"') + local minionTag = line:match('') + local key = minionTag and minionTag:match('stat="(.-)"') + local val = minionTag and minionTag:match('value="(.-)"') if key then minionOutput[key] = val end @@ -27,6 +31,10 @@ local devhnd = io.open(arg[2], "r") if headhnd and devhnd then local playerHEADOutput, minionHEADOutput = buildOutputMap(headhnd:read("*a")) local playerDEVOutput, minionDEVOutput = buildOutputMap(devhnd:read("*a")) + if next(playerHEADOutput) == nil or next(playerDEVOutput) == nil then + print("Missing calculated player output") + os.exit(2) + end local mismatch = {} local mismatchFound = false for key, val in pairs(playerHEADOutput) do @@ -69,4 +77,4 @@ if headhnd and devhnd then end else os.exit(2) -end \ No newline at end of file +end diff --git a/spec/HeadlessSupport.lua b/spec/HeadlessSupport.lua new file mode 100644 index 00000000000..eb55cd20ba8 --- /dev/null +++ b/spec/HeadlessSupport.lua @@ -0,0 +1,10 @@ +-- Runtime adapters shared by both revisions, installed just before Launch.lua. +-- Each revision retains its own HeadlessWrapper and production calculation code. +local zlib = require("zlib") +function GetScriptPath() return "/workdir/src" end +function GetRuntimePath() return "/workdir/runtime" end +function GetUserPath() return "/tmp" end +function GetWorkDir() return "/workdir/src" end +function GetTime() return os.clock() * 1000 end +function Inflate(data) return zlib.inflate()(data) end +function Deflate(data) return zlib.deflate()(data) end diff --git a/spec/RunBuildBatch.lua b/spec/RunBuildBatch.lua new file mode 100644 index 00000000000..1b74ac64250 --- /dev/null +++ b/spec/RunBuildBatch.lua @@ -0,0 +1,47 @@ +local inputListPath = assert(arg[1], "Input list is required") +package.path = "../runtime/lua/?.lua;../runtime/lua/?/init.lua;" .. package.path +local originalDofile = dofile +function dofile(path) + if path == "Launch.lua" then originalDofile("/harness/HeadlessSupport.lua") end + return originalDofile(path) +end +dofile("HeadlessWrapper.lua") +dofile = originalDofile +assert(loadBuildFromXML, "Headless initialization failed") +-- PoB's UI loader can report a failed import without throwing. Make those +-- failures fatal before stale/default calculations can be saved as success. +local originalLoadDB = build.LoadDB +function build:LoadDB(...) + assert(not originalLoadDB(self, ...), "Build XML import failed") +end +function launch:ShowErrMsg(message, ...) error(string.format(message, ...)) end +function launch.main:OpenMessagePopup(title, message) + error("Build import requires UI intervention: " .. tostring(title) .. ": " .. tostring(message)) +end +local list = assert(io.open(inputListPath, "r")) +local posix = require("posix") +posix.signal(posix.SIGALRM, function() error("Build calculation deadline exceeded") end) +local count = 0 +for filename in list:lines() do + assert(filename:match("^[%w%-]+%.xml$"), "Invalid staged input name") + local input = assert(io.open("/inputs/" .. filename, "rb")) + local xml = input:read("*a") + input:close() + print("Calculating input " .. filename) + posix.alarm(30) + loadBuildFromXML(xml, filename) + assert(build.buildName == filename and build.targetVersion, "Build initialization incomplete") + assert(build and build.calcsTab and build.calcsTab.mainOutput, "Missing calculated output") + local saved = assert(build:SaveDB("CI")) + assert(saved:find(" 16 * 1024 * 1024: + raise ValueError("missing/oversized output") + data = path.read_bytes() + if b" 4 * 1024 * 1024: + raise ValueError("invalid fixed fixture") + xml = source.read_bytes() + name = "fixture-" + hashlib.sha256(xml).hexdigest() + ".xml" + (inputs / name).write_bytes(xml) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--corpus", type=Path) + parser.add_argument("--fixtures-only", action="store_true") + parser.add_argument("--extra-fixtures", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--parallel", type=int, default=2, choices=(1, 2)) + parser.add_argument("--strict", action="store_true") + parser.add_argument("--image", help="Already-built image, primarily for offline reproduction") + args = parser.parse_args() + if bool(args.corpus) == args.fixtures_only: + parser.error("choose --corpus or explicit --fixtures-only") + repo = args.repo.resolve() + harness = Path(__file__).resolve().parent + base, head = revision(repo, args.base), revision(repo, args.head) + output = args.output.resolve() + if output.exists(): + parser.error("output must not already exist") + output.mkdir(parents=True) + started = time.monotonic() + image = args.image + if not image: + dockerfile = harness.parent / "Dockerfile.test-builds" + image = "pob-ci:" + hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16] + run(["docker", "build", "-t", image, "-f", str(dockerfile), str(harness.parent)], timeout=600) + image_id = subprocess.check_output(["docker", "image", "inspect", "--format", "{{.Id}}", image], text=True).strip() + print("Runtime base=%s head=%s image=%s" % (base, head, image_id), flush=True) + with tempfile.TemporaryDirectory(prefix="pob-corpus-") as scratch: + scratch = Path(scratch) + inputs = scratch / "inputs" + inputs.mkdir() + if args.corpus: + manifest, codes = read_corpus(args.corpus) + print("Corpus %s; rotating builds=%d" % (manifest["corpusDigest"], len(codes)), flush=True) + for key, code in codes.items(): + (inputs / (key + ".xml")).write_bytes(decode_code(code)) + # Copy the harness's fixed fixtures identically to both runtime revisions. + for folder in (harness / "TestBuilds", args.extra_fixtures): + if folder is None: + continue + stage_fixtures(folder, inputs) + names = sorted(p.name for p in inputs.iterdir()) + if not names: + raise ValueError("empty input set") + lists = scratch / "lists" + lists.mkdir() + jobs = [] + for label, sha in (("base", base), ("head", head)): + source = scratch / label + checkout_runtime(repo, sha, source) + destination = output / label + destination.mkdir() + for index in range(0, len(names), 25): + batch = lists / ("batch-%d.txt" % index) + batch.write_bytes(("\n".join(names[index:index+25]) + "\n").encode("utf-8")) + jobs.append((source, destination, batch.name)) + + def calculate(job): + source, destination, batch = job + print("Calculating %s %s" % (destination.name, batch), flush=True) + # The OS deadline bounds the whole batch; Lua's alarm bounds an input. + command = ["docker", "run", "--rm", "--network", "none", "--cpus", "2", "--memory", "2g", + "--security-opt", "no-new-privileges", "--cap-drop", "ALL"] + command += docker_mount(source, "/workdir") + docker_mount(harness, "/harness") + command += docker_mount(inputs, "/inputs") + docker_mount(lists, "/lists") + docker_mount(destination, "/outputs", False) + command += ["-w", "/workdir/src", "-e", "CI=true", image_id, "timeout", "300", "luajit", "/harness/RunBuildBatch.lua", "/lists/" + batch] + run(command, timeout=330) + + with ThreadPoolExecutor(max_workers=args.parallel) as pool: + futures = [pool.submit(calculate, job) for job in jobs] + try: + for future in as_completed(futures): + future.result() + except BaseException: + for future in futures: + future.cancel() + raise + validate_outputs(output / "base", names) + validate_outputs(output / "head", names) + command = ["docker", "run", "--rm", "--network", "none", "--security-opt", "no-new-privileges", "--cap-drop", "ALL"] + command += docker_mount(harness, "/harness") + docker_mount(output, "/outputs") + command += ["-e", "STRICT_DIFF=" + ("1" if args.strict else "0"), image_id, "sh", "/harness/CompareBuilds.sh"] + compared = subprocess.run(command, timeout=300) + if compared.returncode == 1 and args.strict: + print("Calculated all %d input pairs; differences detected (strict mode)" % len(names), flush=True) + return 1 + compared.check_returncode() + print("Completed calculation and comparison of %d input pairs in %.1fs" % (len(names), time.monotonic()-started), flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ValueError, OSError, KeyError, ElementTree.ParseError, subprocess.SubprocessError) as exc: + print("Build calculation/comparison error (%s): %s" % (type(exc).__name__, exc), flush=True) + raise SystemExit(2) diff --git a/spec/UpdateBuildCorpus.py b/spec/UpdateBuildCorpus.py new file mode 100644 index 00000000000..ec1d161eed2 --- /dev/null +++ b/spec/UpdateBuildCorpus.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Fetch/validate monthly PoB inputs and construct a deterministic, bounded FIFO. + +No calculation or source-provider downloads occur while applying a batch. +Publication is the caller's atomic Git commit, never a partial HTTP refresh. +""" +import argparse +import base64 +import hashlib +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +import zlib +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from xml.etree import ElementTree + +MAX_CODE = 150 * 1024 +MAX_XML = 4 * 1024 * 1024 +MAX_BATCH = 16 * 1024 * 1024 +MAX_CORPUS = 500 +HASH = re.compile(r"[a-f0-9]{64}") + + +def digest(value): + return hashlib.sha256(value).hexdigest() + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def decode_code(code): + if not isinstance(code, str) or not 0 < len(code) <= MAX_CODE or not re.fullmatch(r"[A-Za-z0-9_+/=-]+", code): + raise ValueError("invalid encoded build") + try: + packed = base64.b64decode(code + "=" * (-len(code) % 4), altchars=b"-_", validate=True) + except (ValueError, UnicodeError) as exc: + raise ValueError("invalid base64") from exc + xml = None + for window in (zlib.MAX_WBITS, -zlib.MAX_WBITS): + try: + stream = zlib.decompressobj(window) + candidate = stream.decompress(packed, MAX_XML + 1) + if len(candidate) > MAX_XML or stream.unconsumed_tail: + raise ValueError("inflated build exceeds limit") + if not stream.eof or stream.unused_data: + raise ValueError("incomplete or trailing compressed data") + xml = candidate + break + except zlib.error: + continue + if xml is None or re.search(br" 200: + raise ValueError("invalid batch identity") + if not isinstance(batch.get("period"), str) or not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", batch["period"]): + raise ValueError("invalid period") + if not isinstance(batch.get("patchVersion"), str) or not re.fullmatch(r"\d+\.\d+", batch["patchVersion"]): + raise ValueError("invalid patch") + stamp = batch.get("generatedAt", "") + if not isinstance(stamp, str) or not re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z", stamp): + raise ValueError("invalid generatedAt") + datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%S.%fZ") + if stamp[:7] != batch["period"]: + raise ValueError("period/timestamp mismatch") + builds = batch.get("builds") + if not isinstance(builds, list) or type(batch.get("count")) is not int or not 1 <= batch["count"] <= 100 or len(builds) != batch["count"]: + raise ValueError("invalid batch count") + hashes = set() + for entry in builds: + if not isinstance(entry, dict) or not isinstance(entry.get("sha256"), str) or not HASH.fullmatch(entry["sha256"]): + raise ValueError("invalid hash representation") + code = entry.get("code") + decode_code(code) + if digest(code.encode("utf-8")) != entry["sha256"] or entry["sha256"] in hashes: + raise ValueError("hash mismatch or duplicate build") + hashes.add(entry["sha256"]) + if len(canonical(batch)) > MAX_BATCH: + raise ValueError("batch exceeds limit") + return batch + + +def empty_manifest(): + return {"schemaVersion": 1, "entries": [], "batches": [], "etag": None, + "corpusDigest": digest(canonical([]))} + + +def read_corpus(directory, allow_empty=False): + directory = Path(directory) + manifest_path = directory / "manifest.json" + if not manifest_path.exists(): + if allow_empty and not (directory / "codes").exists(): + return empty_manifest(), {} + raise ValueError("missing corpus manifest") + if manifest_path.is_symlink() or manifest_path.stat().st_size > MAX_BATCH: + raise ValueError("invalid manifest file") + manifest = load_json_bytes(manifest_path.read_bytes()) + if not isinstance(manifest, dict): + raise ValueError("invalid corpus manifest") + entries, batches = manifest.get("entries"), manifest.get("batches") + if manifest.get("schemaVersion") != 1 or not isinstance(entries, list) or not 1 <= len(entries) <= MAX_CORPUS: + raise ValueError("invalid corpus entries") + if not isinstance(batches, list) or not 1 <= len(batches) <= 12000: + raise ValueError("invalid applied batch history") + identities = set() + last_period = "" + for batch in batches: + if not isinstance(batch, dict) or not isinstance(batch.get("batchId"), str) or not batch["batchId"] or batch["batchId"] in identities or not HASH.fullmatch(batch.get("digest", "")): + raise ValueError("invalid applied batch") + period = batch.get("period", "") + if not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", period) or period <= last_period: + raise ValueError("invalid applied batch order") + identities.add(batch["batchId"]) + last_period = period + if manifest.get("etag") is not None and (not isinstance(manifest["etag"], str) or len(manifest["etag"]) > 256 or "\n" in manifest["etag"] or "\r" in manifest["etag"]): + raise ValueError("invalid ETag") + codes = {} + if (directory / "codes").is_symlink(): + raise ValueError("symlink corpus directory") + for entry in entries: + if not isinstance(entry, dict): + raise ValueError("invalid corpus entry") + key = entry.get("sha256", "") + if not isinstance(key, str) or not HASH.fullmatch(key) or key in codes or entry.get("batchId") not in identities: + raise ValueError("invalid corpus identity") + path = directory / "codes" / (key + ".txt") + if path.is_symlink() or path.stat().st_size > MAX_CODE: + raise ValueError("invalid corpus file") + code = path.read_text(encoding="utf-8") + if digest(code.encode("utf-8")) != key: + raise ValueError("corrupt corpus file") + decode_code(code) + codes[key] = code + expected = {key + ".txt" for key in codes} + if {p.name for p in (directory / "codes").iterdir()} != expected: + raise ValueError("extra/missing corpus files") + if manifest.get("corpusDigest") != digest(canonical(entries)): + raise ValueError("corrupt corpus digest") + return manifest, codes + + +def apply_batch(batch, prior, codes, etag=None): + validate_batch(batch) + fingerprint = digest(canonical(batch)) + for applied in prior["batches"]: + if applied["batchId"] == batch["batchId"]: + if applied["digest"] != fingerprint: + raise ValueError("accepted batch identity changed contents") + return prior, codes + if prior["batches"] and batch["period"] <= prior["batches"][-1]["period"]: + return prior, codes + entries = list(prior["entries"]) + next_codes = dict(codes) + for entry in batch["builds"]: + key = entry["sha256"] + if key not in next_codes: + entries.append({"sha256": key, "batchId": batch["batchId"]}) + next_codes[key] = entry["code"] + entries = entries[-MAX_CORPUS:] + next_codes = {entry["sha256"]: next_codes[entry["sha256"]] for entry in entries} + manifest = {"schemaVersion": 1, "entries": entries, "batches": prior["batches"] + [ + {"batchId": batch["batchId"], "period": batch["period"], "digest": fingerprint}], + "etag": etag, "corpusDigest": digest(canonical(entries))} + return manifest, next_codes + + +class SameHostRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + old, new = urllib.parse.urlsplit(req.full_url), urllib.parse.urlsplit(newurl) + if new.scheme != "https" or (old.hostname, old.port) != (new.hostname, new.port): + raise ValueError("unapproved redirect") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def fetch_batch(url, etag, has_corpus, opener=None, sleep=time.sleep, clock=time.monotonic): + target = urllib.parse.urlsplit(url) + if target.scheme != "https" or target.username or target.password or target.fragment: + raise ValueError("feed must use HTTPS without credentials") + opener = opener or urllib.request.build_opener(SameHostRedirect()).open + deadline = clock() + 240 + for attempt in range(3): + headers = {"Accept": "application/json", "User-Agent": "PoB-CI-monthly-corpus/1"} + if etag: + headers["If-None-Match"] = etag + status, response_headers = 0, {} + try: + with opener(urllib.request.Request(url, headers=headers), timeout=30) as response: + status = response.status + response_headers = response.headers + if status == 200: + if response.headers.get_content_type() != "application/json": + raise ValueError("unexpected feed content type") + data = response.read(MAX_BATCH + 1) + if len(data) > MAX_BATCH: + raise ValueError("oversized feed") + accepted_etag = response.headers.get("ETag") + if accepted_etag and (len(accepted_etag) > 256 or "\r" in accepted_etag or "\n" in accepted_etag): + raise ValueError("invalid ETag") + return validate_batch(load_json_bytes(data)), accepted_etag + except urllib.error.HTTPError as exc: + status, response_headers = exc.code, exc.headers + exc.close() + except (urllib.error.URLError, TimeoutError): + status = 0 + if status == 304: + if has_corpus and etag: + return None, etag + if attempt == 0: + etag = None + continue + raise ValueError("unexpected 304 without accepted corpus") + if status not in (0, 429) and not 500 <= status < 600: + raise ValueError("feed unavailable or invalid HTTP response: %s" % status) + delay = 60 + retry = response_headers.get("Retry-After", "") + if retry.isdigit(): + delay = max(delay, int(retry)) + elif retry: + try: + delay = max(delay, (parsedate_to_datetime(retry) - datetime.now(timezone.utc)).total_seconds()) + except (TypeError, ValueError): + pass + if attempt == 2 or clock() + delay + 30 > deadline: + break + sleep(delay) + raise ValueError("feed retry budget exhausted") + + +def write_corpus(directory, manifest, codes): + directory = Path(directory) + if directory.exists() and any(directory.iterdir()): + raise ValueError("output must be a new/empty directory") + directory.mkdir(parents=True, exist_ok=True) + (directory / "codes").mkdir() + for key, code in codes.items(): + (directory / "codes" / (key + ".txt")).write_bytes(code.encode("utf-8")) + (directory / "manifest.json").write_bytes(canonical(manifest) + b"\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prior", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--batch", type=Path) + source.add_argument("--url") + source.add_argument("--materialize", action="store_true") + args = parser.parse_args() + prior, codes = read_corpus(args.prior, allow_empty=not args.materialize) + if args.materialize: + if args.output.exists(): + raise ValueError("materialization directory must not exist") + args.output.mkdir(parents=True) + for entry in prior["entries"]: + key = entry["sha256"] + (args.output / (key + ".xml")).write_bytes(decode_code(codes[key])) + print("Materialized %d verified builds; corpus %s" % (len(codes), prior["corpusDigest"])) + return + if args.batch: + if args.batch.stat().st_size > MAX_BATCH: + raise ValueError("oversized batch file") + batch, etag = load_json_bytes(args.batch.read_bytes()), None + else: + batch, etag = fetch_batch(args.url, prior["etag"], bool(codes)) + if batch is not None: + prior, codes = apply_batch(batch, prior, codes, etag) + write_corpus(args.output, prior, codes) + print("Retained %d builds across %d accepted monthly batches" % (len(codes), len(prior["batches"]))) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, KeyError, TypeError) as exc: + # No raw inputs or URL/HTTP exception bodies in CI logs. + raise SystemExit("Corpus refresh/materialization failed (%s); prior corpus preserved" % type(exc).__name__) diff --git a/tests/fixtures/test-build-batch-v1.json b/tests/fixtures/test-build-batch-v1.json new file mode 100644 index 00000000000..35da0cf34a8 --- /dev/null +++ b/tests/fixtures/test-build-batch-v1.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 1, + "batchId": "fixture-v1-september", + "period": "2026-09", + "generatedAt": "2026-09-01T00:00:00.000Z", + "patchVersion": "3.29", + "requestedCount": 100, + "count": 2, + "builds": [ + { + "code": "eJxdUM1Kw0AQvvsUw7xAop6U3QWtYAPaShe8yphM6-Jko7uTQt5esrEKPc03fH_MmBfSj-3-fgzShXhwpiBQSgfWV045DNHi9VuNIHxksXhTI7RCOW-oZ4sPI0vIikC55dit_hkvNHHCyplGuc9ArYYjz9izWrzEhYDQlWVHKeh0C5vt7vnu6aJJQ4Q1qalm0SL1rCe18TIoxFK0ZulZEYJy3xS6covNs_6i7Iz_DCKnCRzpXbizqGlkdOaR-xLnv7i1uJpaGSL_HX1VI3yPJEEnizWeuee6knqa2Znq7LE_hS94Rw", + "sha256": "ef7902502b0384dfda63c508a1f947a760041281866e21dc39e9ad995cd8a410" + }, + { + "code": "eJxdUMFOwzAMvfMVln-gBU6gJBLswCpBhxoJJC4oS81mkaSQuJP296gtA2mn92w_vydbPTvZbz7uRw49p51RMwNxeUfyQrnwkDRev9cIgQ4UNN7UCD64UloXSeMri98juOIp9av_fks-D9ElTxkroxqhWMB54QNN3JJovMRlANzPRecyy_EW2k33dPd40eQhwdqJqibRIrUkJ7WyYRBIc9qaQiRBYKHYzOPKLGuW5JcVo-wnh3BCoOS2gXqNkkdCox4oznb2i7zGznEheBvilunv-Ksa4Xt0geWoscYziylztj5hMao6e_APen17Fg", + "sha256": "4f945f69ee23b74eb9bfff9fcc89001d61a9bf464a4c49c33c7947ec63bd5353" + } + ] +} diff --git a/tests/test_build_diff_contract.py b/tests/test_build_diff_contract.py new file mode 100644 index 00000000000..35003babc7e --- /dev/null +++ b/tests/test_build_diff_contract.py @@ -0,0 +1,64 @@ +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "spec")) +import RunBuildDiff as runner + + +class RunnerTests(unittest.TestCase): + def test_nested_fixtures_are_included_and_duplicate_bytes_deduplicated(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + fixtures, inputs = root / 'fixtures', root / 'inputs' + (fixtures / '3.13').mkdir(parents=True) + inputs.mkdir() + xml = b'' + (fixtures / '3.13' / 'one.xml').write_bytes(xml) + (fixtures / 'copy.xml').write_bytes(xml) + runner.stage_fixtures(fixtures, inputs) + self.assertEqual([p.read_bytes() for p in inputs.iterdir()], [xml]) + with self.assertRaises(ValueError): + runner.stage_fixtures(root / 'typo', inputs) + empty = root / 'empty' + empty.mkdir() + with self.assertRaises(ValueError): + runner.stage_fixtures(empty, inputs) + + def test_empty_output_missing_inputs_and_extra_outputs_fail(self): + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) + with self.assertRaises(ValueError): + runner.validate_outputs(p, ["one.xml"]) + (p / "one.xml.build").write_text("") + with self.assertRaises(ValueError): + runner.validate_outputs(p, ["one.xml"]) + (p / "one.xml.build").write_text('') + runner.validate_outputs(p, ["one.xml"]) + self.assertEqual(runner.saved_stats(p / "one.xml.build"), {("PlayerStat", "Life"): "123"}) + (p / "extra.build").write_text("extra") + with self.assertRaises(ValueError): + runner.validate_outputs(p, ["one.xml"]) + + def test_subprocess_failure_and_timeout_propagate(self): + with self.assertRaises(subprocess.CalledProcessError): + runner.run([sys.executable, "-c", "raise SystemExit(7)"]) + with patch.object(runner.subprocess, "run", side_effect=subprocess.TimeoutExpired("test", 1)): + with self.assertRaises(subprocess.TimeoutExpired): + runner.run(["test"], timeout=1) + + def test_missing_corpus_is_not_implicit_fixtures_mode(self): + result = subprocess.run([sys.executable, str(Path(runner.__file__)), "--base", "HEAD", "--head", "HEAD", "--output", "unused"], capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"choose --corpus or explicit --fixtures-only", result.stderr) + + def test_both_runtime_sources_are_readonly_and_outputs_writable(self): + self.assertIn("readonly", runner.docker_mount(Path("source"), "/workdir")[1]) + self.assertNotIn("readonly", runner.docker_mount(Path("output"), "/outputs", False)[1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_diff_output.lua b/tests/test_diff_output.lua new file mode 100644 index 00000000000..8c42647761b --- /dev/null +++ b/tests/test_diff_output.lua @@ -0,0 +1,47 @@ +-- Run from the repository root in Dockerfile.test-builds' Linux image. +local function saved(player, minion, reverse) + local lines = { "" } + local function stat(kind, key, value) + local attrs = reverse and ('value="' .. value .. '" stat="' .. key .. '"') + or ('stat="' .. key .. '" value="' .. value .. '"') + table.insert(lines, "<" .. kind .. " " .. attrs .. "/>") + end + if player then stat("PlayerStat", "Life", player) end + if minion then stat("MinionStat", "TotalDPS", minion) end + table.insert(lines, "") + return table.concat(lines, "\n") +end + +local cases = { + { "attribute order", saved(100, 50), saved(100, 50, true), 0 }, + { "player difference", saved(100, 50), saved(101, 50, true), 1 }, + { "minion difference", saved(100, 50), saved(100, 51, true), 1 }, + { "removed minion", saved(100, 50), saved(100, nil, true), 1 }, + { "added minion", saved(100), saved(100, 50, true), 1 }, + { "missing player stats", saved(100), saved(nil, 50, true), 2 }, + { "missing file", saved(100), nil, 2 }, +} +for _, case in ipairs(cases) do + local base, head, log = os.tmpname(), os.tmpname(), os.tmpname() + local function write(path, value) + if value then + local file = assert(io.open(path, "wb")) + file:write(value) + file:close() + else + os.remove(path) + end + end + write(base, case[2]) + write(head, case[3]) + local status = os.execute("luajit spec/DiffOutput.lua " .. head .. " " .. base .. " > " .. log .. " 2>&1") + local file = assert(io.open(log, "rb")) + local details = file:read("*a") + file:close() + os.remove(base) + os.remove(head) + os.remove(log) + assert(status == case[4] * 256, case[1] .. ": " .. details) + print("PASS " .. case[1]) +end +print("Passed " .. #cases .. " output comparison tests") diff --git a/tests/test_update_build_corpus.py b/tests/test_update_build_corpus.py new file mode 100644 index 00000000000..688c108307c --- /dev/null +++ b/tests/test_update_build_corpus.py @@ -0,0 +1,133 @@ +import base64 +from copy import deepcopy +from email.message import Message +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +import urllib.error +import zlib + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "spec")) +import UpdateBuildCorpus as corpus + + +def entry(number): + xml = '%d' % number + code = base64.urlsafe_b64encode(zlib.compress(xml.encode())).decode().rstrip("=") + return {"code": code, "sha256": corpus.digest(code.encode())} + + +def batch(month=1, start=0, count=100): + return {"schemaVersion": 1, "batchId": "batch-%d" % month, "period": "2026-%02d" % month, + "generatedAt": "2026-%02d-01T00:00:00.000Z" % month, "patchVersion": "3.29", + "requestedCount": 100, "count": count, "builds": [entry(i) for i in range(start, start + count)]} + + +class CorpusTests(unittest.TestCase): + def test_shared_provider_wire_fixture(self): + fixture = Path(__file__).parent / "fixtures" / "test-build-batch-v1.json" + self.assertEqual(corpus.validate_batch(corpus.load_json_bytes(fixture.read_bytes()))["count"], 2) + + def test_fifo_500_repeats_duplicates_stale_and_conflicting_batches(self): + manifest, codes = corpus.empty_manifest(), {} + for month in range(1, 7): + manifest, codes = corpus.apply_batch(batch(month, (month-1)*100), manifest, codes) + self.assertEqual(len(codes), 500) + self.assertEqual(manifest["entries"][0]["sha256"], entry(100)["sha256"]) + same = corpus.apply_batch(batch(6, 500), manifest, codes) + self.assertEqual(same, (manifest, codes)) + duplicate, duplicate_codes = corpus.apply_batch(batch(7, 500), manifest, codes) + self.assertEqual(duplicate["entries"], manifest["entries"]) + self.assertEqual(len(duplicate["batches"]), 7) + self.assertEqual(duplicate_codes, codes) + self.assertEqual(corpus.apply_batch(batch(1, 0), manifest, codes), (manifest, codes)) + changed = batch(6, 700) + with self.assertRaises(ValueError): + corpus.apply_batch(changed, manifest, codes) + + def test_shortfall_and_disk_roundtrip_detect_corruption(self): + m, c = corpus.apply_batch(batch(count=2), corpus.empty_manifest(), {}, '"accepted"') + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "corpus" + corpus.write_corpus(path, m, c) + self.assertEqual(corpus.read_corpus(path), (m, c)) + with self.assertRaises(ValueError): + corpus.write_corpus(path, m, c) + next((path / "codes").iterdir()).write_text("corrupt") + with self.assertRaises(ValueError): + corpus.read_corpus(path) + + def test_rejects_malformed_envelopes_before_mutating_prior(self): + valid = batch(count=1) + variants = [] + for field, value in (("schemaVersion", True), ("count", 0), ("count", True), + ("count", 101), ("period", "2026-99"), ("generatedAt", "2026-02-30T00:00:00.000Z"), + ("batchId", ""), ("patchVersion", None)): + broken = deepcopy(valid); broken[field] = value; variants.append(broken) + broken = deepcopy(valid); broken["builds"][0]["sha256"] = broken["builds"][0]["sha256"].upper(); variants.append(broken) + broken = deepcopy(valid); broken["builds"][0]["code"] += " "; variants.append(broken) + broken = deepcopy(valid); broken["builds"] *= 2; broken["count"] = 2; variants.append(broken) + for broken in variants: + with self.subTest(broken=broken.keys()), self.assertRaises(ValueError): + corpus.validate_batch(broken) + + def test_rejects_bombs_trailing_data_and_doctype(self): + for data in (b"x" * (corpus.MAX_XML+1), b']>', + ''.encode("utf-16-le")): + code = base64.urlsafe_b64encode(zlib.compress(data)).decode() + with self.assertRaises(ValueError): + corpus.decode_code(code) + packed = base64.urlsafe_b64decode(entry(0)["code"] + "=" * (-len(entry(0)["code"]) % 4)) + b"trailing" + with self.assertRaises(ValueError): + corpus.decode_code(base64.urlsafe_b64encode(packed).decode()) + + def test_fetch_retries_preserves_etag_and_stops_on_invalid_status(self): + headers = Message(); headers["Content-Type"] = "application/json"; headers["ETag"] = '"new"' + delays, calls = [], [] + class Response(io.BytesIO): + status = 200 + def opener(request, timeout): + calls.append(request) + if len(calls) == 1: + retry = Message(); retry["Retry-After"] = "65" + raise urllib.error.HTTPError(request.full_url, 429, "rate limit", retry, io.BytesIO()) + response = Response(json.dumps(batch(count=1)).encode()); response.headers = headers + return response + accepted, etag = corpus.fetch_batch("https://api.pob.codes/test-builds", '"old"', True, opener=opener, sleep=delays.append, clock=lambda: 0) + self.assertEqual(delays, [65]); self.assertEqual(etag, '"new"'); self.assertEqual(accepted["count"], 1) + self.assertEqual(calls[1].get_header("If-none-match"), '"old"') + for status in (400, 404): + def unavailable(request, timeout): + raise urllib.error.HTTPError(request.full_url, status, "unavailable", Message(), io.BytesIO()) + with self.assertRaises(ValueError): + corpus.fetch_batch("https://api.pob.codes/test-builds", None, False, opener=unavailable) + + def test_304_requires_valid_prior_and_retries_without_etag_once(self): + calls = [] + def opener(request, timeout): + calls.append(request) + raise urllib.error.HTTPError(request.full_url, 304, "not modified", Message(), io.BytesIO()) + self.assertEqual(corpus.fetch_batch("https://api.pob.codes/test-builds", '"ok"', True, opener=opener), (None, '"ok"')) + with self.assertRaises(ValueError): + corpus.fetch_batch("https://api.pob.codes/test-builds", '"bad"', False, opener=opener) + self.assertIsNone(calls[-1].get_header("If-none-match")) + + def test_invalid_cli_refresh_leaves_prior_bytes_unchanged(self): + with tempfile.TemporaryDirectory() as temp: + temp = Path(temp); prior = temp / "prior" + manifest, codes = corpus.apply_batch(batch(count=1), corpus.empty_manifest(), {}) + corpus.write_corpus(prior, manifest, codes) + original = (prior / "manifest.json").read_bytes() + (temp / "bad.json").write_text("{}") + result = subprocess.run([sys.executable, str(Path(corpus.__file__)), "--prior", str(prior), "--batch", str(temp / "bad.json"), "--output", str(temp / "next")], capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual((prior / "manifest.json").read_bytes(), original) + self.assertFalse((temp / "next").exists()) + + +if __name__ == "__main__": + unittest.main()