Tests #57
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # cspell:ignore luarocks dyld DYLD aegisub autoload | |
| name: Tests | |
| on: | |
| pull_request: | |
| branches: | |
| - 'main' | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: 'Release tag or branch to scan.' | |
| required: true | |
| default: 'main' | |
| jobs: | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - windows-latest | |
| - macos-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.ref }} | |
| # Windows uses MSVC for LuaJIT and the C rocks; activate the toolchain (sets VCINSTALLDIR + cl). | |
| - uses: step-security/msvc-dev-cmd@v1 | |
| if: runner.os == 'Windows' | |
| - uses: leafo/gh-actions-lua@v13 | |
| if: runner.os != 'Windows' | |
| with: | |
| luaVersion: "luajit-2.1" | |
| # DepCtrl relies on Lua 5.2 features (table.unpack, __pairs/__len) | |
| luaCompileFlags: "XCFLAGS=-DLUAJIT_ENABLE_LUA52COMPAT" | |
| # Cache the MSVC LuaJIT build, keyed on the live v2.1 commit so it refreshes when upstream moves | |
| # (leafo/gh-actions-lua already caches LuaJIT on Linux/macOS). | |
| - name: Resolve LuaJIT revision | |
| if: runner.os == 'Windows' | |
| id: luajit-rev | |
| shell: pwsh | |
| run: | | |
| $line = git ls-remote https://github.com/LuaJIT/LuaJIT.git refs/heads/v2.1 | Select-Object -First 1 | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "sha=$(($line -split '\s+')[0])" | |
| - name: Cache LuaJIT | |
| if: runner.os == 'Windows' | |
| id: cache-luajit | |
| uses: actions/cache@v6 | |
| with: | |
| path: .lua | |
| key: luajit-msvc-x64-${{ steps.luajit-rev.outputs.sha }} | |
| # The C rocks build with MSVC on Windows, so LuaJIT must too, for a matching lua51.lib and CRT. | |
| # The Lua action only builds LuaJIT with mingw, so build it here with msvcbuild.bat and install | |
| # into .lua/ using the layout the luarocks action expects. | |
| - name: Build LuaJIT with MSVC | |
| if: runner.os == 'Windows' && steps.cache-luajit.outputs.cache-hit != 'true' | |
| shell: pwsh | |
| run: | | |
| git clone --depth 1 --branch v2.1 https://github.com/LuaJIT/LuaJIT.git "$env:RUNNER_TEMP\luajit" | |
| Set-Location "$env:RUNNER_TEMP\luajit\src" | |
| cmd /c msvcbuild.bat lua52compat | |
| $lua = "$env:GITHUB_WORKSPACE\.lua" | |
| New-Item -ItemType Directory -Force -Path "$lua\bin", "$lua\lib", "$lua\include\luajit-2.1" | Out-Null | |
| Copy-Item luajit.exe -Destination "$lua\bin\lua.exe" | |
| Copy-Item lua51.dll -Destination "$lua\bin" | |
| Copy-Item -Path lua51.dll, lua51.lib -Destination "$lua\lib" | |
| Copy-Item -Path lua.h, lualib.h, lauxlib.h, luaconf.h, lua.hpp, luajit.h -Destination "$lua\include\luajit-2.1" | |
| # .lua/bin holds lua.exe whether just built or restored from cache; put it on PATH either way. | |
| - name: Add LuaJIT to PATH | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: Add-Content -Path $env:GITHUB_PATH -Value "$env:GITHUB_WORKSPACE\.lua\bin" | |
| # Installs LuaRocks and builds the C rocks with the active toolchain — MSVC on Windows | |
| # (VCINSTALLDIR set above), the system gcc/clang on Linux/macOS. | |
| - uses: luarocks/gh-actions-luarocks@v7 | |
| # PCRE2 backs the headless aegisub.re stand-in. It isn't on the Linux or macOS runners either, | |
| # and lrexlib-pcre2 finds it through pkg-config / the brew prefix once installed. | |
| - name: Install PCRE2 | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y libpcre2-dev | |
| - name: Install PCRE2 | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install pcre2 | |
| prefix="$(brew --prefix pcre2)" | |
| luarocks --lua-version=5.1 config variables.PCRE2_INCDIR "$prefix/include" | |
| luarocks --lua-version=5.1 config variables.PCRE2_LIBDIR "$prefix/lib" | |
| # Two C rocks need libraries the Windows runner doesn't ship: lzlib (pulled in by Pegasus) links | |
| # zlib, and lrexlib-pcre2 links PCRE2. Cache the vcpkg tree holding both; the key carries the | |
| # library list so adding one rebuilds it. Path matches VCPKG_INSTALLATION_ROOT. | |
| - name: Cache native libraries | |
| if: runner.os == 'Windows' | |
| id: cache-native | |
| uses: actions/cache@v6 | |
| with: | |
| path: C:\vcpkg\installed\x64-windows | |
| key: native-x64-windows-zlib-pcre2-1 | |
| - name: Set up zlib and PCRE2 | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| env: | |
| CACHE_HIT: ${{ steps.cache-native.outputs.cache-hit }} | |
| run: | | |
| $root = "$env:VCPKG_INSTALLATION_ROOT\installed\x64-windows" | |
| if ($env:CACHE_HIT -ne 'true') { | |
| & "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install zlib:x64-windows pcre2:x64-windows | |
| # zlib 1.3.x's import lib is z.lib, but lzlib looks for zlib.lib. | |
| Copy-Item "$root\lib\z.lib" "$root\lib\zlib.lib" | |
| } | |
| luarocks --lua-version=5.1 config variables.ZLIB_INCDIR "$root\include" | |
| luarocks --lua-version=5.1 config variables.ZLIB_LIBDIR "$root\lib" | |
| luarocks --lua-version=5.1 config variables.PCRE2_INCDIR "$root\include" | |
| luarocks --lua-version=5.1 config variables.PCRE2_LIBDIR "$root\lib" | |
| # rex_pcre2.dll resolves pcre2-8.dll when it loads, so the runtime has to be on PATH | |
| Add-Content -Path $env:GITHUB_PATH -Value "$root\bin" | |
| # moonscript loader | |
| - run: luarocks --lua-version=5.1 install moonscript | |
| # lfs (Aegisub provides an internal copy) | |
| - run: luarocks --lua-version=5.1 install luafilesystem | |
| # CLI argument parsing | |
| - run: luarocks --lua-version=5.1 install argparse | |
| # mock HTTP server dependencies | |
| - run: luarocks --lua-version=5.1 install luasocket | |
| - run: luarocks --lua-version=5.1 install copas | |
| - run: luarocks --lua-version=5.1 install pegasus | |
| # json schema validation | |
| - run: luarocks --lua-version=5.1 install lua-schema | |
| - run: luarocks --lua-version=5.1 install lpeg | |
| # regex engine behind the headless aegisub.re stand-in | |
| - run: luarocks --lua-version=5.1 install lrexlib-pcre2 | |
| - name: Run tests | |
| timeout-minutes: 5 | |
| run: lua depctrl.lua test | |
| # Per-platform status check; the raw CTRF reports are uploaded next for the merged-comment job. | |
| - name: Publish status check | |
| uses: ctrf-io/github-test-reporter@v1 | |
| if: ${{ !cancelled() }} | |
| with: | |
| report-path: ./ctrf/*.json | |
| status-check: true | |
| status-check-name: Test results (${{ matrix.os }}) | |
| annotate: false | |
| use-suite-name: true | |
| summary-report: true | |
| failed-report: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload CTRF reports | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ctrf-report-${{ matrix.os }} | |
| path: ctrf/*.json | |
| # Lint the module sources' LuaCATS annotations: missing docs on public members and | |
| # @param/@return that disagree with the real signature fail the build. This also emits every | |
| # definition and verifies it is loadable Lua, so it doubles as type-generation coverage. | |
| - name: Check annotations | |
| if: ${{ !cancelled() }} | |
| run: lua depctrl.lua generate-types --check | |
| # Smoke-test that documentation generation runs over the real feed without a parse or emit | |
| # failure. --site none skips scaffolding; the output goes to a throwaway directory. | |
| - name: Smoke-test doc generation | |
| if: ${{ !cancelled() }} | |
| run: lua depctrl.lua generate-docs --site none -o "${{ runner.temp }}/docs" | |
| # ---------------------------------------------------------------------------------------- | |
| # TEMPORARY — text-extents corpus on macOS. Lives here rather than in its own workflow only | |
| # because workflow_dispatch is offered for workflows on the default branch, so a new file on | |
| # the investigation branch could never be dispatched. Every step below is gated on a manual | |
| # dispatch, so pull-request runs are untouched. Delete this whole block, and the CORPUS_URL | |
| # secret, once the numbers are in. | |
| # | |
| # The corpus is licensed commercial fonts, so it is not in this public repository. CORPUS_URL | |
| # points at a share serving `corpus-fonts.tar.gz`, a gzipped tar of the `corpus-fonts/` | |
| # directory; it is a secret rather than an input so the link stays out of the repository and | |
| # out of the run log. | |
| # ---------------------------------------------------------------------------------------- | |
| - name: Corpus — install FreeType and fontconfig | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() }} | |
| continue-on-error: true | |
| run: brew install freetype fontconfig | |
| # The share returns the tarball from its base URL with `compress=false`, and an HTML page under | |
| # HTTP 200 from its `/download` endpoint, so the stored link is trimmed back to the one form | |
| # that returns bytes and what arrives is identified by magic number before it is unpacked. | |
| # | |
| # Without the corpus every measurement below is meaningless, so a failure here fails the job | |
| # rather than letting it go green having measured substituted fonts. A face the share omits | |
| # substitutes just as silently, so the unpacked count is checked against corpus.lua. | |
| - name: Corpus — fetch and unpack | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() }} | |
| env: | |
| CORPUS_URL: ${{ secrets.CORPUS_URL }} | |
| run: | | |
| url=${CORPUS_URL%%\?*} | |
| url=${url%/} | |
| url=${url%/download} | |
| # a refusal page quotes the share token back, and this log is public | |
| token=${url##*/} | |
| [ ${#token} -lt 12 ] || echo "::add-mask::$token" | |
| # a stock Safari user agent, since the share sits behind bot filtering that has already | |
| # turned one run away | |
| # cspell:ignore KHTML — a fixed token in every browser's user agent string | |
| code=$(curl -sS -L --retry 3 -D headers.txt -o corpus.bin -w '%{http_code}' \ | |
| -A 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15' \ | |
| "$url?compress=false") || code=000 | |
| magic=$(head -c 2 corpus.bin | od -An -tx1 | tr -d ' ') | |
| case $code:$magic in | |
| 200:1f8b) | |
| tar xzf corpus.bin -C text-extents-investigation | |
| ;; | |
| 200:504b) | |
| unzip -q corpus.bin -d corpus-zip | |
| inner=$(find corpus-zip -name '*.tar.gz' | head -1) | |
| [ -n "$inner" ] || { echo "the share returned a zip holding no tarball" >&2; exit 1; } | |
| tar xzf "$inner" -C text-extents-investigation | |
| ;; | |
| *) | |
| echo "the share answered HTTP $code with $(wc -c < corpus.bin) bytes beginning $magic," >&2 | |
| echo "which is neither gzip nor zip; its headers and the head of its body follow" >&2 | |
| grep -viE '^(location|set-cookie):' headers.txt >&2 || true | |
| head -c 800 corpus.bin >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| staged=$(ls text-extents-investigation/corpus-fonts | wc -l | tr -d ' ') | |
| expected=$(lua -e 'local c = assert(loadfile("text-extents-investigation/corpus.lua"))() | |
| local n = 0 | |
| for _, f in ipairs(c) do if f.staged then n = n + 1 end end | |
| io.write(n)') | |
| echo "unpacked $staged font files, corpus.lua expects $expected" | |
| [ "$staged" = "$expected" ] || { echo "corpus is incomplete" >&2; exit 1; } | |
| # CoreText reads the user font directory, so the corpus has to be installed rather than merely | |
| # present. fontconfig is pointed at the same directory so both backends see the same files. | |
| - name: Corpus — install the fonts | |
| id: fonts | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() }} | |
| run: | | |
| mkdir -p ~/Library/Fonts | |
| cp text-extents-investigation/corpus-fonts/* ~/Library/Fonts/ | |
| cat > "$RUNNER_TEMP/fonts.conf" <<CONF | |
| <?xml version="1.0"?> | |
| <!DOCTYPE fontconfig SYSTEM "fonts.dtd"> | |
| <fontconfig> | |
| <include>$(brew --prefix)/etc/fonts/fonts.conf</include> | |
| <dir>$HOME/Library/Fonts</dir> | |
| <dir>$GITHUB_WORKSPACE/text-extents-investigation/corpus-fonts</dir> | |
| </fontconfig> | |
| CONF | |
| echo "FONTCONFIG_FILE=$RUNNER_TEMP/fonts.conf" >> "$GITHUB_ENV" | |
| # Homebrew's lib directory is not on the default dyld search path, and the FFI bindings | |
| # load the shared libraries by bare soname. | |
| echo "DYLD_LIBRARY_PATH=$(brew --prefix)/lib" >> "$GITHUB_ENV" | |
| "$(brew --prefix fontconfig)/bin/fc-match" Compendium file || true | |
| # Every dump below is gated on the fonts being installed. A missing family substitutes without | |
| # complaint, so an ungated dump produces a full set of rows measured against the wrong faces. | |
| # | |
| # `> file` creates the file before the command can fail, so each dump lands on a temp name and | |
| # is moved into place only once it succeeded. The artifact then never holds an empty file. | |
| - name: Corpus — dump CoreText | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() && steps.fonts.outcome == 'success' }} | |
| run: | | |
| lua text-extents-investigation/dump-extents.lua coretext > coretext.partial | |
| mv coretext.partial coretext.tsv | |
| # The same corpus through the FreeType backend on this machine, as the oracle to diff against: | |
| # it is byte-exact with GDI, so agreeing with it verifies CoreText without shipping GDI's numbers. | |
| - name: Corpus — dump FreeType | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() && steps.fonts.outcome == 'success' }} | |
| run: | | |
| lua text-extents-investigation/dump-extents.lua freetype-windows > freetype-macos.partial | |
| mv freetype-macos.partial freetype-macos.tsv | |
| # The macOS contract through both libraries. The CoreText one is what a script on a Mac actually | |
| # gets, and the FreeType one derives the same contract from the same tables, so the two diffed | |
| # against each other separate a mistake in the derivation from one in the CTFont call layer. | |
| - name: Corpus — dump the macOS contract | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() && steps.fonts.outcome == 'success' }} | |
| run: | | |
| lua text-extents-investigation/dump-extents.lua coretext-mac > coretext-mac.partial | |
| mv coretext-mac.partial coretext-mac.tsv | |
| lua text-extents-investigation/dump-extents.lua freetype-mac > freetype-mac-macos.partial | |
| mv freetype-mac-macos.partial freetype-mac-macos.tsv | |
| # Aegisub executes an autoload script's top-level body at startup, so appending a call to the | |
| # macro's entry point runs the whole comparison without touching the GUI — no synthetic | |
| # keystrokes, so no Accessibility grant to arrange. | |
| - name: Corpus — dump what Aegisub itself reports | |
| if: ${{ runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && !cancelled() && steps.fonts.outcome == 'success' }} | |
| continue-on-error: true | |
| timeout-minutes: 20 | |
| run: | | |
| brew install --cask aegisub | |
| AUTOLOAD="$HOME/Library/Application Support/Aegisub/automation/autoload" | |
| mkdir -p "$AUTOLOAD" | |
| cp text-extents-investigation/verify-in-aegisub.moon "$AUTOLOAD/verify.moon" | |
| # Aegisub installs its reporting API — log, progress, debug, dialog, cancel — for the | |
| # duration of a macro callback only, so all of it is stood in for before the entry point is | |
| # called directly, with the report routed to stderr to reach this log. The call is guarded | |
| # so a failure surfaces here instead of dying silently inside a GUI app, and it leaves a | |
| # marker beside the dump so the wait below ends on a failure as well as on a result. | |
| cat >> "$AUTOLOAD/verify.moon" <<'RUN' | |
| aegisub.progress = {title: (->), set: (->), is_cancelled: (-> false)} | |
| aegisub.log = (level, msg) -> | |
| msg = level unless msg | |
| io.stderr\write tostring msg | |
| aegisub.debug = {out: aegisub.log} | |
| aegisub.dialog = {display: (-> nil)} | |
| aegisub.cancel = -> | |
| ok, startupErr = pcall verify | |
| unless ok | |
| io.stderr\write "verify failed: #{startupErr}\n" | |
| marker = io.open "#{os.getenv('DEPCTRL_EXTENTS_DUMP')}.failed", "w" | |
| if marker | |
| marker\write tostring startupErr | |
| marker\close! | |
| RUN | |
| export DEPCTRL_REPO_MODULES="$GITHUB_WORKSPACE/modules" | |
| export DEPCTRL_EXTENTS_DUMP="$GITHUB_WORKSPACE/aegisub-macos.tsv" | |
| # Launched directly rather than through `open`, which would hand the app launchd's | |
| # environment instead of this shell's and lose both variables above. | |
| /Applications/Aegisub.app/Contents/MacOS/aegisub & | |
| # the run writes its dump at the end, so poll for it rather than guessing a duration | |
| for _ in $(seq 1 90); do | |
| [ -s "$DEPCTRL_EXTENTS_DUMP" ] && break | |
| [ -f "$DEPCTRL_EXTENTS_DUMP.failed" ] && break | |
| sleep 10 | |
| done | |
| pkill -if aegisub || true | |
| if [ -s "$DEPCTRL_EXTENTS_DUMP" ]; then | |
| echo "Aegisub wrote $(wc -l < "$DEPCTRL_EXTENTS_DUMP") rows" | |
| else | |
| echo "Aegisub produced no dump; its log follows" >&2 | |
| cat "$HOME/Library/Application Support/Aegisub/log/"* 2>/dev/null | tail -80 || true | |
| fi | |
| # always(), so a cancelled run still yields whatever dumps had already been written; the step | |
| # before this one is the long one, and its result is worth keeping when the wait is cut short. | |
| - name: Corpus — upload the dumps | |
| if: ${{ always() && runner.os == 'macOS' && github.event_name == 'workflow_dispatch' && steps.fonts.outcome == 'success' }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: text-extents-macos | |
| path: | | |
| coretext.tsv | |
| coretext-mac.tsv | |
| freetype-macos.tsv | |
| freetype-mac-macos.tsv | |
| aegisub-macos.tsv | |
| if-no-files-found: error | |
| # Post one PR comment covering every platform. Each test's suite is prefixed with its platform, then | |
| # the per-platform reports are merged into one the reporter renders. | |
| report: | |
| needs: test | |
| if: ${{ !cancelled() && github.event_name == 'pull_request' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: ctrf-report-* | |
| path: reports | |
| - name: Prefix each test's suite with its platform | |
| run: | | |
| mkdir -p merged-input | |
| for dir in reports/ctrf-report-*/; do | |
| os=$(basename "$dir" | sed 's/^ctrf-report-//') | |
| for f in "$dir"*.json; do | |
| jq --arg os "$os" '.results.tests = ((.results.tests // []) | map(.suite = ("[" + $os + "] " + (.suite // ""))))' "$f" > "merged-input/${os}-$(basename "$f")" | |
| done | |
| done | |
| - name: Merge into one report | |
| run: npx --yes ctrf-cli merge merged-input --output merged.json | |
| # Per-platform pass/fail line for the comment title, read off the [platform]-prefixed suites. | |
| - name: Per-platform summary line | |
| id: summary | |
| run: | | |
| line=$(jq -r ' | |
| [ .results.tests[] | { os: (.suite | capture("^\\[(?<os>[^\\]]+)\\]").os), status } ] | |
| | group_by(.os) | |
| | map({ os: .[0].os, | |
| p: ([ .[] | select(.status == "passed") ] | length), | |
| f: ([ .[] | select(.status == "failed") ] | length) }) | |
| | map("\(.os) ✅\(.p)" + (if .f > 0 then " ❌\(.f)" else "" end)) | |
| | join(" · ") | |
| ' merged-input/merged.json) | |
| echo "title=Test results — $line" >> "$GITHUB_OUTPUT" | |
| - name: Publish merged test report | |
| uses: ctrf-io/github-test-reporter@v1 | |
| with: | |
| report-path: merged-input/merged.json | |
| pull-request: true | |
| overwrite-comment: true | |
| comment-tag: all-platforms | |
| title: ${{ steps.summary.outputs.title }} | |
| use-suite-name: true | |
| group-by: suite | |
| # per-test listing with 3k+ passing tests × 3 platforms would blow past GitHub's 65 KB comment limit | |
| suite-folded-report: false | |
| summary-report: true | |
| failed-report: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |