diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00abe5ec24d..ba7a41c7633 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -535,9 +535,10 @@ jobs: # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, # so the same crash that Linux absorbs failed the whole promotion here. # - # Keep the signature list in sync with `is_bun_runtime_crash` in - # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on - # the first attempt — only the crash signature is retried, exactly once. + # The classifier is `is_bun_runtime_crash` from scripts/ci/bun-crash-signatures.sh, which + # this leg, the Windows leg, the macOS control and the Linux batch runner all source. It used + # to be four inline copies kept in sync by a test; one definition cannot drift. An assertion + # failure still fails on the first attempt — only a crash is retried, exactly once. - name: Test env: MACOS_TEST_SHARD: ${{ matrix.shard }} @@ -546,6 +547,8 @@ jobs: # errexit so a Bun crash reaches PIPESTATUS and the bounded retry. set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh run_macos_suite() { local suite_log suite_status attempt @@ -559,7 +562,7 @@ jobs: rm -f "$suite_log" return 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." rm -f "$suite_log" return "$suite_status" @@ -685,15 +688,17 @@ jobs: # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, # so the same crash that Linux absorbs failed the whole promotion here. # - # Keep the signature list in sync with `is_bun_runtime_crash` in - # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on - # the first attempt — only the crash signature is retried, exactly once. + # The classifier is `is_bun_runtime_crash` from scripts/ci/bun-crash-signatures.sh, shared + # with every other lane. An assertion failure still fails on the first attempt — only a + # crash is retried, exactly once. - name: Test run: | # GitHub Actions starts bash `run:` blocks with `-e`. Disable # errexit so a Bun crash reaches PIPESTATUS and the bounded retry. set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)" for attempt in 1 2; do # --timeout: Bun's default 5s per-test ceiling is the recurring flake @@ -708,7 +713,7 @@ jobs: if [ "$suite_status" -eq 0 ]; then exit 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." exit "$suite_status" fi @@ -834,6 +839,8 @@ jobs: run: | set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh suite_log="$(mktemp -t ocx-windows-suite.XXXXXX)" for attempt in 1 2; do bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/6 2>&1 | tee "$suite_log" @@ -841,7 +848,7 @@ jobs: if [ "$suite_status" -eq 0 ]; then exit 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::Windows shard ${{ matrix.shard }}/6 failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." exit "$suite_status" fi diff --git a/Dockerfile b/Dockerfile index 8d3e72c6194..5987bfa991d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Keep the runtime aligned with package.json and pin the multi-platform image index. -ARG BUN_IMAGE=oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5084b95fe9473970bb6bcb5b232ab1630f0546895 +ARG BUN_IMAGE=oven/bun:1.4.0@sha256:5ff609364c049b54eb0ff560ec96319729a972078ef2c755d758f0c6ef89c2d6 FROM ${BUN_IMAGE} AS build WORKDIR /home/bun/app diff --git a/bun.lock b/bun.lock index 6161766917a..c4619c28666 100644 --- a/bun.lock +++ b/bun.lock @@ -8,11 +8,11 @@ "@bufbuild/protobuf": "^2.14.0", "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/keyring": "1.3.0", - "bun": "1.4.2", + "bun": "1.4.0", "zod": "4.4.3", }, "devDependencies": { - "@types/bun": "1.4.2", + "@types/bun": "1.4.0", "typescript": "7.0.2", }, }, @@ -60,31 +60,31 @@ "@napi-rs/keyring-win32-x64-msvc": ["@napi-rs/keyring-win32-x64-msvc@1.3.0", "", { "os": "win32", "cpu": "x64" }, "sha512-4DnCWXwDc0HRKwyRlG5y0VhKZW2tNRQfKKfyj6IX/KWfDNyq9hn4n+GL1auyDcOO/v8PwnhmYo2+rOOqCkvvOg=="], - "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.4.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-MXdZkP1featqxZ+/VTXWG1BVjM4OGBehVY2Q88EeUj/7L0UMeCGItmyPYTN+wxvlGJ6F66JEtzsw+GvQWewnag=="], + "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-GCpf8QuFLsyioVawP5HrMxA1ZRBlu6Hq9RNnSc3UTUWAzIxBso9trjoZczw1HdgpqSssFkszfIV2zmOzFTjhkw=="], - "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.4.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-gZTxZuLjkUhAWjTETu3tw0WhsEdNkJ64daj60ybhPf835a2yollV3yTkK9JozvzKPx4TRFzLSl8C+U525pxVbw=="], + "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.4.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-cIrhwOr0SPEraewznhC+c/k6TG8bwFn5uZ4EJuXwjiKJLcAF36q7/bGjWkeXSe48JwMcPRUR054JXF7+cRwSSA=="], - "@oven/bun-freebsd-aarch64": ["@oven/bun-freebsd-aarch64@1.4.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-SMNItMw1Z8QeeQVKnw8jA7xQNkeXdP+OPgin4Wi/QTx/B8RHHLnuZfqmFy7NtVeT2NF0kKYppW4WWd2CCYZjhQ=="], + "@oven/bun-freebsd-aarch64": ["@oven/bun-freebsd-aarch64@1.4.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-09x7wnjMR6M5KGBDBhVl2CpfoCIQOkVDbPX2KfIhpXv4N6grbWE7dfLPw/Ydi9gaUMGhU7UKhoz444Nu6RCycA=="], - "@oven/bun-freebsd-x64": ["@oven/bun-freebsd-x64@1.4.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-THbPKXhO54N0DpFRKZNDZpQ7dpbX0bWASuARckAUS9wRtFIHsiY+uULXJvxJGo2YD1YewvXQ4G8Fj7XT5oBCiw=="], + "@oven/bun-freebsd-x64": ["@oven/bun-freebsd-x64@1.4.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-dRwzti/qJqV1HWplU27iUWUqp+f2DtFSf2yqQKSb+HH2dDOC//Uqd9u/A5h1DMsLszfP5OGP9UwQIKxVwFODaA=="], - "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-3BBP9ovJ2RGHFH6Ae1CAtxNtG1+YY6GD6rmYbsUosoAk9+OEl6zeDQ/k4fBkc6dYOJCtWnx8hUxzNzQATSmvYQ=="], + "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.4.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-Y5yAtCbHK6JjprXEtkdklDQFPADgs+CkfcliyY5g4JJ8baGHyQSrfpSkX3XVJ2C+aBLsdwNDdW+oczMsAwx6uA=="], - "@oven/bun-linux-aarch64-android": ["@oven/bun-linux-aarch64-android@1.4.2", "", { "os": "android", "cpu": "arm64" }, "sha512-3mZKO2rhsNgbAUtAHC1UKUlF2zTxFraDZT/Elv8wzyH0fJL9h+Iv3TgB9lO63w89PRn3eFe+NRA1bhVgikKNPQ=="], + "@oven/bun-linux-aarch64-android": ["@oven/bun-linux-aarch64-android@1.4.0", "", { "os": "android", "cpu": "arm64" }, "sha512-HpPIxJfDNPBPhiBNMyZoo/dOLijARfsx5j72vNuLtaTvl0Hh7HUculxjsOQ2WSyGoCgqXMEr1Qqjab1im9u1RA=="], - "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-+Sm6y+lSiSFBOtXmnekp5Q6n1tUKlyv71FCPWBc61Cgb14T5eBs8SN/nh4MUCOKzONkI3O+as3MGUgikS4aCBQ=="], + "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.4.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RUjAAkJ/CdNV++zVxyANWshPc73CECYsfhk0fWAkoJjtywxJ2BwXzI6nopBBDMfs0HS+fhRGn6zGwU8ccxLeJg=="], - "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-9/E/UXOTpSo3YsV5g+FhtTd/qTpiWoKuxS12cqtuYA1ssu9fRAoPQnipFgGyck3tWO63iUdxBiygq+kELFawng=="], + "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.4.0", "", { "os": "linux", "cpu": "x64" }, "sha512-Du44zebtPXJujvMLmtIxEQ6ykOhYt7L/Q+YIGVm+Yy+Pj/fpOnq60ggwIpKp/pGAFbYHNiTrA3JTjuZ9MTbZIg=="], - "@oven/bun-linux-x64-android": ["@oven/bun-linux-x64-android@1.4.2", "", { "os": "android", "cpu": "x64" }, "sha512-6HC5tzcC79113n2IHCTJMWv+HsQImv4ZFEK2XpYLxY6HbT8tM4cUM2Zv1bHZBQsS3jv/zYBamDJ1UX7If0d5tw=="], + "@oven/bun-linux-x64-android": ["@oven/bun-linux-x64-android@1.4.0", "", { "os": "android", "cpu": "x64" }, "sha512-u++KyLlfMn36yWz+AgJs+fZtS46UFDNpSSZhrcitkytONtNwq0X6Q9BDVEFXxYl/+Eec0xme1rb6MgW+U35WeA=="], - "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-vVTKUg1bnPhRP/Hp73jIVoFh2vPFNYEqYX0ERKfZBOQEEHitNAeukZzzuUDZS0SoDCIpuWUGSpd/CDMbjdR+Uw=="], + "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.4.0", "", { "os": "linux", "cpu": "x64" }, "sha512-C1Dv+ISL8YKEKM9jAHzNifOcRUoziy6UMxh+yVXjUCP6QnbRhENDHLaIWWkQZJyBLTn0I3xozflorAlHiGzGqA=="], - "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.4.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-8EJ1ST7339WJE3poPW5nBgVW/lWf9HBz4W27ZUNhburKmcBLOByPyE6DP9fHD8FQGm5c+ilUN2hX1mrW0jxq9Q=="], + "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.4.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-FBAYaQpJBP0asgqzL6NFUfjdQqsV+kvTpJ/eWxPKj+RcDgIfPSuE8kvQuPYu5pa8u8JTujYMjmuyvHxVuQsInA=="], - "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-+bN6OuVld/9diT/RLSXSW7JE6CvNE3gL9XsAEjULi1nUsXd6DNO6GuA9jNdNb3r8PdJFnYHr5aypNV1Oj3Rd9g=="], + "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.4.0", "", { "os": "win32", "cpu": "x64" }, "sha512-jRKv1NPLznMSZY5BEWciMF7zv0Tiyo2pQSxAJ3w+YWJ6y3VWNJQQQdLlV5Jx8lbOFDrJdrc9dD3GV17k3BP41A=="], - "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + "@types/bun": ["@types/bun@1.4.0", "", { "dependencies": { "bun-types": "1.4.0" } }, "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ=="], "@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], @@ -136,9 +136,9 @@ "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], - "bun": ["bun@1.4.2", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.4.2", "@oven/bun-darwin-x64": "1.4.2", "@oven/bun-freebsd-aarch64": "1.4.2", "@oven/bun-freebsd-x64": "1.4.2", "@oven/bun-linux-aarch64": "1.4.2", "@oven/bun-linux-aarch64-android": "1.4.2", "@oven/bun-linux-aarch64-musl": "1.4.2", "@oven/bun-linux-x64": "1.4.2", "@oven/bun-linux-x64-android": "1.4.2", "@oven/bun-linux-x64-musl": "1.4.2", "@oven/bun-windows-aarch64": "1.4.2", "@oven/bun-windows-x64": "1.4.2" }, "os": [ "!aix", "!sunos", "!openbsd", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA=="], + "bun": ["bun@1.4.0", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.4.0", "@oven/bun-darwin-x64": "1.4.0", "@oven/bun-freebsd-aarch64": "1.4.0", "@oven/bun-freebsd-x64": "1.4.0", "@oven/bun-linux-aarch64": "1.4.0", "@oven/bun-linux-aarch64-android": "1.4.0", "@oven/bun-linux-aarch64-musl": "1.4.0", "@oven/bun-linux-x64": "1.4.0", "@oven/bun-linux-x64-android": "1.4.0", "@oven/bun-linux-x64-musl": "1.4.0", "@oven/bun-windows-aarch64": "1.4.0", "@oven/bun-windows-x64": "1.4.0" }, "os": [ "!aix", "!sunos", "!openbsd", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-iRiFkc2W7UVpCyZXO9tod45TP9QCyN19fWqbpeN/jaM/K7uzeHYx/OSPsahMJazGKBgPsnxRt+4Jc43d8BcHZw=="], - "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], diff --git a/package.json b/package.json index a856df46861..22449e507ac 100644 --- a/package.json +++ b/package.json @@ -76,11 +76,11 @@ "@bufbuild/protobuf": "^2.14.0", "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/keyring": "1.3.0", - "bun": "1.4.2", + "bun": "1.4.0", "zod": "4.4.3" }, "devDependencies": { - "@types/bun": "1.4.2", + "@types/bun": "1.4.0", "typescript": "7.0.2" }, "overrides": { diff --git a/scripts/ci/bun-crash-signatures.sh b/scripts/ci/bun-crash-signatures.sh new file mode 100755 index 00000000000..476f6539c6f --- /dev/null +++ b/scripts/ci/bun-crash-signatures.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# The single definition of "this was a Bun runtime crash, not a test result". +# +# There were four copies of this list: one in the Linux batch runner and three inline in ci.yml +# (platform-windows, platform-macos, macos-control). ci-workflows.test.ts pinned them in sync +# rather than removing the duplication, because #2152 had already broken one copy by anchoring on +# `panic(thread 2852)` when Bun also emits `panic(main thread)` for the same class, and half the +# crashes stopped matching. Pinning four copies in sync only detects the drift it was written to +# expect; one definition cannot drift at all. +# +# Source it from the repository root: source scripts/ci/bun-crash-signatures.sh + +# shellcheck shell=bash + +# Sourced by nested shells in the same job, so a second source must be a no-op rather than a +# readonly-reassignment error. +if [[ -n "${OCX_BUN_CRASH_SIGNATURES_LOADED:-}" ]]; then + return 0 +fi +OCX_BUN_CRASH_SIGNATURES_LOADED=1 + +# Never anchor on the thread-numbered form. `Internal assertion failure` is the stable fingerprint +# recorded in devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md. +OCX_BUN_CRASH_SIGNATURE_PATTERN='oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' + +# True when the process output carries a Bun panic banner. +bun_log_has_crash_signature() { + grep -Eqi "$OCX_BUN_CRASH_SIGNATURE_PATTERN" "$1" +} + +# True for a status that can only be a fatal signal. +# +# 128+N for SIGILL/SIGABRT/SIGBUS/SIGKILL/SIGSEGV and neighbours. Windows exit 3 is deliberately +# NOT in this list: it is the status a Windows Bun returns alongside a SIGSEGV panic, but unlike +# 132-139 it is an ordinary small exit code any process may return for its own reasons. Trusting +# it bare would reclassify a real test failure as a crash and hide it, which is the exact mistake +# this file exists to stop. Exit 3 is still covered, through the signature arm below, which is +# corroborated by the panic banner Bun actually printed -- that is how the Windows shard 5/6 +# crashes of runs 35087572377, 35093667426 and 35098735960 are recognised. +bun_status_is_crash_code() { + case "$1" in + 132|133|134|135|136|137|139) return 0 ;; + esac + return 1 +} + +# The shared predicate: is_bun_runtime_crash +is_bun_runtime_crash() { + local status="$1" + local log_file="$2" + + if bun_status_is_crash_code "$status"; then + return 0 + fi + + # Bun 1.3.14 can surface a Linux epoll registration failure as exit 1, even though the failure + # comes from Bun's internal WriteStream setup rather than a test assertion. Treat only that + # narrow runtime signature as a crash. + if [[ "$status" == "1" ]] \ + && grep -Fq '# Unhandled error between tests' "$log_file" \ + && grep -Fq 'error: EEXIST: file already exists, epoll_ctl' "$log_file" \ + && grep -Fq 'at new WriteStream (internal:fs/streams:' "$log_file"; then + return 0 + fi + + bun_log_has_crash_signature "$log_file" +} + diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index ae34310258f..147c87c9804 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -11,6 +11,10 @@ readonly BATCH_KILL_GRACE_SECONDS="${BUN_TEST_BATCH_KILL_GRACE_SECONDS:-15}" # bundled stable runtime anyway, and report a qualification it never performed. readonly BUN_BIN="${OPENCODEX_BUN_PATH:-bun}" +# One definition of the crash classifier, shared with the Windows and macOS legs in ci.yml. +# shellcheck source=scripts/ci/bun-crash-signatures.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/bun-crash-signatures.sh" + usage() { echo "usage: $0 " >&2 exit 64 @@ -64,31 +68,6 @@ is_general_test_file() { esac } -is_bun_runtime_crash() { - local status="$1" - local log_file="$2" - - case "$status" in - 132|133|134|135|136|137|139) - return 0 - ;; - esac - - # Bun 1.3.14 can surface a Linux epoll registration failure as exit 1, - # even though the failure comes from Bun's internal WriteStream setup rather - # than a test assertion. Treat only that narrow runtime signature as a crash. - if (( status == 1 )) \ - && grep -Fq '# Unhandled error between tests' "$log_file" \ - && grep -Fq 'error: EEXIST: file already exists, epoll_ctl' "$log_file" \ - && grep -Fq 'at new WriteStream (internal:fs/streams:' "$log_file"; then - return 0 - fi - - grep -Eqi \ - 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' \ - "$log_file" -} - LAST_FAILURE_KIND="" run_test_once() { @@ -189,7 +168,14 @@ recover_batch_file_by_file() { return "$status" done - echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed under singleton isolation after the original ${batch_failure_kind}; continuing." + if [[ "$batch_failure_kind" == "runtime" ]]; then + # Not "recovered". One file per process is a configuration in which this class of defect + # cannot occur, so the sweep was always going to pass and always going to report nothing. + # What it does prove is that the files themselves are sound, which is the half worth keeping. + echo "::error::Shard ${SHARD_SPEC} batch ${batch_number} crashed the Bun runtime. Every file in it then passed alone, so the defect is in multi-file process state, not in any test." + else + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed under singleton isolation after the original ${batch_failure_kind}; continuing." + fi return 0 } @@ -218,7 +204,12 @@ fi readonly TOTAL_BATCHES=$(( (${#SELECTED_FILES[@]} + BATCH_SIZE - 1) / BATCH_SIZE )) echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} primary Bun processes (batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." -echo "Runtime crashes and timeouts fall back to one-file-per-process isolation; assertion/test failures do not retry." +echo "Timeouts fall back to one-file-per-process isolation and may recover; assertion/test failures do not retry." +echo "A Bun runtime crash is swept one-file-per-process for attribution and then FAILS this shard: it is a defect in the interpreter, and a green report would be a lie." + +# Every batch that crashed the runtime, so one run attributes all of them instead of only the +# first. Linux was producing twelve to fourteen of these per run while reporting success. +CRASHED_BATCHES=() for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do start=$(( batch_index * BATCH_SIZE )) @@ -237,8 +228,22 @@ for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do failure_kind="$LAST_FAILURE_KIND" if recover_batch_file_by_file "$batch_number" "$failure_kind" "${batch[@]}"; then - continue + recovery_status=0 else - exit $? + recovery_status=$? + fi + + if [[ "$failure_kind" == "runtime" ]]; then + CRASHED_BATCHES+=("$batch_number") + fi + + # A sweep that found a real failing file still reports that file, and immediately. + if (( recovery_status != 0 )); then + exit "$recovery_status" fi done + +if (( ${#CRASHED_BATCHES[@]} > 0 )); then + echo "::error::Shard ${SHARD_SPEC} crashed the Bun runtime in batch(es): ${CRASHED_BATCHES[*]}. Each batch was re-run one file per process and every file passed, so no test is at fault -- the interpreter is. Failing rather than reporting green." + exit 1 +fi diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2106a9614be..96e44b509cb 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -319,6 +319,7 @@ "chatgpt-oauth.test.ts": "oauth", "chatgpt-token-expiry.test.ts": "oauth", "chutes-provider.test.ts": "providers", + "ci-bun-crash-classifier.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", diff --git a/src/server/index.ts b/src/server/index.ts index 4892f4aaf35..6cd079fd717 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -268,11 +268,11 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server { + const classifier = read("scripts", "ci", "bun-crash-signatures.sh"); + const batchScript = read("scripts", "ci", "run-bun-test-batches.sh"); + const workflow = read(".github", "workflows", "ci.yml"); + + const lanes = { + windows: runBlockContaining(workflow, "bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/6"), + "macos-shard": runBlockContaining(workflow, "run_macos_suite tests"), + "macos-control": runBlockContaining(workflow, "bun test --isolate --timeout 60000 tests 2>&1"), + }; + + test("the signatures exist in the classifier", () => { + for (const signature of CRASH_SIGNATURES) { + expect(`classifier:${signature}:${classifier.includes(signature)}`).toBe(`classifier:${signature}:true`); + } + }); + + test("no lane and no script carries an inline copy of them", () => { + const others: Array = [ + ["batch-script", batchScript], + ...Object.entries(lanes), + ]; + for (const [name, text] of others) { + for (const signature of CRASH_SIGNATURES) { + expect(`${name}:inline:${signature}:${text.includes(signature)}`) + .toBe(`${name}:inline:${signature}:false`); + } + } + }); + + test("every lane sources the classifier and calls the shared predicate", () => { + for (const [name, text] of Object.entries(lanes)) { + expect(`${name}:sources:${text.includes(SOURCE_LINE)}`).toBe(`${name}:sources:true`); + expect(`${name}:calls:${text.includes("is_bun_runtime_crash \"$suite_status\" \"$suite_log\"")}`) + .toBe(`${name}:calls:true`); + } + expect(batchScript).toContain("bun-crash-signatures.sh"); + expect(batchScript).toContain('is_bun_runtime_crash "$status" "$log_file"'); + }); + + test("the thread-numbered panic form is the anchor nowhere", () => { + // `panic(thread 2852)` and `panic(main thread)` are the same class (#2152). + for (const [name, text] of [["classifier", classifier], ["batch-script", batchScript], ...Object.entries(lanes)] as Array) { + expect(`${name}:${text.includes("panic\\(thread")}`).toBe(`${name}:false`); + } + }); + + test("fatal signal codes classify on the status alone, and exit 3 never does", () => { + // 128+N is unambiguous. 3 is an ordinary small exit code any process may return, so it is + // recognised only when Bun also printed a panic banner -- which is how the Windows shard 5/6 + // crashes of runs 35087572377, 35093667426 and 35098735960 are caught. Trusting 3 bare would + // reclassify a real failure as a crash and hide it, which is the mistake this file prevents. + expect(classifier).toContain("132|133|134|135|136|137|139) return 0 ;;"); + expect(classifier).not.toMatch(/^\s*3\|/m); + expect(classifier).not.toContain("|3)"); + }); +}); + +describe("a Bun runtime crash fails the Linux shard", () => { + const batchScript = read("scripts", "ci", "run-bun-test-batches.sh"); + + test("crashed batches are collected and the shard exits non-zero", () => { + expect(batchScript).toContain("CRASHED_BATCHES=()"); + expect(batchScript).toContain('CRASHED_BATCHES+=("$batch_number")'); + expect(batchScript).toContain("if (( ${#CRASHED_BATCHES[@]} > 0 )); then"); + // The failure is an error annotation and a non-zero exit, not a warning and a green shard. + const tail = batchScript.slice(batchScript.indexOf("if (( ${#CRASHED_BATCHES[@]} > 0 )); then")); + expect(tail).toContain("::error::"); + expect(tail).toContain("exit 1"); + }); + + test("the singleton sweep reports a crash as an error rather than a recovery", () => { + expect(batchScript).toContain('if [[ "$batch_failure_kind" == "runtime" ]]; then'); + // The old wording promised recovery. A crash may not be announced that way again. + const sweepEnd = batchScript.slice(batchScript.indexOf("recover_batch_file_by_file")); + expect(sweepEnd).not.toContain("passed under singleton isolation after the original runtime"); + }); + + test("a real failing file found by the sweep still reports that file immediately", () => { + // Failing on the crash must not swallow an assertion the sweep genuinely attributed. + expect(batchScript).toContain("Singleton isolation identified ${file} as a failing test file."); + expect(batchScript).toContain("if (( recovery_status != 0 )); then"); + }); + + test("a timeout may still recover, because a timeout is a load condition", () => { + expect(batchScript).toContain('if [[ "$LAST_FAILURE_KIND" != "runtime" && "$LAST_FAILURE_KIND" != "timeout" ]]; then'); + expect(batchScript).toContain("passed under singleton isolation after the original ${batch_failure_kind}; continuing."); + }); +}); + diff --git a/tests/ci-workflows/ci-workflows.test.ts b/tests/ci-workflows/ci-workflows.test.ts index b098220521e..45763262634 100644 --- a/tests/ci-workflows/ci-workflows.test.ts +++ b/tests/ci-workflows/ci-workflows.test.ts @@ -271,8 +271,8 @@ describe("GitHub Actions hardening", () => { // must disable errexit before the crash-prone command or exit 133 aborts // the step before PIPESTATUS can be inspected and the retry can run. expect(hasExactShellCommand(macosTestRun, "set +e")).toBe(true); - expect(macosTestRun).toContain("Segmentation fault at address"); - expect(macosTestRun).toContain("oh no: Bun has crashed"); + // The crash signatures themselves moved to scripts/ci/bun-crash-signatures.sh; that one + // definition and every lane that sources it are pinned by ci-bun-crash-classifier.test.ts. expect(macosTestRun).toContain("assertion failures are not retried"); expect(macosTestRun).toContain("failing after one retry"); // `for attempt in 1 2` — one retry, never an unbounded loop. @@ -355,35 +355,10 @@ describe("GitHub Actions hardening", () => { expect(winSteps.some(step => step.if === "runner.environment == 'self-hosted'" && step.run?.includes("git clean -xffd"))).toBe(true); - // The three crash-signature lists must stay identical, and they must not key on - // `panic(thread`. - // - // Bun emits BOTH `panic(thread 2852)` and `panic(main thread)` for the same class of - // failure, so a grep anchored on the numbered form silently misses half of them and the - // shard fails on a crash it was supposed to retry. This repository already learned that - // once — `devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md` names - // `Internal assertion failure` as the stable fingerprint — and #2152 reintroduced it. - // Three copies of one list is the real hazard, so pin the sync rather than the text. - const crashSignatures = [ - "oh no: Bun has crashed", - "Internal assertion failure", - "Segmentation fault at address", - "Illegal instruction", - "Bus error", - ]; + // The crash-signature list lives in exactly one file now, and every lane sources it. + // ci-bun-crash-classifier.test.ts owns that contract, including the rule that no lane may + // reintroduce an inline copy and that a runtime crash fails the shard instead of being swept. const windowsTestRun = windowsTestSteps[0]?.run ?? ""; - const batchScript = await readText("scripts/ci/run-bun-test-batches.sh"); - for (const signature of crashSignatures) { - expect(`macos:${signature}:${macosTestRun.includes(signature)}`).toBe(`macos:${signature}:true`); - expect(`macos-control:${signature}:${macosControlTestRun.includes(signature)}`).toBe(`macos-control:${signature}:true`); - expect(`windows:${signature}:${windowsTestRun.includes(signature)}`).toBe(`windows:${signature}:true`); - expect(`script:${signature}:${batchScript.includes(signature)}`).toBe(`script:${signature}:true`); - } - // The thread-numbered form must not be the anchor anywhere. - expect(macosTestRun).not.toContain("panic\\(thread"); - expect(macosControlTestRun).not.toContain("panic\\(thread"); - expect(windowsTestRun).not.toContain("panic\\(thread"); - expect(batchScript).not.toContain("panic\\(thread"); // Windows carries the same bounded retry as macOS: one attempt, crash-only. expect(hasExactShellCommand(windowsTestRun, "set +e")).toBe(true); diff --git a/tests/ci-workflows/install-scripts.test.ts b/tests/ci-workflows/install-scripts.test.ts index fc419501155..2d1436d4726 100644 --- a/tests/ci-workflows/install-scripts.test.ts +++ b/tests/ci-workflows/install-scripts.test.ts @@ -65,10 +65,29 @@ describe("install scripts", () => { expect(pkg.main).toBe("./bin/package-main.mjs"); expect(pkg.exports?.["."]?.bun).toBe("./src/index.ts"); expect(pkg.exports?.["."]?.default).toBe("./bin/package-main.mjs"); - expect(pkg.dependencies?.bun).toBe("1.4.2"); + // Bun stays at 1.4.0 until 1.4.2's test-runner crash is fixed upstream. 1.4.2 segfaults + // while RE-LOADING the bunfig preload that `--isolate` re-enters once per test file: + // `load_preloads -> JSModuleLoader::loadModule -> JSPromise::status` dereferences a dead + // promise and the process dies with "Segmentation fault at address 0x10". It is a crash in + // the interpreter, not a test result, and no test content avoids it. + // + // The A/B is two Linux CI runs 40 minutes apart, either side of #4064 (02bc10e8af), which + // is the commit that moved this line to 1.4.2: + // run 34274464811, 6bd3274ab2, Bun 1.4.0 -- 0 panics across all four shards + // run 34278407438, 02bc10e8af, Bun 1.4.2 -- 12 panics across all four shards + // Every sampled 1.4.0 tree is clean and every sampled 1.4.2 tree crashes 12-14 times per + // run. Linux hid it because `scripts/ci/run-bun-test-batches.sh` re-runs a crashed batch one + // file per process, and one preload load cannot reach the second that faults, so the sweep + // always "passes". Windows had no such sweep: shard 5/6 died on both attempts of runs + // 35087572377, 35093667426 and 35098735960, always at the 67th file, and the file sitting at + // that position changed between them. + // + // Moving this back to 1.4.2, or on to a later release, needs a green `lane=all` dispatch as + // the evidence -- an ordinary PR run cannot show it, because the Linux sweep masks it. + expect(pkg.dependencies?.bun).toBe("1.4.0"); expect(pkg.dependencies?.zod).toBe("4.4.3"); expect(pkg.devDependencies?.typescript).toBe("7.0.2"); - expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.2"); + expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.0"); expect(pkg.scripts?.dev).toBe("bun run src/cli/index.ts start"); expect(pkg.scripts?.["dev:proxy"]).toBe("bun run src/cli/index.ts start"); expect(pkg.scripts?.["dev:gui"]).toBe("cd gui && bun run dev"); diff --git a/tests/ci-workflows/macos-serial-lanes.test.ts b/tests/ci-workflows/macos-serial-lanes.test.ts index c9677cddf4d..b290617f8b4 100644 --- a/tests/ci-workflows/macos-serial-lanes.test.ts +++ b/tests/ci-workflows/macos-serial-lanes.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { spawn, type ChildProcessByStdio } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import type { Readable } from "node:stream"; @@ -19,6 +19,9 @@ const SERIAL_FILES = [ const GENERAL_FILES = ["general/ordinary.test.ts", "general/falcon-extra.test.ts"]; const ASSERTION_STATUS = 23; const CRASH_STATUS = 139; +// A status the classifier cannot recognise on its own, so a crash carrying it is only detected +// through the panic banner. 139 is a fatal signal and matches on the code alone. +const SIGNATURE_ONLY_CRASH_STATUS = 3; const CRASH_SIGNATURES = [ "oh no: Bun has crashed", "Internal assertion failure", @@ -98,6 +101,13 @@ process.exit(0); function createFixture(directory: string, options: FixtureOptions): void { mkdirSync(join(directory, "bin")); mkdirSync(join(directory, "tmp")); + // The lane sources its crash classifier from the working directory, so the sandbox gets the + // REAL file rather than a stand-in. That is deliberate: the harness executes the actual run + // block, so a copy here would let the block and the classifier drift apart unnoticed, which is + // the exact failure mode that collapsing four inline signature lists into one file removed. + mkdirSync(join(directory, "scripts", "ci"), { recursive: true }); + copyFileSync(repoPath("scripts", "ci", "bun-crash-signatures.sh"), + join(directory, "scripts", "ci", "bun-crash-signatures.sh")); for (const file of [...SERIAL_FILES, ...GENERAL_FILES]) { if (file === options.missing) continue; mkdirSync(dirname(join(directory, "tests", file)), { recursive: true }); @@ -304,7 +314,14 @@ describe.skipIf(process.platform === "win32")("macOS serial lane shell ownership for (const [caseIndex, signature] of CRASH_SIGNATURES.entries()) { test(`${target}: retries one runtime crash (case ${caseIndex + 1}), then finishes`, async () => { - const run = await runShard(1, { target, outcomes: ["crash"], crashSignature: signature }); + // Exit 3, not 139, so the SIGNATURE arm of the shared classifier is what is under test. + // With 139 the status arm matches first and this case would pass even if the signature + // list were empty -- which is how a lane can carry a broken list and look covered (#2152). + // Exit 3 is also the real Windows shape: Bun prints the panic banner and returns 3, + // and a bare 3 must NOT be treated as a crash, so the banner is doing the work here. + const run = await runShard(1, { + target, outcomes: ["crash"], crashSignature: signature, crashStatus: SIGNATURE_ONLY_CRASH_STATUS, + }); expect(run.status, run.output).toBe(0); const calls = testCalls(run); const attempts = calls.filter(call => targets(call, target)); diff --git a/tests/codex-integration/client-injection-guard.test.ts b/tests/codex-integration/client-injection-guard.test.ts index 829db8a5a00..262ede77b0e 100644 --- a/tests/codex-integration/client-injection-guard.test.ts +++ b/tests/codex-integration/client-injection-guard.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, beforeAll, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync, realpathSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, dirname, join } from "node:path"; @@ -134,7 +134,7 @@ async function within(promise: Promise, milliseconds: number): Promise })]); } finally { if (timer !== undefined) clearTimeout(timer); } } -async function runScenario(mode: string): Promise> { +async function runScenario(mode: string, childDeadlineMs = 30_000): Promise> { const root = mkdtempSync(join(tmpdir(), "ocx-client-guard-")); roots.push(root); const codex = join(root, "codex"); const ocx = join(root, "ocx"); const desktop = join(root, "desktop"); for (const directory of [codex, ocx, desktop]) mkdirSync(directory, { recursive: true }); @@ -171,14 +171,14 @@ async function runScenario(mode: string): Promise> { children.push(child); const stdout = new Response(child.stdout).text(); const stderr = new Response(child.stderr).text(); - const code = await within(child.exited, 30_000); + const code = await within(child.exited, childDeadlineMs); const output = await within(Promise.all([stdout, stderr]), 5_000); const result = JSON.parse(output[0].trim().split("\n").at(-1) ?? "{}"); if (code !== 0) throw new Error("injection fixture failed: " + String(result.fatal ?? output[1])); return result; } -afterEach(async () => { +async function cleanupFixtures(): Promise { const errors: unknown[] = []; for (const child of children.splice(0)) { try { @@ -193,7 +193,28 @@ afterEach(async () => { try { removeTreeWithRetry(root); } catch (error) { errors.push(error); } } if (errors.length) throw new AggregateError(errors, "injection fixture cleanup failed"); -}, 30_000); +} + +beforeAll(async () => { + try { + // Windows run 35098735960 spent 30.446s in the first child, then 17.038s and + // 1.3-2.0s in its siblings. Run 35093667426 likewise paid 3.814s first versus + // 1.6-2.7s later. Exercise the real write/SQLite/ACL path once before a scenario's + // 30s execution budget so Bun and Windows first-touch work is not timed + // as guard behavior. The 45s warm-up ceiling is the file's existing test budget. + await runScenario("deny", 45_000); + } catch { + // Deliberately swallowed. This hook exists only to pay first-touch cost; it asserts + // nothing. Letting it throw would convert one broken scenario into seven failures whose + // messages all point at a warm-up rather than at the guard, and the real "deny" test below + // reproduces any genuine breakage with its own assertions. A warm-up that merely ran out of + // its own budget on a loaded runner must not fail a file it was added to stabilise. + } finally { + await cleanupFixtures(); + } +}, 55_000); + +afterEach(cleanupFixtures, 30_000); for (const mode of ["deny", "queued-native", "legacy", "external", "malformed", "async", "async-reject"]) { test("client commit guard preserves every routing artifact (" + mode + ")", async () => { diff --git a/tests/codex-integration/codex-composed-acceptance.test.ts b/tests/codex-integration/codex-composed-acceptance.test.ts index 44730e54537..ee6aebb8cb2 100644 --- a/tests/codex-integration/codex-composed-acceptance.test.ts +++ b/tests/codex-integration/codex-composed-acceptance.test.ts @@ -39,6 +39,7 @@ import { canonicalizeCodexHome, } from "../../src/codex/codex-write-lock"; import { + resolveCodexCatalogSerializationDatabasePath, resolveCodexCoordinatorDatabasePath, resolveEffectiveUserIdentity, } from "../../src/codex/user-identity"; @@ -71,6 +72,35 @@ type StartedServer = { stderr: Promise; }; +type CapturedChildStream = { + completed: Promise; + snapshot: () => string; + closed: () => boolean; +}; + +/** Drain a child pipe while retaining the bytes already emitted before EOF. */ +function captureChildStream(stream: ReadableStream): CapturedChildStream { + let text = ""; + let closed = false; + const completed = (async () => { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + text += decoder.decode(chunk.value, { stream: true }); + } + text += decoder.decode(); + return text; + } finally { + closed = true; + reader.releaseLock(); + } + })(); + return { completed, snapshot: () => text, closed: () => closed }; +} + /** A byte manifest: paths plus bytes, not mtimes or parsed JSON. */ function manifest(root: string): Record { const entries: Record = {}; @@ -130,6 +160,8 @@ class Fixture { readonly managementToken = "composed-admin-token"; readonly lockPath: string; readonly lockAllowlist: string[]; + readonly catalogLockPath: string; + readonly catalogLockAllowlist: string[]; readonly serviceManagerEnv: Record; readonly serviceManagerPreloadPath: string | undefined; readonly powerShellCacheEnv: Record = {}; @@ -168,9 +200,18 @@ class Fixture { rmSync(this.root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); throw error; } - this.lockPath = resolveCodexCoordinatorDatabasePath(resolveEffectiveUserIdentity(), realpathSync.native(this.codex)); + const identity = resolveEffectiveUserIdentity(); + const canonicalCodexHome = realpathSync.native(this.codex); + this.lockPath = resolveCodexCoordinatorDatabasePath(identity, canonicalCodexHome); this.lockAllowlist = [this.lockPath, `${this.lockPath}-journal`, `${this.lockPath}-wal`, `${this.lockPath}-shm`]; - for (const path of this.lockAllowlist) { + this.catalogLockPath = resolveCodexCatalogSerializationDatabasePath(identity, canonicalCodexHome); + this.catalogLockAllowlist = [ + this.catalogLockPath, + `${this.catalogLockPath}-journal`, + `${this.catalogLockPath}-wal`, + `${this.catalogLockPath}-shm`, + ]; + for (const path of [...this.lockAllowlist, ...this.catalogLockAllowlist]) { if (existsSync(path)) throw new Error(`lock preflight found pre-existing case path: ${path}`); } writeFileSync(join(this.codex, "config.toml"), 'model = "gpt-5"\n'); @@ -260,23 +301,35 @@ class Fixture { async start(): Promise { const child = this.spawnCli(["start"]); + const pidPath = join(this.ocx, "ocx.pid"); const runtimePath = join(this.ocx, "runtime-port.json"); - // Capture the child's streams while we wait. Without this, a start that dies for a - // concrete reason — a throw, a port bind refusal, a missing artifact — surfaces only as - // "timed out waiting for runtime-port record", which is the symptom and never the cause. - // That is exactly how the Windows failures read for two CI rounds. - const stderr = new Response(child.stderr).text(); - const stdout = new Response(child.stdout).text(); + // Run 35093667426 waited the full 45 s Windows watchdog with the child alive, but + // Response(stream).text() reported only "still open": it cannot reveal bytes until EOF. + // Healthy controls in 35054231781 and 35098735960 finished this whole case in ~14 s, so + // preserve the budget and expose the child's actual progress plus its two startup records. + const stderr = captureChildStream(child.stderr); + const stdout = captureChildStream(child.stdout); const diagnose = async (label: string): Promise => { const exited = child.exitCode ?? (await Promise.race([ child.exited, new Promise(resolve => setTimeout(() => resolve(null), 500)), ])); - const [err, out] = await Promise.all([ - Promise.race([stderr, new Promise(resolve => setTimeout(() => resolve(""), 500))]), - Promise.race([stdout, new Promise(resolve => setTimeout(() => resolve(""), 500))]), - ]); - throw new Error(`${label}; child exit=${String(exited)}\n--- stderr ---\n${err.slice(-4000)}\n--- stdout ---\n${out.slice(-2000)}`); + let pidRecord = existsSync(pidPath) ? "present(unreadable)" : "missing"; + try { pidRecord = `present(${readFileSync(pidPath, "utf8").trim()})`; } catch { /* diagnostic only */ } + let runtimeRecord = existsSync(runtimePath) ? "present(unreadable)" : "missing"; + try { + const record = JSON.parse(readFileSync(runtimePath, "utf8")) as Partial; + runtimeRecord = `present(pid=${String(record.pid)}, port=${String(record.port)}, matches-child=${record.pid === child.pid})`; + } catch { /* diagnostic only; never print the record's attestation secret */ } + const streamText = (capture: CapturedChildStream, limit: number) => { + const value = capture.snapshot().slice(-limit); + return value || `<${capture.closed() ? "closed" : "open"}; no output captured>`; + }; + throw new Error( + `${label}; child exit=${String(exited)}; pid-record=${pidRecord}; runtime-record=${runtimeRecord}` + + `\n--- stderr (${stderr.closed() ? "closed" : "open"}) ---\n${streamText(stderr, 4000)}` + + `\n--- stdout (${stdout.closed() ? "closed" : "open"}) ---\n${streamText(stdout, 2000)}`, + ); }; const runtime = await waitFor(() => { if (!existsSync(runtimePath)) return null; @@ -297,9 +350,9 @@ class Fixture { } catch { return null; } - }, "child /healthz"); + }, "child /healthz").catch(() => diagnose("timed out waiting for child /healthz")); expect(health).toMatchObject({ pid: child.pid, port: runtime.port }); - return { process: child, runtime, stdout, stderr }; + return { process: child, runtime, stdout: stdout.completed, stderr: stderr.completed }; } async stop(server: StartedServer): Promise { @@ -368,9 +421,13 @@ class Fixture { } // Re-resolve before the limited four-name removal: never glob or inspect a // shared runtime namespace beyond the exact identities this case created. - const checked = resolveCodexCoordinatorDatabasePath(resolveEffectiveUserIdentity(), realpathSync.native(this.codex)); + const identity = resolveEffectiveUserIdentity(); + const canonicalCodexHome = realpathSync.native(this.codex); + const checked = resolveCodexCoordinatorDatabasePath(identity, canonicalCodexHome); if (checked !== this.lockPath) throw new Error("lock teardown identity changed"); - for (const path of this.lockAllowlist) { + const checkedCatalog = resolveCodexCatalogSerializationDatabasePath(identity, canonicalCodexHome); + if (checkedCatalog !== this.catalogLockPath) throw new Error("catalog lock teardown identity changed"); + for (const path of [...this.lockAllowlist, ...this.catalogLockAllowlist]) { if (existsSync(path)) unlinkSync(path); } removeTreeWithRetry(this.root); @@ -449,6 +506,10 @@ describe("WP13 composed toggle acceptance", () => { const before = manifest(fx.codex); const server = await fx.start(); try { + // OFF must short-circuit before K. On Windows, merely resolving K starts separate + // SID and LocalAppData PowerShell children with 30 s budgets each; run 35093667426 + // exceeded healthy controls by 33.8 s before the runtime-port watchdog fired at 45 s. + expect(existsSync(fx.catalogLockPath)).toBe(false); expect(manifest(fx.codex)).toEqual(before); for (const argv of [["ensure"], ["restore"]]) { const result = await fx.runCli(argv); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f5c49f9959c..c1f826b816f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -151,6 +151,7 @@ "chatgpt-oauth.test.ts": "oauth", "chatgpt-token-expiry.test.ts": "oauth", "chutes-provider.test.ts": "providers", + "ci-bun-crash-classifier.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", diff --git a/tests/server/server-management-auth.test.ts b/tests/server/server-management-auth.test.ts index 2e5f3db0616..0443c3329c3 100644 --- a/tests/server/server-management-auth.test.ts +++ b/tests/server/server-management-auth.test.ts @@ -7,6 +7,7 @@ import { mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { getConfigPath, saveConfig } from "../../src/config"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; import { clearContextSessionOwnersForTests } from "../../src/codex/context-owner"; import { resetContextRelayActivationForTests } from "../../src/codex/context-compat"; import { startServer } from "../../src/server"; @@ -189,7 +190,23 @@ beforeEach(() => { process.env.OPENCODEX_ADMIN_AUTH_TOKEN = "admin-secret"; }); -afterEach(() => { +afterEach(async () => { + // Settle every in-flight config-directory harden before anything here removes a directory. + // + // `hardenConfigDir()` spawns `icacls.exe`, which holds the directory open until it exits, and + // Windows file locking is mandatory: removing that tree while the child lives returns EPERM no + // matter how long the caller waits. Windows shard 1/6 of run 35108652486 proved the waiting is + // not the answer -- it exhausted the full 15s exponential budget and still threw + // `EPERM: operation not permitted, rm .../tmp/ocx-management-auth-fDchUb` out of this hook. + // #4789 filed the same failure at this same line when the budget was 2.5s, and raising it to + // 15s in #4796 bought six times the wait and changed nothing, because the handle was never + // going to close on its own schedule. The process that started the child has to wait for it. + // + // The all-directories variant is the required one. `server.stop` already flushes, but through + // `flushConfigDirHardening()`, which defaults to `getConfigDir()` read at stop time -- and this + // hook moves OPENCODEX_HOME back to the developer's real home a few lines below, so a + // directory-scoped flush here would settle the wrong tree and leave this one held. + await flushConfigDirHardeningForTests(); resetContextRelayActivationForTests(); if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; diff --git a/tests/server/server-search.test.ts b/tests/server/server-search.test.ts index 721ca155593..667979427ca 100644 --- a/tests/server/server-search.test.ts +++ b/tests/server/server-search.test.ts @@ -315,16 +315,21 @@ test("an account-qualified search model uses that exact account and sends the ba test("an exact search 429 never switches to the active Pool account and reports only its public selector", async () => { const captured: CapturedRequest[] = []; const upstream = fakeSearchUpstream(captured, 429, { error: { message: "rate limited" } }); - saveConfig(exactSearchConfig()); + const config = exactSearchConfig(); + saveConfig(config); saveExactSearchCredentials(); - const server = startServer(0); try { - const requestExactSearch = () => fetch(new URL("/v1/alpha/search", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ id: "search-session", model: "side/gpt-test" }), - }); + // Run 35093667426 returned a local 401 before this fixture could return its 429. + // Full server startup is unrelated to this routing contract and widens the interval + // between writing and reading the credential store selected by process-wide + // OPENCODEX_HOME. Call the handler while that fixture home is current. + const logCtx = { model: "", provider: "" }; + const requestExactSearch = () => handleSearch( + alphaSearchRequest({ id: "search-session", model: "side/gpt-test" }), + config, + logCtx, + ); const first = await requestExactSearch(); expect(first.status).toBe(429); @@ -342,11 +347,9 @@ test("an exact search 429 never switches to the active Pool account and reports expect(captured).toHaveLength(1); expect(loadConfig().activeCodexAccountId).toBe("pool-b"); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); - const entry = getRequestLogEntries().findLast(candidate => candidate.model === "side/gpt-test"); - expect(entry?.provider).toBe("openai-side"); - expect(JSON.stringify(entry)).not.toContain("pool-a"); + expect(logCtx.provider).toBe("openai-side"); + expect(JSON.stringify(logCtx)).not.toContain("pool-a"); } finally { - await server.stop(true); await upstream.stop(true); } });