From aebff67372c5adfb5412d5362c69939d3d7cb896 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Sun, 7 Jun 2026 08:05:02 -0500 Subject: [PATCH 1/5] fix(token-metrics): key usage sidecar per-call, not by $$ (concurrency) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit review-one-pr.sh backgrounds `run_agentic &` and `(run_duck) &` at the same time. $$ stays the parent PID inside both subshells, so the `${TOKEN_LOG_FILE}.last-usage.$$` sidecar collided between the two concurrent tier-2 calls — one engine's usage could overwrite or be read by the other before _record_engine_tokens logged it. (BASHPID doesn't work either: it differs between the chain's pipe subshell and the reader.) Fix: each run_* exports _ENGINE_USAGE_OUT, derived from its own per-call mktemp path (unique by construction) and inherited by the engine's pipeline subshell, so writer and reader agree while concurrent calls never share a file. $$ remains a fallback for non-concurrent direct callers. - token-metrics.sh: _engine_usage_sidecar prefers _ENGINE_USAGE_OUT. - engine.sh: run_triage/run_agentic/run_duck/run_writer export the per-call key. - tests: concurrency-isolation test (two jobs sharing $$ stay separate), per-call-key and fallback assertions; correct the misleading "$$ isolates parallel" test. - token_report.bats: make the malformed-JSONL test deterministic (jq 1.7 exits 0 on NUL bytes; use truncated JSON, which jq rejects across versions). Addresses PR #456 review (chatgpt-codex-connector P2: key usage sidecars by BASHPID). Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/engine.sh | 15 +++++++ scripts/lib/token-metrics.sh | 28 +++++++----- tests/dev-lead/unit/test_token_metrics.bats | 50 +++++++++++++++++++-- tests/token_report.bats | 6 ++- 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index 77b302f4b..cc1d7a290 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -497,6 +497,10 @@ run_triage() { local _tok_tmp="" if [ -n "${TOKEN_LOG_FILE:-}" ]; then _tok_tmp="$(mktemp 2>/dev/null || true)" + # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's + # pipeline subshell and _record_engine_tokens agree on it. Unique per call → + # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. + [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi while [ "$attempt" -le "$RETRY_MAX_ATTEMPTS" ]; do rc=0 @@ -568,6 +572,10 @@ run_agentic() { local _tok_tmp="" rc=0 if [ -n "${TOKEN_LOG_FILE:-}" ]; then _tok_tmp="$(mktemp 2>/dev/null || true)" + # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's + # pipeline subshell and _record_engine_tokens agree on it. Unique per call → + # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. + [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$REVIEW_ENGINE" in claude) @@ -689,6 +697,10 @@ run_duck() { local _tok_tmp="" rc=0 if [ -n "${TOKEN_LOG_FILE:-}" ]; then _tok_tmp="$(mktemp 2>/dev/null || true)" + # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's + # pipeline subshell and _record_engine_tokens agree on it. Unique per call → + # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. + [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$DUCK_ENGINE" in claude) @@ -843,6 +855,9 @@ run_writer() { # were never tried. local _tmp rc=0 _tmp="$(mktemp 2>/dev/null || true)" + # Per-call usage-sidecar key (see run_triage); unique mktemp path shared with + # the engine subshell via export. + [ -n "$_tmp" ] && export _ENGINE_USAGE_OUT="${_tmp}.usage" case "$REVIEW_ENGINE" in claude) diff --git a/scripts/lib/token-metrics.sh b/scripts/lib/token-metrics.sh index ba22b9f1e..e849621bb 100644 --- a/scripts/lib/token-metrics.sh +++ b/scripts/lib/token-metrics.sh @@ -131,18 +131,26 @@ emit_token_record() { # ───────────────────────────────────────────────────────────────────────────── # _engine_usage_sidecar -# Prints the path of the per-process usage sidecar (empty when TOKEN_LOG_FILE is -# unset). Engine invocations often run inside a `cmd | tee` pipeline — i.e. a -# SUBSHELL — so the LAST_* globals they set never reach the parent. Writing usage -# to this file (the filesystem is shared) lets _record_engine_tokens read it back. -# $$ (shell PID) is constant across pipeline subshells: both the writing left-hand -# side and the reading parent see the same value. Separate background processes -# (e.g. `run_agentic &` forking a new process vs `(run_duck) &` running as a -# subshell) each have a distinct $$, so concurrent tier-2 calls never share a -# sidecar file. +# Prints the path of the per-CALL usage sidecar (empty when TOKEN_LOG_FILE is +# unset). Engine invocations run inside a `cmd | tee` pipeline — i.e. a SUBSHELL — +# so the LAST_* globals they set never reach the parent; writing usage to this file +# (the filesystem is shared) lets _record_engine_tokens read it back. +# +# The key must be unique per concurrent engine call AND identical between the +# writer (the pipeline subshell) and the reader (the calling shell). $$ fails the +# first requirement: review-one-pr.sh backgrounds `run_agentic &` and `(run_duck) &` +# at the same time, and $$ stays the parent PID inside both, so they would collide +# on one sidecar. BASHPID fails the second: it differs between the pipe subshell +# and the caller. So each run_* exports _ENGINE_USAGE_OUT, derived from its own +# per-call mktemp file (unique by construction) and inherited by the pipeline +# subshell. The $$ form is only a fallback for any non-concurrent direct caller. _engine_usage_sidecar() { [ -n "${TOKEN_LOG_FILE:-}" ] || return 0 - printf '%s' "${TOKEN_LOG_FILE}.last-usage.$$" + if [ -n "${_ENGINE_USAGE_OUT:-}" ]; then + printf '%s' "$_ENGINE_USAGE_OUT" + else + printf '%s' "${TOKEN_LOG_FILE}.last-usage.$$" + fi } # reset_engine_usage diff --git a/tests/dev-lead/unit/test_token_metrics.bats b/tests/dev-lead/unit/test_token_metrics.bats index 485f2ff07..be5084469 100644 --- a/tests/dev-lead/unit/test_token_metrics.bats +++ b/tests/dev-lead/unit/test_token_metrics.bats @@ -246,10 +246,11 @@ teardown() { [ "$LAST_OUTPUT_TOKENS" = "85" ] # 60 candidates + 25 thoughts } -@test "_engine_usage_sidecar: path includes shell PID to isolate parallel invocations" { - # $$ is constant across pipeline subshells (left-hand side inherits parent PID) - # so both the writing subshell and the reading parent agree on the same path, - # while separate background processes each get their own distinct $$. +@test "_engine_usage_sidecar: fallback path is \$\$-keyed when no per-call key set" { + # Without a per-call _ENGINE_USAGE_OUT the sidecar falls back to a $$-keyed path. + # NOTE: $$ does NOT isolate concurrent background jobs (they share the parent PID) + # — per-call isolation comes from _ENGINE_USAGE_OUT (see the concurrency test). + unset _ENGINE_USAGE_OUT local sidecar sidecar="$(_engine_usage_sidecar)" [ -n "$sidecar" ] @@ -320,3 +321,44 @@ line two' run emit_token_record "pr-review" "triage" "claude" "haiku" 100 0 50 "" [ "$status" -eq 0 ] } + +# ── usage sidecar keying (concurrency safety) ───────────────────────────────── + +@test "_engine_usage_sidecar: uses the exported per-call key when set" { + export _ENGINE_USAGE_OUT="$TOKEN_LOG_FILE.callX.usage" + run _engine_usage_sidecar + unset _ENGINE_USAGE_OUT + [ "$output" = "$TOKEN_LOG_FILE.callX.usage" ] +} + +@test "_engine_usage_sidecar: empty when TOKEN_LOG_FILE is unset" { + unset TOKEN_LOG_FILE _ENGINE_USAGE_OUT + run _engine_usage_sidecar + [ -z "$output" ] +} + +@test "usage sidecar: concurrent calls sharing \$\$ stay isolated (no cross-read)" { + # Reproduces the review-one-pr.sh pattern: two engine calls backgrounded at once. + # Background subshells share the parent's $$, so a $$-keyed sidecar would collide; + # the per-call _ENGINE_USAGE_OUT (unique mktemp path) must keep them isolated. + local dir; dir="$(mktemp -d)" + printf '%s' '{"result":"a","usage":{"input_tokens":111,"cache_read_input_tokens":1,"cache_creation_input_tokens":2,"output_tokens":11}}' > "$dir/a.json" + printf '%s' '{"result":"b","usage":{"input_tokens":222,"cache_read_input_tokens":3,"cache_creation_input_tokens":4,"output_tokens":22}}' > "$dir/b.json" + ( + export _ENGINE_USAGE_OUT="$dir/A.usage" + parse_engine_usage claude "$dir/a.json" + sleep 0.3 # widen the race window for any colliding writer + cut -f1 < "$(_engine_usage_sidecar)" > "$dir/a.out" + ) & + ( + export _ENGINE_USAGE_OUT="$dir/B.usage" + parse_engine_usage claude "$dir/b.json" + sleep 0.3 + cut -f1 < "$(_engine_usage_sidecar)" > "$dir/b.out" + ) & + wait + local a b; a="$(cat "$dir/a.out")"; b="$(cat "$dir/b.out")" + rm -rf "$dir" + [ "$a" = "111" ] + [ "$b" = "222" ] +} diff --git a/tests/token_report.bats b/tests/token_report.bats index 117447fb1..0421c3897 100644 --- a/tests/token_report.bats +++ b/tests/token_report.bats @@ -162,9 +162,11 @@ setup() { run bash -c " set -euo pipefail source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh' - # Simulate the inner conversion loop for a binary-corrupted source file. + # Simulate the inner conversion loop for a corrupted source file. Use truncated + # JSON, which jq rejects deterministically across versions (jq 1.7 exits 0 on + # NUL bytes, so binary input made this test flaky). f='$tmpdir/bad.jsonl' - printf '\x00\x01\x02\x03' > \"\$f\" + printf '{\"a\":' > \"\$f\" dest='$jsonl_dir/999-bad.jsonl' if jq -c --arg repo 'r' 'select(type==\"object\") | . + {repo: \$repo}' \ \"\$f\" > \"\$dest\" 2>/dev/null; then From 7a60dd79800a9c608406c6b16ffa00d4c98357a5 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 7 Jun 2026 13:09:29 +0000 Subject: [PATCH 2/5] fix(reviews): address review comments [skip ci-relay] --- scripts/engine.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index cc1d7a290..71f139110 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -500,7 +500,7 @@ run_triage() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi while [ "$attempt" -le "$RETRY_MAX_ATTEMPTS" ]; do rc=0 @@ -575,7 +575,7 @@ run_agentic() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$REVIEW_ENGINE" in claude) @@ -700,7 +700,7 @@ run_duck() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && export _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$DUCK_ENGINE" in claude) @@ -857,7 +857,7 @@ run_writer() { _tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key (see run_triage); unique mktemp path shared with # the engine subshell via export. - [ -n "$_tmp" ] && export _ENGINE_USAGE_OUT="${_tmp}.usage" + [ -n "$_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tmp}.usage" case "$REVIEW_ENGINE" in claude) From 3c65b64c89e63551d37130399fcec660ddaf17e5 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 7 Jun 2026 13:15:13 +0000 Subject: [PATCH 3/5] chore: apply manual instructions [skip ci-relay] --- scripts/engine.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/engine.sh b/scripts/engine.sh index 71f139110..7318cfff4 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -496,6 +496,7 @@ run_triage() { # Capture output for token estimation when logging is enabled (tee is a no-op otherwise). local _tok_tmp="" if [ -n "${TOKEN_LOG_FILE:-}" ]; then + unset _ENGINE_USAGE_OUT _tok_tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → @@ -571,6 +572,7 @@ run_agentic() { local tier="${3:-deep}" local _tok_tmp="" rc=0 if [ -n "${TOKEN_LOG_FILE:-}" ]; then + unset _ENGINE_USAGE_OUT _tok_tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → @@ -696,6 +698,7 @@ run_duck() { local model="$2" local _tok_tmp="" rc=0 if [ -n "${TOKEN_LOG_FILE:-}" ]; then + unset _ENGINE_USAGE_OUT _tok_tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → @@ -854,6 +857,7 @@ run_writer() { # to stdout, not stderr), so is_rate_limited never fired and fallback engines # were never tried. local _tmp rc=0 + unset _ENGINE_USAGE_OUT _tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key (see run_triage); unique mktemp path shared with # the engine subshell via export. From 6ccc187bca000415b52766d9566ae98bf510c6bc Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Sun, 7 Jun 2026 08:20:23 -0500 Subject: [PATCH 4/5] test(token-metrics): cover mktemp-failure fallback for per-call usage key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds regression coverage for the fix that clears _ENGINE_USAGE_OUT before mktemp (PR #460 review, copilot-pull-request-reviewer ×3): - unit: a set-but-empty per-call key falls back to the $$-keyed sidecar (the mktemp-failure state), so it never reuses a prior/inherited key. - engine: a stale exported _ENGINE_USAGE_OUT + a forced mktemp failure does NOT reuse the stale sidecar — run_triage logs an estimate, not the planted 999/9/9/9. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/dev-lead/unit/test_engine_writer.bats | 31 +++++++++++++++++++++ tests/dev-lead/unit/test_token_metrics.bats | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/tests/dev-lead/unit/test_engine_writer.bats b/tests/dev-lead/unit/test_engine_writer.bats index 76a5e33d5..e3e281bda 100644 --- a/tests/dev-lead/unit/test_engine_writer.bats +++ b/tests/dev-lead/unit/test_engine_writer.bats @@ -486,6 +486,37 @@ STUB [ "$(jq -r '.output_tokens' < "$log")" = "60" ] } +@test "usage: failed mktemp + stale _ENGINE_USAGE_OUT does NOT reuse the stale sidecar" { + # Regression for the per-call-key fix: run_* clears _ENGINE_USAGE_OUT before + # mktemp, so a failed mktemp falls back to estimation instead of reading a prior + # call's sidecar. Without it, the planted 999/9/9/9 below would be logged. + _source_engine "claude" + export STUB_ENGINE_EXIT=0 + export STUB_ENGINE_RESPONSE="verdict" + local log; log=$(mktemp) + export TOKEN_LOG_FILE="$log" + export TEST_OWNED_TOKEN_LOG="$log" + + # A leftover per-call key + sidecar content from a "previous" call. + local stale; stale="$(mktemp)" + printf '999\t9\t9\t9\n' > "$stale" + export _ENGINE_USAGE_OUT="$stale" + + # Force mktemp to fail inside the engine via a PATH shim. + local shim; shim="$(mktemp -d)" + printf '#!/usr/bin/env bash\nexit 1\n' > "$shim/mktemp" + chmod +x "$shim/mktemp" + + PATH="$shim:$PATH" run run_triage "$TEST_PROMPT" + + rm -rf "$shim"; rm -f "$stale"; unset _ENGINE_USAGE_OUT + [ "$status" -eq 0 ] + # The stale cache figures must NOT appear in the record. + [ "$(jq -r '.cache_read_tokens' < "$log")" != "9" ] + [ "$(jq -r '.cache_creation_tokens' < "$log")" != "9" ] + [ "$(jq -r '.input_tokens' < "$log")" != "999" ] +} + @test "token: run_writer_with_fallback logs the fallback engine (not rate-limited primary)" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false diff --git a/tests/dev-lead/unit/test_token_metrics.bats b/tests/dev-lead/unit/test_token_metrics.bats index be5084469..86123033d 100644 --- a/tests/dev-lead/unit/test_token_metrics.bats +++ b/tests/dev-lead/unit/test_token_metrics.bats @@ -331,6 +331,15 @@ line two' [ "$output" = "$TOKEN_LOG_FILE.callX.usage" ] } +@test "_engine_usage_sidecar: set-but-empty key falls back to \$\$ (mktemp-failure state)" { + # run_* clears _ENGINE_USAGE_OUT before mktemp; if mktemp fails the key is empty/ + # unset and MUST fall back rather than reuse a prior/inherited key. + export _ENGINE_USAGE_OUT="" + run _engine_usage_sidecar + unset _ENGINE_USAGE_OUT + [ "$output" = "$TOKEN_LOG_FILE.last-usage.$$" ] +} + @test "_engine_usage_sidecar: empty when TOKEN_LOG_FILE is unset" { unset TOKEN_LOG_FILE _ENGINE_USAGE_OUT run _engine_usage_sidecar From a77aa42f0f80f14c4593e576e497b6a795d5ba31 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 7 Jun 2026 13:25:30 +0000 Subject: [PATCH 5/5] fix(reviews): address review comments [skip ci-relay] --- scripts/engine.sh | 8 ++++---- scripts/lib/token-metrics.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index 7318cfff4..56fd97db6 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -501,7 +501,7 @@ run_triage() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [[ -n "$_tok_tmp" ]] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi while [ "$attempt" -le "$RETRY_MAX_ATTEMPTS" ]; do rc=0 @@ -577,7 +577,7 @@ run_agentic() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [[ -n "$_tok_tmp" ]] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$REVIEW_ENGINE" in claude) @@ -703,7 +703,7 @@ run_duck() { # Per-call usage-sidecar key: a unique mktemp path, exported so the engine's # pipeline subshell and _record_engine_tokens agree on it. Unique per call → # concurrent run_agentic/run_duck (review-one-pr.sh) never collide. - [ -n "$_tok_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" + [[ -n "$_tok_tmp" ]] && local -x _ENGINE_USAGE_OUT="${_tok_tmp}.usage" fi case "$DUCK_ENGINE" in claude) @@ -861,7 +861,7 @@ run_writer() { _tmp="$(mktemp 2>/dev/null || true)" # Per-call usage-sidecar key (see run_triage); unique mktemp path shared with # the engine subshell via export. - [ -n "$_tmp" ] && local -x _ENGINE_USAGE_OUT="${_tmp}.usage" + [[ -n "$_tmp" ]] && local -x _ENGINE_USAGE_OUT="${_tmp}.usage" case "$REVIEW_ENGINE" in claude) diff --git a/scripts/lib/token-metrics.sh b/scripts/lib/token-metrics.sh index e849621bb..e29facda6 100644 --- a/scripts/lib/token-metrics.sh +++ b/scripts/lib/token-metrics.sh @@ -146,7 +146,7 @@ emit_token_record() { # subshell. The $$ form is only a fallback for any non-concurrent direct caller. _engine_usage_sidecar() { [ -n "${TOKEN_LOG_FILE:-}" ] || return 0 - if [ -n "${_ENGINE_USAGE_OUT:-}" ]; then + if [[ -n "${_ENGINE_USAGE_OUT:-}" ]]; then printf '%s' "$_ENGINE_USAGE_OUT" else printf '%s' "${TOKEN_LOG_FILE}.last-usage.$$"