Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions scripts/engine.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,37 @@ echo " engine: $REVIEW_ENGINE ($ENGINE_LABEL)"
# API-level (429), subscription/billing caps (plan limit, out of tokens, HTTP 402),
# or service overload acting as a hard block (529).
# review-one-pr.sh exits with code 2 when this fires so the caller can switch engines.
#
# Patterns intentionally excluded to prevent false positives:
# - bare "exhausted" (too broad: matches "retry attempts exhausted", OS errors, etc.)
# Retained as "token.*exhaust" / "out of.*token" for the specific token-depletion case.
# - CLI syntax errors ("Invalid command format", "unknown flag", etc.) — see is_cli_error.
is_rate_limited() {
local text="$1"
echo "$text" | grep -qiE \
"(hit your limit|rate[ -]?limit|resets [0-9]+(am|pm)|usage limit|quota exceeded|too many requests|exceeded.*quota|([^0-9]|^)429([^0-9]|$)|exhausted|out of.*token|token.*exhaust|claude.*usage|usage.*claude|plan.*limit|subscription.*limit|billing.*limit|daily.*limit|monthly.*limit|([^0-9]|^)402([^0-9]|$)|([^0-9]|^)529([^0-9]|$))"
# Build pattern in segments for readability — one category per line.
local _pat
_pat="hit your limit|rate[ -]?limit|resets [0-9]+(am|pm)" # soft cap / throttle
_pat="$_pat|usage limit|quota exceeded|too many requests|exceeded.*quota"
_pat="$_pat|([^0-9]|^)429([^0-9]|$)" # HTTP 429
_pat="$_pat|out of.*token|token.*exhaust" # token depletion
_pat="$_pat|overloaded_error|service.*overload|overload.*error" # service overload
_pat="$_pat|([^0-9]|^)529([^0-9]|$)" # HTTP 529
_pat="$_pat|claude.*usage|usage.*claude" # Claude-specific cap
_pat="$_pat|plan.*limit|subscription.*limit|billing.*limit|daily.*limit|monthly.*limit"
_pat="$_pat|([^0-9]|^)402([^0-9]|$)" # HTTP 402 (payment)
printf '%s\n' "$text" | grep -qiE "($_pat)"
}
Comment thread
don-petry marked this conversation as resolved.
Resolved

# is_cli_error <text>
# Returns 0 (true) if the text looks like a CLI invocation error —
# bad flags, wrong syntax, or a missing command.
# These are NOT rate limits: callers must exit with code 1 (per-PR failure),
# NOT code 2 (rate-limit / engine fallback), so the session can continue
# processing the remaining PR queue rather than aborting entirely.
is_cli_error() {
local text="$1"
printf '%s\n' "$text" | grep -qiE \
"(invalid command format|invalid (flag|argument|option|command)|unknown (flag|command|option|argument)|command not found|no such command|did you mean:|unrecognized (command|flag|argument|option)|bad (flag|argument|option))"
}

# is_transient_failure <exit_code>
Expand Down
56 changes: 40 additions & 16 deletions scripts/review-one-pr.sh
Original file line number Diff line number Diff line change
Expand Up @@ -265,16 +265,24 @@ TRIAGE_RESULT=$(
# subprocess forks (jq, claude) from hitting E2BIG on hundreds-of-KB diffs.
unset PR_DIFF PR_METADATA

# Detect rate limit before JSON validation — check both stdout and stderr,
# then exit 2 so the caller can fall back to a different engine rather than
# burning through the remaining PR queue.
# Detect error category before JSON validation.
# Claude Code writes its usage-cap error to stdout (captured in TRIAGE_RESULT);
# some other providers write to stderr (TRIAGE_LOG) — check both channels.
TRIAGE_STDERR=$(cat "$TRIAGE_LOG" 2>/dev/null || true)
if [ "$TRIAGE_RC" -ne 0 ] && (is_rate_limited "$TRIAGE_RESULT" || is_rate_limited "$TRIAGE_STDERR"); then
echo " [tier1] usage/rate limit detected — exiting with code 2 for engine fallback"
echo " limit message: ${TRIAGE_RESULT:-}${TRIAGE_STDERR:+ (stderr: $TRIAGE_STDERR)}"
exit 2
if [ "$TRIAGE_RC" -ne 0 ]; then
# CLI invocation errors (bad flags, wrong syntax) are per-PR failures — exit 1
# so the session can continue with the next PR rather than aborting entirely.
if is_cli_error "$TRIAGE_RESULT" || is_cli_error "$TRIAGE_STDERR"; then
echo " [error] CLI format error — treating as per-PR failure (not a rate limit)"
echo " error message: ${TRIAGE_RESULT:-}${TRIAGE_STDERR:+ (stderr: $TRIAGE_STDERR)}"
exit 1
fi
# Rate-limit / overload — exit 2 so the caller can fall back to a different engine.
if is_rate_limited "$TRIAGE_RESULT" || is_rate_limited "$TRIAGE_STDERR"; then
echo " [tier1] usage/rate limit detected — exiting with code 2 for engine fallback"
echo " limit message: ${TRIAGE_RESULT:-}${TRIAGE_STDERR:+ (stderr: $TRIAGE_STDERR)}"
exit 2
fi
fi
Comment thread
don-petry marked this conversation as resolved.
Resolved

# Hard-fail on triage process exit. Previously this silently synthesized a
Expand Down Expand Up @@ -331,14 +339,23 @@ if [ "$TRIAGE_ESCALATE" = "false" ]; then
run_agentic prompts/single-review.md "$ENGINE_SINGLE_MODEL" \
> "$VERDICT_JSON.raw" 2>"$SINGLE_LOG" || single_rc=$?

# Check for rate limit before retrying — exit code 2 lets review-batch.sh
# trigger engine fallback rather than burning retries on the same engine.
# Check error category on failure only — guards against false positives from
# reviewed PR content that happens to mention rate-limit keywords.
SINGLE_STDOUT=$(cat "$VERDICT_JSON.raw" 2>/dev/null || true)
SINGLE_STDERR=$(cat "$SINGLE_LOG" 2>/dev/null || true)
if is_rate_limited "$SINGLE_STDOUT" || is_rate_limited "$SINGLE_STDERR"; then
echo " [approve] rate limit detected — exiting with code 2 for engine fallback"
[ -n "$SINGLE_STDERR" ] && echo " limit stderr: $SINGLE_STDERR"
exit 2
if [ "$single_rc" -ne 0 ]; then
# CLI invocation errors are per-PR failures (exit 1), not engine fallbacks (exit 2).
if is_cli_error "$SINGLE_STDOUT" || is_cli_error "$SINGLE_STDERR"; then
echo " [error] CLI format error — treating as per-PR failure (not a rate limit)"
[ -n "$SINGLE_STDERR" ] && echo " error stderr: $SINGLE_STDERR"
exit 1
fi
# Rate-limit / overload — exit code 2 lets review-batch.sh trigger engine fallback.
if is_rate_limited "$SINGLE_STDOUT" || is_rate_limited "$SINGLE_STDERR"; then
echo " [approve] rate limit detected — exiting with code 2 for engine fallback"
[ -n "$SINGLE_STDERR" ] && echo " limit stderr: $SINGLE_STDERR"
exit 2
fi
fi

if [ "$single_rc" -eq 0 ] && extract_verdict_json "$VERDICT_JSON.raw" "$VERDICT_JSON"; then
Expand Down Expand Up @@ -402,12 +419,19 @@ wait $DEEP_PID || true
# Validate primary deep review (required)
OUTPUT_FILE="/tmp/cascade/deep.json"
if [ ! -s "$OUTPUT_FILE" ] || ! jq empty "$OUTPUT_FILE" 2>/dev/null; then
# Check the model's stdout for a rate-limit message. We intentionally do NOT
# check deep.log (stderr/process noise) to avoid false positives from PR diff
# content that happens to mention rate-limiting in code or comments.
# Check the model's stdout for a rate-limit or CLI-error message. We
# intentionally do NOT check deep.log (stderr/process noise) to avoid false
# positives from PR diff content that happens to mention these keywords in
# code or comments.
DEEP_STDOUT_CONTENT=$(cat /tmp/cascade/deep-stdout.txt 2>/dev/null || true)
kill $DUCK_PID 2>/dev/null || true
wait $DUCK_PID 2>/dev/null || true
# CLI invocation errors are per-PR failures (exit 1), not engine fallbacks (exit 2).
if is_cli_error "$DEEP_STDOUT_CONTENT"; then
echo " [error] CLI format error — treating as per-PR failure (not a rate limit)"
echo "$DEEP_STDOUT_CONTENT"
exit 1
fi
if is_rate_limited "$DEEP_STDOUT_CONTENT"; then
echo " [tier2] rate limit detected — exiting with code 2 for engine fallback"
echo "$DEEP_STDOUT_CONTENT"
Expand Down
123 changes: 123 additions & 0 deletions tests/test_cli_error_routing.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#!/usr/bin/env bats
# Integration test: CLI format errors must route to per-PR failure (exit 1),
# NOT to session abort / engine fallback (exit 2).
#
# This tests the error-routing logic that review-one-pr.sh applies at the
# triage and single-review tiers (both check stdout + stderr). The deep-review
# tier intentionally inspects stdout only and is covered by the unit tests in
# test_rate_limit.bats via the is_rate_limited / is_cli_error functions.
#
# Run with: bats tests/test_cli_error_routing.bats

setup() {
export REVIEW_ENGINE="claude"
source "$(dirname "$BATS_TEST_FILENAME")/../scripts/engine.sh" >/dev/null 2>&1 || true
}

# Helper: emulates the error-routing block used at all three call sites in
# review-one-pr.sh. Returns the exit code that review-one-pr.sh would use.
#
# Note: the deep-review tier intentionally checks only stdout (not stderr) to
# prevent PR diff content in log files from causing false positives. This
# helper checks both channels, which is accurate for the triage and
# single-review tiers. The deep-review tier's stdout-only behaviour is tested
# separately via the is_rate_limited / is_cli_error unit tests.
route_error() {
local rc="$1" # exit code from the CLI / model invocation
local stdout="$2" # captured stdout
local stderr="$3" # captured stderr

if [ "$rc" -ne 0 ]; then
if is_cli_error "$stdout" || is_cli_error "$stderr"; then
echo "per-pr-failure"
return 1 # maps to exit 1
fi
if is_rate_limited "$stdout" || is_rate_limited "$stderr"; then
echo "engine-fallback"
return 2 # maps to exit 2
fi
fi
echo "pass-through"
return 0
}

# ---------------------------------------------------------------------------
# Scenario: Copilot CLI exits with "Invalid command format" (the triggering
# incident described in issue #148 / run #715).
# ---------------------------------------------------------------------------

@test "integration: Copilot 'Invalid command format' routes to per-PR failure (exit 1)" {
local stderr
stderr="$(printf 'error: Invalid command format.\nDid you mean: copilot -i "suggest -p ..."')"

run route_error 1 "" "$stderr"

[ "$status" -eq 1 ]
[ "$output" = "per-pr-failure" ]
}

# ---------------------------------------------------------------------------
# Scenario: rate-limit hit — must still route to engine fallback (exit 2).
# ---------------------------------------------------------------------------

@test "integration: 'rate limit exceeded' routes to engine fallback (exit 2)" {
run route_error 1 "rate limit exceeded" ""

[ "$status" -eq 2 ]
[ "$output" = "engine-fallback" ]
}

@test "integration: 'overloaded_error' routes to engine fallback (exit 2)" {
run route_error 1 "overloaded_error" ""

[ "$status" -eq 2 ]
[ "$output" = "engine-fallback" ]
}

@test "integration: '529' routes to engine fallback (exit 2)" {
run route_error 1 "HTTP status 529" ""

[ "$status" -eq 2 ]
[ "$output" = "engine-fallback" ]
}

@test "integration: 'quota exceeded' routes to engine fallback (exit 2)" {
run route_error 1 "" "quota exceeded"

[ "$status" -eq 2 ]
[ "$output" = "engine-fallback" ]
}

# ---------------------------------------------------------------------------
# Scenario: successful invocation (rc=0) — routing must be a no-op even if
# the reviewed PR happens to mention rate-limit keywords in its diff/output.
# (Regression guard for the single-review false-positive bug fixed in #148.)
# ---------------------------------------------------------------------------

@test "integration: rc=0 with 'rate limit' in stdout does NOT trigger exit 2" {
# A PR review that discusses rate-limit handling in code should not abort the session.
run route_error 0 "This PR adds rate limit handling to the API client." ""

[ "$status" -eq 0 ]
[ "$output" = "pass-through" ]
}

@test "integration: rc=0 with 'unknown flag' in stdout does NOT trigger exit 1" {
# A PR review that mentions CLI flags should not be misclassified as a CLI error.
run route_error 0 "The unknown flag --debug was passed to the CLI." ""

[ "$status" -eq 0 ]
[ "$output" = "pass-through" ]
}

# ---------------------------------------------------------------------------
# Scenario: unknown error (neither CLI nor rate-limit) — must pass through
# so review-one-pr.sh can apply its own fallback logic.
# ---------------------------------------------------------------------------

@test "integration: generic error (rc=1, no pattern match) passes through" {
run route_error 1 "some unexpected error occurred" "stack trace here"

[ "$status" -eq 0 ]
[ "$output" = "pass-through" ]
}
Loading
Loading