From 0cc21670fd13cee3f5c696d0bcea89b6bac930f4 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:24:05 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#195=20?= =?UTF-8?q?=E2=80=94=20bug(dev-lead):=20engine.sh=20defines=20multi-tier?= =?UTF-8?q?=20model=20pipeline=20but=20all=20tasks=20always=20invoke=20ENG?= =?UTF-8?q?INE=5FACTION=5FMODEL=20(sonnet-4-6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/dev-lead-fix-ci.sh | 2 +- scripts/dev-lead-fix-issue.sh | 2 +- scripts/dev-lead-fix-reviews.sh | 2 +- scripts/engine.sh | 102 +++++++- tests/dev-lead/unit/test_model_dispatch.bats | 212 +++++++++++++++++ .../dev-lead/unit/test_provider_headroom.bats | 221 ++++++++++++++++++ 6 files changed, 533 insertions(+), 8 deletions(-) create mode 100644 tests/dev-lead/unit/test_model_dispatch.bats create mode 100644 tests/dev-lead/unit/test_provider_headroom.bats diff --git a/scripts/dev-lead-fix-ci.sh b/scripts/dev-lead-fix-ci.sh index 15efa279f..d46a20a6d 100755 --- a/scripts/dev-lead-fix-ci.sh +++ b/scripts/dev-lead-fix-ci.sh @@ -226,7 +226,7 @@ main() { echo " [fix-ci] cycle $cycle/$MAX_CI_CYCLES" local engine_rc=0 - run_writer_with_fallback "$prompt_file" || engine_rc=$? + run_writer_with_fallback "$prompt_file" "fix-ci" || engine_rc=$? if [ "$engine_rc" -ne 0 ]; then if [ "$engine_rc" -eq 2 ]; then diff --git a/scripts/dev-lead-fix-issue.sh b/scripts/dev-lead-fix-issue.sh index d2d1cf127..09503d126 100755 --- a/scripts/dev-lead-fix-issue.sh +++ b/scripts/dev-lead-fix-issue.sh @@ -73,7 +73,7 @@ main() { pre_engine_sha=$(git rev-parse HEAD) local engine_rc=0 - run_writer_with_fallback "$prompt_file" || engine_rc=$? + run_writer_with_fallback "$prompt_file" "fix-issue" || engine_rc=$? if [ "$engine_rc" -eq 2 ]; then echo "::warning::All engines rate-limited — cannot implement issue #${ISSUE_NUMBER}; re-apply the label to retry" local reset_msg="" diff --git a/scripts/dev-lead-fix-reviews.sh b/scripts/dev-lead-fix-reviews.sh index dbb32bb8a..5c0532249 100755 --- a/scripts/dev-lead-fix-reviews.sh +++ b/scripts/dev-lead-fix-reviews.sh @@ -74,7 +74,7 @@ build_and_run() { fi local rc=0 - run_writer_with_fallback "$prompt_file" || rc=$? + run_writer_with_fallback "$prompt_file" "${INTENT_TYPE:-}" || rc=$? rm -f "$prompt_file" return "$rc" } diff --git a/scripts/engine.sh b/scripts/engine.sh index 195f33d15..b30b3b682 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -133,6 +133,83 @@ set_engine_config() { set_engine_config echo " engine: $REVIEW_ENGINE ($ENGINE_LABEL)" +# model_for_intent +# Returns the engine model appropriate for the given dev-lead intent type. +# Called after set_engine_config so the returned value reflects the active engine. +# Tier mapping (engine-neutral — each engine maps its own model variables): +# human-pr, fix-bot-comment → ENGINE_TRIAGE_MODEL (lightweight read/classify) +# fix-reviews, fix-ci, rebase → ENGINE_ACTION_MODEL (write operations) +# fix-issue, human → ENGINE_DEEP_MODEL (full agentic work) +# * (unknown/empty) → ENGINE_ACTION_MODEL (safe default) +model_for_intent() { + case "${1:-}" in + human-pr|fix-bot-comment) echo "$ENGINE_TRIAGE_MODEL" ;; + fix-reviews|fix-ci|rebase) echo "$ENGINE_ACTION_MODEL" ;; + fix-issue|human) echo "$ENGINE_DEEP_MODEL" ;; + *) echo "$ENGINE_ACTION_MODEL" ;; + esac +} + +# check_provider_headroom +# Returns 0 (ok to proceed) or 1 (at/above threshold — skip to next engine). +# Falls back to 0 (proceed) on any query failure so a missing API or network +# error never blocks work (fail-open by design). +# Threshold is DEV_LEAD_USAGE_THRESHOLD (default: 75%). +# Logs a one-line headroom status to stderr for the step summary. +check_provider_headroom() { + local engine="$1" + local used_pct=0 + local threshold="${DEV_LEAD_USAGE_THRESHOLD:-75}" + + case "$engine" in + claude) + # Probe the Anthropic API for rate-limit headers. Uses a minimal + # 1-token request so the probe itself barely consumes quota. + local _resp remaining_tokens limit_tokens + _resp=$(curl -sI -X POST https://api.anthropic.com/v1/messages \ + -H "x-api-key: ${ANTHROPIC_API_KEY:-}" \ + -H "anthropic-version: 2023-06-01" \ + -H "content-type: application/json" \ + --data-raw '{"model":"claude-haiku-4-5-20251001","max_tokens":1,"messages":[{"role":"user","content":"."}]}' \ + 2>/dev/null || true) + remaining_tokens=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-remaining-tokens:' \ + | awk '{print $2}' | tr -d '\r' || true) + limit_tokens=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-limit-tokens:' \ + | awk '{print $2}' | tr -d '\r' || true) + if [ -n "$remaining_tokens" ] && [ -n "$limit_tokens" ] && [ "$limit_tokens" -gt 0 ] 2>/dev/null; then + used_pct=$(( 100 - (remaining_tokens * 100 / limit_tokens) )) + fi + ;; + gemini) + # Gemini does not expose a usage header on free-tier probe endpoints. + echo " [headroom] gemini — no usage API, proceeding" >&2 + return 0 + ;; + copilot) + # Probe GitHub Models API rate-limit headers via a lightweight HEAD. + local _resp _remaining _limit + _resp=$(curl -sI https://models.github.ai/inference/chat/completions \ + -H "Authorization: Bearer ${COPILOT_GITHUB_TOKEN:-}" \ + -H "X-GitHub-Api-Version: 2022-11-28" 2>/dev/null || true) + _remaining=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-remaining-requests:' \ + | awk '{print $2}' | tr -d '\r' || true) + _limit=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-limit-requests:' \ + | awk '{print $2}' | tr -d '\r' || true) + if [ -n "$_remaining" ] && [ -n "$_limit" ] && [ "$_limit" -gt 0 ] 2>/dev/null; then + used_pct=$(( 100 - (_remaining * 100 / _limit) )) + fi + ;; + esac + + if [ "$used_pct" -ge "$threshold" ] 2>/dev/null; then + echo " [headroom] $engine usage ${used_pct}% >= threshold ${threshold}% — skipping" >&2 + return 1 + fi + + echo " [headroom] $engine usage ${used_pct}% — ok" >&2 + return 0 +} + # Load token metrics library unconditionally (non-fatal). # emit_token_record and friends are no-ops when TOKEN_LOG_FILE is unset. _TOKEN_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/token-metrics.sh" @@ -484,6 +561,7 @@ _record_engine_tokens() { } # run_triage +# Used by: review-one-pr.sh only (not the dev-lead writer pipeline). # No-tool mode. The prompt file already has all PR context inlined by the # caller (review-one-pr.sh builds it). Every tool is denied so the model # can't wander into the working directory and discover prs.txt or other @@ -560,6 +638,7 @@ run_triage() { } # run_agentic [tier] +# Used by: review-one-pr.sh only (not the dev-lead writer pipeline). # Full tool access (Bash, Read, Grep, Glob). Output to stdout. # # No retry here: callers redirect stdout to a file, so a retry inside this @@ -685,6 +764,7 @@ sys.exit(1) } # run_duck +# Used by: review-one-pr.sh only (not the dev-lead writer pipeline). # Cross-engine adversarial "rubber duck" review. # DUCK_ENGINE is set by engine.sh init: claude→copilot, gemini→claude, copilot→gemini. # All three engine branches (claude, gemini, copilot) are reachable — the gemini @@ -949,11 +1029,15 @@ run_writer() { return "$rc" } -# run_writer_with_fallback +# run_writer_with_fallback [intent_type] # Tries primary engine, falls back through claude → gemini → copilot on rate-limit. -# Only rate-limit (exit 2) triggers fallback; other failures propagate immediately. +# intent_type is passed to model_for_intent() so each engine uses the appropriate +# tier model for the given task complexity (e.g. haiku for triage, sonnet for writes). +# Only rate-limit (exit 2) and missing-binary (exit 127) trigger fallback; +# other failures propagate immediately. run_writer_with_fallback() { local prompt_file="$1" + local intent="${2:-}" local engines=("$REVIEW_ENGINE") for e in claude gemini copilot; do @@ -972,13 +1056,21 @@ run_writer_with_fallback() { continue fi + if ! check_provider_headroom "$engine"; then + echo "::warning::$engine at/above usage threshold — trying next engine" >&2 + any_rate_limited=1 + continue + fi + local saved="$REVIEW_ENGINE" export REVIEW_ENGINE="$engine" - # Re-evaluate model names for the new engine + # Re-evaluate model names for the new engine so model_for_intent returns + # the correct engine-specific model for the requested tier. set_engine_config + local model + model="$(model_for_intent "$intent")" local rc=0 - # Don't pass 'model' argument; run_writer will use the updated $ENGINE_ACTION_MODEL - run_writer "$prompt_file" || rc=$? + run_writer "$prompt_file" "$model" || rc=$? export REVIEW_ENGINE="$saved" # Restore original config for subsequent PRs in the same session set_engine_config diff --git a/tests/dev-lead/unit/test_model_dispatch.bats b/tests/dev-lead/unit/test_model_dispatch.bats new file mode 100644 index 000000000..f8cba6d3d --- /dev/null +++ b/tests/dev-lead/unit/test_model_dispatch.bats @@ -0,0 +1,212 @@ +#!/usr/bin/env bats +# Unit tests for engine.sh — model_for_intent() intent-based model dispatch + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +ENGINE_SCRIPT="$SCRIPT_DIR/scripts/engine.sh" + +setup() { + export GITHUB_ENV="$(mktemp)" + export GITHUB_OUTPUT="$(mktemp)" +} + +teardown() { + rm -f "$GITHUB_ENV" "$GITHUB_OUTPUT" +} + +_source_engine() { + local engine="${1:-claude}" + export REVIEW_ENGINE="$engine" + source "$ENGINE_SCRIPT" 2>/dev/null || true +} + +# ── triage-tier intents (haiku / lightest model) ────────────────────────────── + +@test "dispatch: human-pr intent returns ENGINE_TRIAGE_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "human-pr")" + [ "$result" = "$ENGINE_TRIAGE_MODEL" ] +} + +@test "dispatch: fix-bot-comment intent returns ENGINE_TRIAGE_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-bot-comment")" + [ "$result" = "$ENGINE_TRIAGE_MODEL" ] +} + +# ── action-tier intents (sonnet / write operations) ─────────────────────────── + +@test "dispatch: fix-reviews intent returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-reviews")" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +@test "dispatch: fix-ci intent returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-ci")" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +@test "dispatch: rebase intent returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "rebase")" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +# ── deep-tier intents (sonnet/opus / full agentic work) ─────────────────────── + +@test "dispatch: fix-issue intent returns ENGINE_DEEP_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-issue")" + [ "$result" = "$ENGINE_DEEP_MODEL" ] +} + +@test "dispatch: human intent returns ENGINE_DEEP_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "human")" + [ "$result" = "$ENGINE_DEEP_MODEL" ] +} + +# ── default/unknown intents fall back to ENGINE_ACTION_MODEL ────────────────── + +@test "dispatch: unknown intent returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "some-unknown-intent")" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +@test "dispatch: empty intent returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent "")" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +@test "dispatch: no-arg call returns ENGINE_ACTION_MODEL" { + _source_engine "claude" + local result + result="$(model_for_intent)" + [ "$result" = "$ENGINE_ACTION_MODEL" ] +} + +# ── triage model differs from action model on claude engine ─────────────────── + +@test "dispatch: triage model is different from action model on claude engine" { + _source_engine "claude" + [ "$ENGINE_TRIAGE_MODEL" != "$ENGINE_ACTION_MODEL" ] +} + +@test "dispatch: human-pr gets haiku (claude-haiku-4-5) not sonnet" { + _source_engine "claude" + local result + result="$(model_for_intent "human-pr")" + # Must be haiku, not sonnet + [[ "$result" == *"haiku"* ]] + [[ "$result" != *"sonnet"* ]] +} + +@test "dispatch: fix-reviews gets sonnet (claude-sonnet-4-6)" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-reviews")" + [[ "$result" == *"sonnet"* ]] +} + +# ── engine-specific models are respected ────────────────────────────────────── + +@test "dispatch: gemini engine — human-pr returns gemini triage model" { + _source_engine "gemini" + local result + result="$(model_for_intent "human-pr")" + [ "$result" = "$ENGINE_TRIAGE_MODEL" ] + [[ "$result" == *"gemini"* ]] +} + +@test "dispatch: gemini engine — fix-issue returns gemini deep model" { + _source_engine "gemini" + local result + result="$(model_for_intent "fix-issue")" + [ "$result" = "$ENGINE_DEEP_MODEL" ] + [[ "$result" == *"gemini"* ]] +} + +@test "dispatch: copilot engine — all intents return copilot model" { + _source_engine "copilot" + # For copilot, all tier models are the same (o4-mini) + local triage action deep + triage="$(model_for_intent "human-pr")" + action="$(model_for_intent "fix-reviews")" + deep="$(model_for_intent "fix-issue")" + [ "$triage" = "$ENGINE_TRIAGE_MODEL" ] + [ "$action" = "$ENGINE_ACTION_MODEL" ] + [ "$deep" = "$ENGINE_DEEP_MODEL" ] +} + +# ── run_writer_with_fallback passes intent through to model selection ───────── + +@test "dispatch: run_writer_with_fallback uses triage model for human-pr intent" { + _source_engine "claude" + export DEV_LEAD_DRY_RUN=false + # Capture which model was used by recording STUB_ENGINE_RECORD_MODELS + local record_file + record_file="$(mktemp)" + export STUB_ENGINE_RECORD_MODELS="$record_file" + + # Install a stub claude that records the --model arg + local stub_bin + stub_bin="$(mktemp -d)" + cp "$SCRIPT_DIR/tests/dev-lead/fixtures/engines/stub-claude" "$stub_bin/claude" + chmod +x "$stub_bin/claude" + export PATH="$stub_bin:$PATH" + + local prompt + prompt="$(mktemp)" + echo "test" > "$prompt" + + STUB_ENGINE_EXIT=0 run_writer_with_fallback "$prompt" "human-pr" + + # The model recorded by the stub must be the triage model (haiku) + local used_model + used_model="$(head -1 "$record_file")" + [[ "$used_model" == *"haiku"* ]] + + rm -f "$record_file" "$prompt" + rm -rf "$stub_bin" + unset STUB_ENGINE_RECORD_MODELS +} + +@test "dispatch: run_writer_with_fallback uses action model for fix-reviews intent" { + _source_engine "claude" + export DEV_LEAD_DRY_RUN=false + local record_file + record_file="$(mktemp)" + export STUB_ENGINE_RECORD_MODELS="$record_file" + + local stub_bin + stub_bin="$(mktemp -d)" + cp "$SCRIPT_DIR/tests/dev-lead/fixtures/engines/stub-claude" "$stub_bin/claude" + chmod +x "$stub_bin/claude" + export PATH="$stub_bin:$PATH" + + local prompt + prompt="$(mktemp)" + echo "test" > "$prompt" + + STUB_ENGINE_EXIT=0 run_writer_with_fallback "$prompt" "fix-reviews" + + local used_model + used_model="$(head -1 "$record_file")" + [[ "$used_model" == *"sonnet"* ]] + + rm -f "$record_file" "$prompt" + rm -rf "$stub_bin" + unset STUB_ENGINE_RECORD_MODELS +} diff --git a/tests/dev-lead/unit/test_provider_headroom.bats b/tests/dev-lead/unit/test_provider_headroom.bats new file mode 100644 index 000000000..a5bc5d8b7 --- /dev/null +++ b/tests/dev-lead/unit/test_provider_headroom.bats @@ -0,0 +1,221 @@ +#!/usr/bin/env bats +# Unit tests for engine.sh — check_provider_headroom() + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +ENGINE_SCRIPT="$SCRIPT_DIR/scripts/engine.sh" + +setup() { + export GITHUB_ENV="$(mktemp)" + export GITHUB_OUTPUT="$(mktemp)" + STUB_BIN_DIR="$(mktemp -d)" + export STUB_BIN_DIR + export PATH="$STUB_BIN_DIR:$PATH" +} + +teardown() { + rm -f "$GITHUB_ENV" "$GITHUB_OUTPUT" + rm -rf "$STUB_BIN_DIR" +} + +_source_engine() { + local engine="${1:-claude}" + export REVIEW_ENGINE="$engine" + source "$ENGINE_SCRIPT" 2>/dev/null || true +} + +# Helper: install a curl stub that outputs given headers and exits with given code +_make_curl_stub() { + local headers="$1" exit_code="${2:-0}" + cat > "$STUB_BIN_DIR/curl" < "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/curl" + + run check_provider_headroom "claude" + [ "$status" -eq 0 ] +} + +@test "headroom: returns 0 (proceed) when headers are absent" { + _source_engine "claude" + # curl succeeds but returns no rate-limit headers + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\ncontent-type: application/json\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + + run check_provider_headroom "claude" + [ "$status" -eq 0 ] +} + +# ── gemini always proceeds ──────────────────────────────────────────────────── + +@test "headroom: gemini returns 0 unconditionally (no headroom API)" { + _source_engine "gemini" + # Even if curl would fail, gemini should short-circuit and return 0 + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/curl" + + run check_provider_headroom "gemini" + [ "$status" -eq 0 ] +} + +# ── claude headroom threshold ───────────────────────────────────────────────── + +@test "headroom: claude returns 0 when usage is below threshold" { + _source_engine "claude" + # remaining=80000, limit=100000 → used_pct=20 < 75 threshold + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 80000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export ANTHROPIC_API_KEY="dummy" + + run check_provider_headroom "claude" + [ "$status" -eq 0 ] +} + +@test "headroom: claude returns 1 when usage is at or above threshold" { + _source_engine "claude" + # remaining=10000, limit=100000 → used_pct=90 >= 75 threshold + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 10000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export ANTHROPIC_API_KEY="dummy" + + run check_provider_headroom "claude" + [ "$status" -eq 1 ] +} + +@test "headroom: claude returns 1 when usage is exactly at threshold" { + _source_engine "claude" + # remaining=25000, limit=100000 → used_pct=75 = threshold + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 25000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export ANTHROPIC_API_KEY="dummy" + + run check_provider_headroom "claude" + [ "$status" -eq 1 ] +} + +# ── copilot headroom threshold ──────────────────────────────────────────────── + +@test "headroom: copilot returns 0 when usage is below threshold" { + _source_engine "copilot" + # remaining=100, limit=200 → used_pct=50 < 75 threshold + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-requests: 100\r\nx-ratelimit-limit-requests: 200\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export COPILOT_GITHUB_TOKEN="dummy" + + run check_provider_headroom "copilot" + [ "$status" -eq 0 ] +} + +@test "headroom: copilot returns 1 when usage is above threshold" { + _source_engine "copilot" + # remaining=10, limit=200 → used_pct=95 >= 75 threshold + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-requests: 10\r\nx-ratelimit-limit-requests: 200\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export COPILOT_GITHUB_TOKEN="dummy" + + run check_provider_headroom "copilot" + [ "$status" -eq 1 ] +} + +# ── configurable threshold ──────────────────────────────────────────────────── + +@test "headroom: DEV_LEAD_USAGE_THRESHOLD=50 skips claude at 60% usage" { + _source_engine "claude" + # remaining=40000, limit=100000 → used_pct=60 >= threshold 50 + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 40000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export ANTHROPIC_API_KEY="dummy" + export DEV_LEAD_USAGE_THRESHOLD=50 + + run check_provider_headroom "claude" + [ "$status" -eq 1 ] + unset DEV_LEAD_USAGE_THRESHOLD +} + +@test "headroom: DEV_LEAD_USAGE_THRESHOLD=90 allows claude at 80% usage" { + _source_engine "claude" + # remaining=20000, limit=100000 → used_pct=80 < threshold 90 + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 20000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +STUB + chmod +x "$STUB_BIN_DIR/curl" + export ANTHROPIC_API_KEY="dummy" + export DEV_LEAD_USAGE_THRESHOLD=90 + + run check_provider_headroom "claude" + [ "$status" -eq 0 ] + unset DEV_LEAD_USAGE_THRESHOLD +} + +# ── headroom check skips engine in run_writer_with_fallback ────────────────── + +@test "headroom: exhausted-claude skips to gemini in run_writer_with_fallback" { + _source_engine "claude" + export DEV_LEAD_DRY_RUN=false + + # Stub curl: report 95% usage for claude (returns 1 → skip), 0% for others + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +# claude probe hits api.anthropic.com; gemini/copilot use models.github.ai +if printf '%s' "$*" | grep -q "anthropic.com"; then + printf 'HTTP/2 200\r\nx-ratelimit-remaining-tokens: 5000\r\nx-ratelimit-limit-tokens: 100000\r\n\r\n' +else + printf 'HTTP/2 200\r\n\r\n' +fi +STUB + chmod +x "$STUB_BIN_DIR/curl" + + # Gemini stub succeeds + cp "$SCRIPT_DIR/tests/dev-lead/fixtures/engines/stub-gemini" "$STUB_BIN_DIR/gemini" + chmod +x "$STUB_BIN_DIR/gemini" + export STUB_ENGINE_EXIT=0 + export GEMINI_API_KEY="dummy-key" + export ANTHROPIC_API_KEY="dummy" + + local prompt + prompt="$(mktemp)" + echo "test" > "$prompt" + + run run_writer_with_fallback "$prompt" + + [ "$status" -eq 0 ] + rm -f "$prompt" +} From 02b08551efd386c5248c55730c8339b856de47c8 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:28:51 +0000 Subject: [PATCH 2/3] chore: apply manual instructions [skip ci-relay] --- scripts/engine.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index b30b3b682..92e1798cb 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -166,17 +166,17 @@ check_provider_headroom() { # Probe the Anthropic API for rate-limit headers. Uses a minimal # 1-token request so the probe itself barely consumes quota. local _resp remaining_tokens limit_tokens - _resp=$(curl -sI -X POST https://api.anthropic.com/v1/messages \ + _resp=$(curl -s -D - -o /dev/null -X POST https://api.anthropic.com/v1/messages \ -H "x-api-key: ${ANTHROPIC_API_KEY:-}" \ -H "anthropic-version: 2023-06-01" \ -H "content-type: application/json" \ --data-raw '{"model":"claude-haiku-4-5-20251001","max_tokens":1,"messages":[{"role":"user","content":"."}]}' \ 2>/dev/null || true) remaining_tokens=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-remaining-tokens:' \ - | awk '{print $2}' | tr -d '\r' || true) + | cut -d: -f2 | tr -d '[:space:]' || true) limit_tokens=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-limit-tokens:' \ - | awk '{print $2}' | tr -d '\r' || true) - if [ -n "$remaining_tokens" ] && [ -n "$limit_tokens" ] && [ "$limit_tokens" -gt 0 ] 2>/dev/null; then + | cut -d: -f2 | tr -d '[:space:]' || true) + if [[ "$remaining_tokens" =~ ^[0-9]+$ ]] && [[ "$limit_tokens" =~ ^[0-9]+$ ]] && [ "$limit_tokens" -gt 0 ]; then used_pct=$(( 100 - (remaining_tokens * 100 / limit_tokens) )) fi ;; @@ -192,10 +192,10 @@ check_provider_headroom() { -H "Authorization: Bearer ${COPILOT_GITHUB_TOKEN:-}" \ -H "X-GitHub-Api-Version: 2022-11-28" 2>/dev/null || true) _remaining=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-remaining-requests:' \ - | awk '{print $2}' | tr -d '\r' || true) + | cut -d: -f2 | tr -d '[:space:]' || true) _limit=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-limit-requests:' \ - | awk '{print $2}' | tr -d '\r' || true) - if [ -n "$_remaining" ] && [ -n "$_limit" ] && [ "$_limit" -gt 0 ] 2>/dev/null; then + | cut -d: -f2 | tr -d '[:space:]' || true) + if [[ "$_remaining" =~ ^[0-9]+$ ]] && [[ "$_limit" =~ ^[0-9]+$ ]] && [ "$_limit" -gt 0 ]; then used_pct=$(( 100 - (_remaining * 100 / _limit) )) fi ;; From 6c7c332cf08487bfba9c0ea5efa67111ec3c8aee Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:52:10 +0000 Subject: [PATCH 3/3] chore: apply manual instructions [skip ci-relay] --- scripts/engine.sh | 12 +++++-- .../dev-lead/unit/test_provider_headroom.bats | 34 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index 92e1798cb..e9962682f 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -186,10 +186,18 @@ check_provider_headroom() { return 0 ;; copilot) + # Skip probe when no real GitHub token is present — avoids unnecessary + # external calls in unit tests and CI environments where the token is + # unset or set to a placeholder value. + local _tok="${COPILOT_GITHUB_TOKEN:-}" + if [[ -z "$_tok" ]] || [[ ! "$_tok" =~ ^(github_pat_|ghp_|ghs_) ]]; then + echo " [headroom] copilot — no valid token, proceeding" >&2 + return 0 + fi # Probe GitHub Models API rate-limit headers via a lightweight HEAD. local _resp _remaining _limit - _resp=$(curl -sI https://models.github.ai/inference/chat/completions \ - -H "Authorization: Bearer ${COPILOT_GITHUB_TOKEN:-}" \ + _resp=$(curl -sI --max-time 5 https://models.github.ai/inference/chat/completions \ + -H "Authorization: Bearer ${_tok}" \ -H "X-GitHub-Api-Version: 2022-11-28" 2>/dev/null || true) _remaining=$(printf '%s' "$_resp" | grep -i 'x-ratelimit-remaining-requests:' \ | cut -d: -f2 | tr -d '[:space:]' || true) diff --git a/tests/dev-lead/unit/test_provider_headroom.bats b/tests/dev-lead/unit/test_provider_headroom.bats index a5bc5d8b7..1f74e5497 100644 --- a/tests/dev-lead/unit/test_provider_headroom.bats +++ b/tests/dev-lead/unit/test_provider_headroom.bats @@ -131,7 +131,7 @@ STUB printf 'HTTP/2 200\r\nx-ratelimit-remaining-requests: 100\r\nx-ratelimit-limit-requests: 200\r\n\r\n' STUB chmod +x "$STUB_BIN_DIR/curl" - export COPILOT_GITHUB_TOKEN="dummy" + export COPILOT_GITHUB_TOKEN="github_pat_testdummyvalue123" run check_provider_headroom "copilot" [ "$status" -eq 0 ] @@ -145,12 +145,42 @@ STUB printf 'HTTP/2 200\r\nx-ratelimit-remaining-requests: 10\r\nx-ratelimit-limit-requests: 200\r\n\r\n' STUB chmod +x "$STUB_BIN_DIR/curl" - export COPILOT_GITHUB_TOKEN="dummy" + export COPILOT_GITHUB_TOKEN="github_pat_testdummyvalue123" run check_provider_headroom "copilot" [ "$status" -eq 1 ] } +@test "headroom: copilot returns 0 (proceed) when COPILOT_GITHUB_TOKEN is unset" { + _source_engine "copilot" + unset COPILOT_GITHUB_TOKEN + # curl stub fails so an accidental call would be caught + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/curl" + + run check_provider_headroom "copilot" + [ "$status" -eq 0 ] + [[ "$output" == *"no valid token"* ]] +} + +@test "headroom: copilot returns 0 (proceed) when COPILOT_GITHUB_TOKEN is a placeholder" { + _source_engine "copilot" + export COPILOT_GITHUB_TOKEN="dummy" + # curl stub fails so an accidental call would be caught + cat > "$STUB_BIN_DIR/curl" <<'STUB' +#!/usr/bin/env bash +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/curl" + + run check_provider_headroom "copilot" + [ "$status" -eq 0 ] + [[ "$output" == *"no valid token"* ]] +} + # ── configurable threshold ──────────────────────────────────────────────────── @test "headroom: DEV_LEAD_USAGE_THRESHOLD=50 skips claude at 60% usage" {