From 43a2c655200558edd7c4252ef0675fa3ea694b6a Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:50:08 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#382=20?= =?UTF-8?q?=E2=80=94=20Enhance=20dev-lead=20and=20pr-review=20engines=20fo?= =?UTF-8?q?r=20Gemini=203.5=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/engine.sh | 181 ++++++++++- scripts/lib/model-pricing.tsv | 1 + scripts/validate-engines.sh | 2 +- tests/dev-lead/fixtures/engines/stub-gemini | 33 +- tests/dev-lead/unit/test_engine_chain.bats | 8 +- .../dev-lead/unit/test_engine_gemini_35.bats | 290 ++++++++++++++++++ 6 files changed, 494 insertions(+), 21 deletions(-) create mode 100644 tests/dev-lead/unit/test_engine_gemini_35.bats diff --git a/scripts/engine.sh b/scripts/engine.sh index 1ea2fe35f..4ebdbcfc2 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -76,17 +76,29 @@ set_engine_config() { CLAUDE_SINGLE_MODEL_CHAIN="${CLAUDE_SINGLE_MODEL_CHAIN:-claude-fable-5,claude-opus-4-8,claude-opus-4-7}" ;; gemini) - ENGINE_TRIAGE_MODEL="gemini-2.0-flash" - ENGINE_DEEP_MODEL="gemini-2.5-pro" - ENGINE_AUDIT_MODEL="gemini-2.5-pro" - ENGINE_ACTION_MODEL="gemini-2.5-pro" - ENGINE_SINGLE_MODEL="gemini-2.5-pro" - ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-2.5-pro + duck: sonnet 4.6 → audit: gemini-2.5-pro" - ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-2.5-pro" + # Per-engine model overrides via env (env → default). + # GEMINI_FLASH_MODEL controls the speed/cost tier (triage + action). + # GEMINI_PRO_MODEL controls the quality tier (deep + audit + single). + local _gflash="${GEMINI_FLASH_MODEL:-gemini-3.5-flash}" + local _gpro="${GEMINI_PRO_MODEL:-gemini-2.5-pro}" + ENGINE_TRIAGE_MODEL="$_gflash" + ENGINE_DEEP_MODEL="$_gpro" + ENGINE_AUDIT_MODEL="$_gpro" + ENGINE_ACTION_MODEL="$_gflash" + ENGINE_SINGLE_MODEL="$_gpro" + ENGINE_LABEL="triage: $_gflash → deep: $_gpro + duck: sonnet 4.6 → audit: $_gpro" + ENGINE_SINGLE_LABEL="single-reviewer mode: $_gpro" # Cross-engine rubber duck: use Claude for diversity DUCK_ENGINE="claude" DUCK_MODEL="claude-sonnet-4-6" - # No in-engine chain for Gemini — only one production model in use today. + # In-Gemini model fallback chains (comma-separated, walked left-to-right on rate-limit). + # Flash chain: 3.5-flash (speed/cost) → 2.5-pro (quality fallback on exhaustion). + # Pro chain: 2.5-pro (quality) → 2.0-flash (graceful degradation on exhaustion). + # Override per workflow via env to tune cost/capability trade-offs. + GEMINI_FLASH_MODEL_CHAIN="${GEMINI_FLASH_MODEL_CHAIN:-${_gflash},gemini-2.5-pro}" + GEMINI_PRO_MODEL_CHAIN="${GEMINI_PRO_MODEL_CHAIN:-${_gpro},gemini-2.0-flash}" + # Clear the Claude-only chain vars so callers that check them unconditionally + # do not accidentally apply a stale Claude chain to the Gemini engine. CLAUDE_TRIAGE_MODEL_CHAIN="" CLAUDE_DEEP_MODEL_CHAIN="" CLAUDE_AUDIT_MODEL_CHAIN="" @@ -110,7 +122,7 @@ set_engine_config() { ENGINE_SINGLE_LABEL="single-reviewer mode: o4-mini (GitHub Models API)" # Cross-engine rubber duck: use Gemini when Copilot is primary DUCK_ENGINE="gemini" - DUCK_MODEL="gemini-2.0-flash" + DUCK_MODEL="gemini-3.5-flash" # No in-engine chain for Copilot — single GitHub Models endpoint. CLAUDE_TRIAGE_MODEL_CHAIN="" CLAUDE_DEEP_MODEL_CHAIN="" @@ -131,6 +143,7 @@ set_engine_config() { export CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN export CLAUDE_AUDIT_MODEL_CHAIN CLAUDE_ACTION_MODEL_CHAIN export CLAUDE_SINGLE_MODEL_CHAIN + export GEMINI_FLASH_MODEL_CHAIN GEMINI_PRO_MODEL_CHAIN } # Initial config @@ -384,6 +397,113 @@ _gemini_invoke() { --output-format text < "$prompt_file" } +# _gemini_chain_invoke [extra_args...] +# Walks a comma-separated list of Gemini models, invoking _gemini_invoke with +# each model in sequence. Semantics mirror _claude_chain_invoke: +# - First model that succeeds (exit 0) wins; its stdout is emitted, exit 0 returned. +# - Rate-limited models (is_rate_limited_files) are logged and skipped; next tried. +# - Non-rate-limit failures stop the chain immediately; exit code propagated. +# - If every model rate-limits, returns 2 and writes the parsed reset time. +# - Empty/whitespace-only chain → config error exit 1. +# +# Sets _GEMINI_CHAIN_MODEL_USED to the model that produced the final output. +# Note: when called inside a pipeline (e.g. `... | tee ...`), the variable is +# set in a subshell and will not propagate to the parent — same limitation as +# _CLAUDE_CHAIN_MODEL_USED in _claude_chain_invoke. +_gemini_chain_invoke() { + local chain_csv="$1" prompt_file="$2" timeout_sec="$3" + shift 3 + local extra_args=("$@") + + if [ -z "$chain_csv" ]; then + echo "::error::_gemini_chain_invoke called with empty chain" >&2 + return 1 + fi + + local saved_ifs="$IFS" + IFS=',' read -ra models <<< "$chain_csv" + IFS="$saved_ifs" + + local stdout_tmp="" stderr_tmp="" + local final_stdout="" final_stderr="" final_model="" final_rc=0 + local rc=0 attempted=0 all_rl=1 + local model + + for model in "${models[@]}"; do + # Trim whitespace + model="${model#"${model%%[![:space:]]*}"}" + model="${model%"${model##*[![:space:]]}"}" + [ -z "$model" ] && continue + + attempted=$((attempted + 1)) + stdout_tmp="$(mktemp 2>/dev/null)" || stdout_tmp="" + stderr_tmp="$(mktemp 2>/dev/null)" || stderr_tmp="" + rc=0 + + if [ -n "$stdout_tmp" ] && [ -n "$stderr_tmp" ]; then + # stderr intentionally passed through from _gemini_invoke (rate-limit msgs live there). + _gemini_invoke "$prompt_file" "$timeout_sec" "$model" "${extra_args[@]}" \ + > "$stdout_tmp" 2> "$stderr_tmp" || rc=$? + else + # mktemp failure — clean up partial allocations and fall through without capture. + [ -n "$stdout_tmp" ] && rm -f "$stdout_tmp" + [ -n "$stderr_tmp" ] && rm -f "$stderr_tmp" + [ -n "$final_stdout" ] && rm -f "$final_stdout" + [ -n "$final_stderr" ] && rm -f "$final_stderr" + _gemini_invoke "$prompt_file" "$timeout_sec" "$model" "${extra_args[@]}" || rc=$? + _GEMINI_CHAIN_MODEL_USED="$model" + export _GEMINI_CHAIN_MODEL_USED + return "$rc" + fi + + # Keep only the latest attempt's buffers + [ -n "$final_stdout" ] && rm -f "$final_stdout" + [ -n "$final_stderr" ] && rm -f "$final_stderr" + final_stdout="$stdout_tmp" + final_stderr="$stderr_tmp" + final_model="$model" + + if [ "$rc" -eq 0 ]; then + final_rc=0 + all_rl=0 + break + fi + if ! is_rate_limited_files "$stdout_tmp" "$stderr_tmp"; then + # Non-rate-limit failure — propagate immediately without trying next model. + final_rc="$rc" + all_rl=0 + break + fi + # Rate-limited; log and try next model. + # Phrasing avoids tokens that match _rate_limit_pattern so downstream callers + # that scan stderr do not misclassify a successful chain fallback as a rate-limit. + echo "::warning::[gemini] model $model throttled (rc=$rc) — trying next in chain" >&2 + done + + if [ "$attempted" -eq 0 ]; then + echo "::error::_gemini_chain_invoke: chain '$chain_csv' had no valid model entries" >&2 + return 1 + fi + + if [ "$all_rl" -eq 1 ]; then + parse_reset_time_files "$final_stdout" "$final_stderr" + final_rc=2 + fi + + if [ -n "$final_stdout" ]; then + cat "$final_stdout" + rm -f "$final_stdout" + fi + if [ -n "$final_stderr" ]; then + cat "$final_stderr" >&2 + rm -f "$final_stderr" + fi + + _GEMINI_CHAIN_MODEL_USED="$final_model" + export _GEMINI_CHAIN_MODEL_USED + return "$final_rc" +} + # _claude_chain_invoke [extra_args...] # Walks a comma-separated list of Claude models, invoking `claude --print --model X` # with the given extra arguments. The first model whose run does NOT trigger @@ -612,11 +732,12 @@ run_triage() { fi ;; gemini) + local _triage_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$ENGINE_TRIAGE_MODEL}" if [ -n "$_tok_tmp" ]; then - _gemini_invoke "$prompt_file" "$TRIAGE_TIMEOUT_SEC" "$ENGINE_TRIAGE_MODEL" \ + _gemini_chain_invoke "$_triage_gemini_chain" "$prompt_file" "$TRIAGE_TIMEOUT_SEC" \ --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - _gemini_invoke "$prompt_file" "$TRIAGE_TIMEOUT_SEC" "$ENGINE_TRIAGE_MODEL" \ + _gemini_chain_invoke "$_triage_gemini_chain" "$prompt_file" "$TRIAGE_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$? fi ;; @@ -630,7 +751,14 @@ run_triage() { ;; esac if [ "$rc" -eq 0 ]; then - local _triage_used="${_CLAUDE_CHAIN_MODEL_USED:-$ENGINE_TRIAGE_MODEL}" + local _triage_used + if [ -n "${_CLAUDE_CHAIN_MODEL_USED:-}" ]; then + _triage_used="$_CLAUDE_CHAIN_MODEL_USED" + elif [ -n "${_GEMINI_CHAIN_MODEL_USED:-}" ]; then + _triage_used="$_GEMINI_CHAIN_MODEL_USED" + else + _triage_used="$ENGINE_TRIAGE_MODEL" + fi _record_engine_tokens "triage" "$REVIEW_ENGINE" "$_triage_used" "$prompt_file" "$_tok_tmp" [ -n "$_tok_tmp" ] && rm -f "$_tok_tmp" return 0 @@ -709,11 +837,23 @@ run_agentic() { fi ;; gemini) + # Capability-aware chain selection: flash chain for speed tiers, pro chain + # for quality tiers. Honor explicit model pin when it differs from the tier default. + local _agentic_gemini_chain _gemini_tier_default="" + case "$tier" in + deep|audit|single) _agentic_gemini_chain="${GEMINI_PRO_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_DEEP_MODEL:-}" ;; + *) _agentic_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_ACTION_MODEL:-}" ;; + esac + if [ -n "$_gemini_tier_default" ] && [ "$model" != "$_gemini_tier_default" ]; then + _agentic_gemini_chain="$model" + fi if [ -n "$_tok_tmp" ]; then - _gemini_invoke "$prompt_file" "$DEEP_TIMEOUT_SEC" "$model" \ + _gemini_chain_invoke "$_agentic_gemini_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \ --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - _gemini_invoke "$prompt_file" "$DEEP_TIMEOUT_SEC" "$model" \ + _gemini_chain_invoke "$_agentic_gemini_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$? fi ;; @@ -734,6 +874,8 @@ run_agentic() { local _agentic_used="$model" if [ "$REVIEW_ENGINE" = "claude" ] && [ -n "${_CLAUDE_CHAIN_MODEL_USED:-}" ]; then _agentic_used="$_CLAUDE_CHAIN_MODEL_USED" + elif [ "$REVIEW_ENGINE" = "gemini" ] && [ -n "${_GEMINI_CHAIN_MODEL_USED:-}" ]; then + _agentic_used="$_GEMINI_CHAIN_MODEL_USED" fi _record_engine_tokens "$tier" "$REVIEW_ENGINE" "$_agentic_used" "$prompt_file" "$_tok_tmp" fi @@ -968,11 +1110,16 @@ run_writer() { fi ;; gemini) + # Flash chain for writer (action) tier; honor explicit model pin. + local _writer_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$model}" + if [ -n "${ENGINE_ACTION_MODEL:-}" ] && [ "$model" != "$ENGINE_ACTION_MODEL" ]; then + _writer_gemini_chain="$model" + fi if [ -n "$_tmp" ]; then - _gemini_invoke "$prompt_file" "$ACTION_TIMEOUT_SEC" "$model" \ + _gemini_chain_invoke "$_writer_gemini_chain" "$prompt_file" "$ACTION_TIMEOUT_SEC" \ --approval-mode auto_edit 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else - _gemini_invoke "$prompt_file" "$ACTION_TIMEOUT_SEC" "$model" \ + _gemini_chain_invoke "$_writer_gemini_chain" "$prompt_file" "$ACTION_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$? fi ;; @@ -997,6 +1144,8 @@ run_writer() { local _writer_used="$model" if [ "$REVIEW_ENGINE" = "claude" ] && [ -n "${_CLAUDE_CHAIN_MODEL_USED:-}" ]; then _writer_used="$_CLAUDE_CHAIN_MODEL_USED" + elif [ "$REVIEW_ENGINE" = "gemini" ] && [ -n "${_GEMINI_CHAIN_MODEL_USED:-}" ]; then + _writer_used="$_GEMINI_CHAIN_MODEL_USED" fi _record_engine_tokens "writer" "$REVIEW_ENGINE" "$_writer_used" "$prompt_file" "$_tmp" fi diff --git a/scripts/lib/model-pricing.tsv b/scripts/lib/model-pricing.tsv index 20c6961ac..5499461cb 100644 --- a/scripts/lib/model-pricing.tsv +++ b/scripts/lib/model-pricing.tsv @@ -40,5 +40,6 @@ claude-haiku-3-5* 2024-01-01 0.80 0.08 1.00 4.00 # ── Cross-engine (VERIFY — provisional; cache_write placeholder = input) ─────── o4-mini* 2025-01-01 2.00 0.50 2.00 8.00 +gemini-3.5-flash* 2026-06-11 0.30 0.03 0.30 1.20 *gemini*flash* 2025-01-01 0.50 0.05 0.50 2.00 *gemini*pro* 2025-01-01 2.00 0.20 2.00 8.00 diff --git a/scripts/validate-engines.sh b/scripts/validate-engines.sh index cb82eabc2..b2c4a9248 100644 --- a/scripts/validate-engines.sh +++ b/scripts/validate-engines.sh @@ -40,7 +40,7 @@ _gemini_billing_probe() { -X POST \ -H "Content-Type: application/json" \ -H "X-Goog-Api-Key: ${GOOGLE_API_KEY}" \ - "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent" \ + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent" \ -d '{"contents":[{"parts":[{"text":"Hi"}]}],"generationConfig":{"maxOutputTokens":1}}' \ -w '\n%{http_code}' 2>/dev/null ) || true diff --git a/tests/dev-lead/fixtures/engines/stub-gemini b/tests/dev-lead/fixtures/engines/stub-gemini index aec46d83e..5d1908c10 100755 --- a/tests/dev-lead/fixtures/engines/stub-gemini +++ b/tests/dev-lead/fixtures/engines/stub-gemini @@ -1,6 +1,13 @@ #!/usr/bin/env bash # Configurable via: # STUB_ENGINE_RESPONSE, STUB_ENGINE_EXIT, STUB_ENGINE_DELAY +# STUB_ENGINE_EXIT_BY_MODEL — pipe-delimited list of "model=exit_code" pairs. +# If set and a pair matches --model, that exit +# code overrides STUB_ENGINE_EXIT. +# STUB_ENGINE_RESPONSE_BY_MODEL — pipe-delimited "model=response" pairs, +# same lookup as EXIT_BY_MODEL. +# STUB_ENGINE_RECORD_MODELS — if set to a writable path, every --model value +# seen is appended as a new line for inspection. # STUB_GEMINI_USAGE — "prompt cached candidates" emitted under # .stats.models..tokens when --output-format json # is requested. Default "120 20 40". @@ -19,9 +26,23 @@ while [[ $# -gt 0 ]]; do *) shift ;; esac done + +if [ -n "${STUB_ENGINE_RECORD_MODELS:-}" ]; then + printf '%s\n' "$model" >> "$STUB_ENGINE_RECORD_MODELS" 2>/dev/null || true +fi + sleep "${STUB_ENGINE_DELAY:-0}" +# Resolve response: per-model override → default. response="${STUB_ENGINE_RESPONSE:-stub engine response}" +if [ -n "${STUB_ENGINE_RESPONSE_BY_MODEL:-}" ] && [ -n "$model" ]; then + IFS='|' read -ra _pairs <<< "$STUB_ENGINE_RESPONSE_BY_MODEL" + for _p in "${_pairs[@]}"; do + case "$_p" in + "$model="*) response="${_p#"$model="}"; break ;; + esac + done +fi if [ "$output_format" = "json" ]; then read -r _p _c _cand <<< "${STUB_GEMINI_USAGE:-120 20 40}" @@ -43,4 +64,14 @@ else printf '%s\n' "$response" fi -exit "${STUB_ENGINE_EXIT:-0}" +# Resolve exit code: per-model override → default. +exit_code="${STUB_ENGINE_EXIT:-0}" +if [ -n "${STUB_ENGINE_EXIT_BY_MODEL:-}" ] && [ -n "$model" ]; then + IFS='|' read -ra _epairs <<< "$STUB_ENGINE_EXIT_BY_MODEL" + for _ep in "${_epairs[@]}"; do + case "$_ep" in + "$model="*) exit_code="${_ep#"$model="}"; break ;; + esac + done +fi +exit "$exit_code" diff --git a/tests/dev-lead/unit/test_engine_chain.bats b/tests/dev-lead/unit/test_engine_chain.bats index c1c066e09..3042e3e3c 100644 --- a/tests/dev-lead/unit/test_engine_chain.bats +++ b/tests/dev-lead/unit/test_engine_chain.bats @@ -187,15 +187,17 @@ _source_engine() { # ── Gemini/Copilot unchanged: no chain applied ──────────────────────────────── -@test "gemini: no in-engine chain; single-model behavior unchanged" { +@test "gemini: in-engine chain uses gemini model (not claude)" { _source_engine "gemini" export STUB_ENGINE_EXIT=0 run run_writer "$TEST_PROMPT" [ "$status" -eq 0 ] - # No claude model recorded (we ran gemini) - [ ! -s "$MODEL_RECORD" ] + # A gemini model was invoked (stub-gemini now records to MODEL_RECORD) + grep -q "gemini" "$MODEL_RECORD" + # No claude model was invoked + ! grep -q "claude" "$MODEL_RECORD" } # ── File-based rate-limit + reset-time helpers ───────────────────────────── diff --git a/tests/dev-lead/unit/test_engine_gemini_35.bats b/tests/dev-lead/unit/test_engine_gemini_35.bats new file mode 100644 index 000000000..b84fd6e37 --- /dev/null +++ b/tests/dev-lead/unit/test_engine_gemini_35.bats @@ -0,0 +1,290 @@ +#!/usr/bin/env bats +# Unit tests for engine.sh — Gemini 3.5 model tier assignments, chains, and fallback. +# +# Verifies that gemini-3.5-flash is wired into the speed-first tiers and +# gemini-2.5-pro remains the quality-first tier for the gemini engine: +# triage → gemini-3.5-flash (was gemini-2.0-flash) +# action → gemini-3.5-flash (was gemini-2.5-pro) +# deep → gemini-2.5-pro (unchanged) +# audit → gemini-2.5-pro (unchanged) +# single → gemini-2.5-pro (unchanged) + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +ENGINE_SCRIPT="$SCRIPT_DIR/scripts/engine.sh" +STUB_ENGINES_DIR="$SCRIPT_DIR/tests/dev-lead/fixtures/engines" + +setup() { + export GITHUB_ENV="$(mktemp)" + export GITHUB_OUTPUT="$(mktemp)" + + unset GEMINI_FLASH_MODEL GEMINI_PRO_MODEL + unset GEMINI_FLASH_MODEL_CHAIN GEMINI_PRO_MODEL_CHAIN + + STUB_BIN_DIR="$(mktemp -d)" + cp "$STUB_ENGINES_DIR/stub-gemini" "$STUB_BIN_DIR/gemini" + chmod +x "$STUB_BIN_DIR/gemini" + export PATH="$STUB_BIN_DIR:$PATH" + export STUB_BIN_DIR + + TEST_PROMPT="$(mktemp)" + echo "test prompt content" > "$TEST_PROMPT" + export TEST_PROMPT + + MODEL_RECORD="$(mktemp)" + export STUB_ENGINE_RECORD_MODELS="$MODEL_RECORD" + + export DEV_LEAD_DRY_RUN="false" +} + +teardown() { + rm -f "$GITHUB_ENV" "$GITHUB_OUTPUT" "$TEST_PROMPT" "$MODEL_RECORD" + rm -rf "$STUB_BIN_DIR" + unset GEMINI_FLASH_MODEL GEMINI_PRO_MODEL + unset GEMINI_FLASH_MODEL_CHAIN GEMINI_PRO_MODEL_CHAIN + unset STUB_ENGINE_EXIT_BY_MODEL STUB_ENGINE_RESPONSE_BY_MODEL + unset STUB_ENGINE_RECORD_MODELS STUB_ENGINE_EXIT STUB_ENGINE_RESPONSE +} + +_source_engine() { + local engine="${1:-claude}" + export REVIEW_ENGINE="$engine" + source "$ENGINE_SCRIPT" 2>/dev/null || true +} + +# ── Tier default model assignments ─────────────────────────────────────────── + +@test "gemini-3.5: ENGINE_TRIAGE_MODEL is gemini-3.5-flash by default" { + _source_engine "gemini" + [ "$ENGINE_TRIAGE_MODEL" = "gemini-3.5-flash" ] +} + +@test "gemini-3.5: ENGINE_ACTION_MODEL is gemini-3.5-flash by default" { + _source_engine "gemini" + [ "$ENGINE_ACTION_MODEL" = "gemini-3.5-flash" ] +} + +@test "gemini-3.5: ENGINE_DEEP_MODEL is gemini-2.5-pro (quality tier unchanged)" { + _source_engine "gemini" + [ "$ENGINE_DEEP_MODEL" = "gemini-2.5-pro" ] +} + +@test "gemini-3.5: ENGINE_AUDIT_MODEL is gemini-2.5-pro (quality tier unchanged)" { + _source_engine "gemini" + [ "$ENGINE_AUDIT_MODEL" = "gemini-2.5-pro" ] +} + +@test "gemini-3.5: ENGINE_SINGLE_MODEL is gemini-2.5-pro (quality tier unchanged)" { + _source_engine "gemini" + [ "$ENGINE_SINGLE_MODEL" = "gemini-2.5-pro" ] +} + +# ── Fallback chain configuration ────────────────────────────────────────────── + +@test "gemini-3.5: GEMINI_FLASH_MODEL_CHAIN starts with gemini-3.5-flash" { + _source_engine "gemini" + local first + first="${GEMINI_FLASH_MODEL_CHAIN%%,*}" + first="${first// /}" + [ "$first" = "gemini-3.5-flash" ] +} + +@test "gemini-3.5: GEMINI_FLASH_MODEL_CHAIN includes gemini-2.5-pro as fallback" { + _source_engine "gemini" + [[ "$GEMINI_FLASH_MODEL_CHAIN" == *"gemini-2.5-pro"* ]] +} + +@test "gemini-3.5: GEMINI_PRO_MODEL_CHAIN starts with gemini-2.5-pro" { + _source_engine "gemini" + local first + first="${GEMINI_PRO_MODEL_CHAIN%%,*}" + first="${first// /}" + [ "$first" = "gemini-2.5-pro" ] +} + +@test "gemini-3.5: GEMINI_PRO_MODEL_CHAIN includes gemini-2.0-flash as fallback" { + _source_engine "gemini" + [[ "$GEMINI_PRO_MODEL_CHAIN" == *"gemini-2.0-flash"* ]] +} + +# ── Env var overrides ───────────────────────────────────────────────────────── + +@test "gemini-3.5: GEMINI_FLASH_MODEL env var overrides triage model" { + export GEMINI_FLASH_MODEL="gemini-2.0-flash" + _source_engine "gemini" + [ "$ENGINE_TRIAGE_MODEL" = "gemini-2.0-flash" ] + [ "$ENGINE_ACTION_MODEL" = "gemini-2.0-flash" ] +} + +@test "gemini-3.5: GEMINI_PRO_MODEL env var overrides deep model" { + export GEMINI_PRO_MODEL="gemini-2.0-flash" + _source_engine "gemini" + [ "$ENGINE_DEEP_MODEL" = "gemini-2.0-flash" ] + [ "$ENGINE_AUDIT_MODEL" = "gemini-2.0-flash" ] +} + +@test "gemini-3.5: GEMINI_FLASH_MODEL_CHAIN env var override honored" { + export GEMINI_FLASH_MODEL_CHAIN="gemini-2.0-flash,gemini-2.5-pro" + _source_engine "gemini" + local first + first="${GEMINI_FLASH_MODEL_CHAIN%%,*}" + first="${first// /}" + [ "$first" = "gemini-2.0-flash" ] +} + +# ── Intent dispatch returns correct tier models ─────────────────────────────── + +@test "gemini-3.5: model_for_intent(human-pr) returns gemini-3.5-flash (triage)" { + _source_engine "gemini" + local result + result="$(model_for_intent "human-pr")" + [ "$result" = "gemini-3.5-flash" ] +} + +@test "gemini-3.5: model_for_intent(fix-reviews) returns gemini-3.5-flash (action)" { + _source_engine "gemini" + local result + result="$(model_for_intent "fix-reviews")" + [ "$result" = "gemini-3.5-flash" ] +} + +@test "gemini-3.5: model_for_intent(fix-issue) returns gemini-2.5-pro (deep)" { + _source_engine "gemini" + local result + result="$(model_for_intent "fix-issue")" + [ "$result" = "gemini-2.5-pro" ] +} + +# ── ENGINE_LABEL reflects new models ───────────────────────────────────────── + +@test "gemini-3.5: ENGINE_LABEL contains gemini-3.5-flash" { + _source_engine "gemini" + [[ "$ENGINE_LABEL" == *"gemini-3.5-flash"* ]] +} + +# ── Copilot engine duck uses gemini-3.5-flash ──────────────────────────────── + +@test "gemini-3.5: copilot engine duck model is gemini-3.5-flash" { + _source_engine "copilot" + [ "$DUCK_ENGINE" = "gemini" ] + [ "$DUCK_MODEL" = "gemini-3.5-flash" ] +} + +# ── _gemini_chain_invoke unit tests ─────────────────────────────────────────── + +@test "gemini-3.5 chain: first model succeeds → no fallback attempted" { + _source_engine "gemini" + export STUB_ENGINE_EXIT=0 + + run _gemini_chain_invoke "gemini-3.5-flash,gemini-2.5-pro" \ + "$TEST_PROMPT" 30 --approval-mode auto_edit + + [ "$status" -eq 0 ] + [ "$(wc -l < "$MODEL_RECORD")" -eq 1 ] + grep -q "gemini-3.5-flash" "$MODEL_RECORD" + ! grep -q "gemini-2.5-pro" "$MODEL_RECORD" +} + +@test "gemini-3.5 chain: first model rate-limited → falls through to second" { + _source_engine "gemini" + export STUB_ENGINE_EXIT_BY_MODEL="gemini-3.5-flash=1|gemini-2.5-pro=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="gemini-3.5-flash=429 too many requests|gemini-2.5-pro=pro response" + + run _gemini_chain_invoke "gemini-3.5-flash,gemini-2.5-pro" \ + "$TEST_PROMPT" 30 --approval-mode auto_edit + + [ "$status" -eq 0 ] + grep -q "gemini-3.5-flash" "$MODEL_RECORD" + grep -q "gemini-2.5-pro" "$MODEL_RECORD" + [[ "$output" == *"pro response"* ]] + [[ "$output" != *"too many requests"* ]] +} + +@test "gemini-3.5 chain: all models rate-limited → returns exit 2" { + _source_engine "gemini" + export STUB_ENGINE_EXIT_BY_MODEL="gemini-3.5-flash=1|gemini-2.5-pro=1" + export STUB_ENGINE_RESPONSE_BY_MODEL="gemini-3.5-flash=quota exceeded|gemini-2.5-pro=quota exceeded" + + run _gemini_chain_invoke "gemini-3.5-flash,gemini-2.5-pro" \ + "$TEST_PROMPT" 30 --approval-mode auto_edit + + [ "$status" -eq 2 ] + grep -q "gemini-3.5-flash" "$MODEL_RECORD" + grep -q "gemini-2.5-pro" "$MODEL_RECORD" +} + +@test "gemini-3.5 chain: non-rate-limit failure propagates immediately without trying next" { + _source_engine "gemini" + export STUB_ENGINE_EXIT_BY_MODEL="gemini-3.5-flash=1|gemini-2.5-pro=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="gemini-3.5-flash=unexpected internal error|gemini-2.5-pro=should not run" + + run _gemini_chain_invoke "gemini-3.5-flash,gemini-2.5-pro" \ + "$TEST_PROMPT" 30 --approval-mode auto_edit + + [ "$status" -eq 1 ] + grep -q "gemini-3.5-flash" "$MODEL_RECORD" + ! grep -q "gemini-2.5-pro" "$MODEL_RECORD" +} + +@test "gemini-3.5 chain: empty chain → config error (rc=1)" { + _source_engine "gemini" + run _gemini_chain_invoke " , , " "$TEST_PROMPT" 30 --approval-mode auto_edit + [ "$status" -eq 1 ] +} + +@test "gemini-3.5 chain: throttled warning phrase does NOT match is_rate_limited" { + _source_engine "gemini" + run is_rate_limited \ + "::warning::[gemini] model gemini-3.5-flash throttled (rc=1) — trying next in chain" + [ "$status" -ne 0 ] +} + +# ── End-to-end: run_writer uses flash chain on rate-limit ──────────────────── + +@test "gemini-3.5 writer: gemini-3.5-flash rate-limited → gemini-2.5-pro tried via GEMINI_FLASH_MODEL_CHAIN" { + _source_engine "gemini" + export STUB_ENGINE_EXIT_BY_MODEL="gemini-3.5-flash=1|gemini-2.5-pro=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="gemini-3.5-flash=too many requests (429)|gemini-2.5-pro=pro result" + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 0 ] + grep -q "gemini-3.5-flash" "$MODEL_RECORD" + grep -q "gemini-2.5-pro" "$MODEL_RECORD" +} + +@test "gemini-3.5 writer: custom GEMINI_FLASH_MODEL_CHAIN env override is honored" { + _source_engine "gemini" + export GEMINI_FLASH_MODEL_CHAIN="gemini-2.0-flash,gemini-3.5-flash" + export STUB_ENGINE_EXIT=0 + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 0 ] + head -1 "$MODEL_RECORD" | grep -q "gemini-2.0-flash" +} + +@test "gemini-3.5 writer: both models rate-limited → exit 2 (cross-provider fallback signal)" { + _source_engine "gemini" + export STUB_ENGINE_EXIT_BY_MODEL="gemini-3.5-flash=1|gemini-2.5-pro=1" + export STUB_ENGINE_RESPONSE_BY_MODEL="gemini-3.5-flash=quota exceeded|gemini-2.5-pro=quota exceeded" + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 2 ] +} + +# ── Non-gemini engines unaffected ──────────────────────────────────────────── + +@test "gemini-3.5: claude engine unchanged — no gemini model in any tier" { + _source_engine "claude" + [[ "$ENGINE_TRIAGE_MODEL" != *"gemini"* ]] + [[ "$ENGINE_DEEP_MODEL" != *"gemini"* ]] + [[ "$ENGINE_AUDIT_MODEL" != *"gemini"* ]] +} + +@test "gemini-3.5: copilot engine unchanged — all tiers still o4-mini" { + _source_engine "copilot" + [ "$ENGINE_AUDIT_MODEL" = "o4-mini" ] + [ "$ENGINE_TRIAGE_MODEL" = "o4-mini" ] + [ "$ENGINE_DEEP_MODEL" = "o4-mini" ] +} From c4ac980aa556745993cdd5a70584b54e99b506d0 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:55:41 +0000 Subject: [PATCH 2/3] fix(reviews): address review comments [skip ci-relay] --- scripts/engine.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index 4ebdbcfc2..fb09a4f13 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -420,6 +420,7 @@ _gemini_chain_invoke() { return 1 fi + local -a models local saved_ifs="$IFS" IFS=',' read -ra models <<< "$chain_csv" IFS="$saved_ifs" @@ -752,9 +753,9 @@ run_triage() { esac if [ "$rc" -eq 0 ]; then local _triage_used - if [ -n "${_CLAUDE_CHAIN_MODEL_USED:-}" ]; then + if [ "$REVIEW_ENGINE" = "claude" ] && [ -n "${_CLAUDE_CHAIN_MODEL_USED:-}" ]; then _triage_used="$_CLAUDE_CHAIN_MODEL_USED" - elif [ -n "${_GEMINI_CHAIN_MODEL_USED:-}" ]; then + elif [ "$REVIEW_ENGINE" = "gemini" ] && [ -n "${_GEMINI_CHAIN_MODEL_USED:-}" ]; then _triage_used="$_GEMINI_CHAIN_MODEL_USED" else _triage_used="$ENGINE_TRIAGE_MODEL" @@ -841,10 +842,14 @@ run_agentic() { # for quality tiers. Honor explicit model pin when it differs from the tier default. local _agentic_gemini_chain _gemini_tier_default="" case "$tier" in - deep|audit|single) _agentic_gemini_chain="${GEMINI_PRO_MODEL_CHAIN:-$model}" - _gemini_tier_default="${ENGINE_DEEP_MODEL:-}" ;; - *) _agentic_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$model}" - _gemini_tier_default="${ENGINE_ACTION_MODEL:-}" ;; + deep) _agentic_gemini_chain="${GEMINI_PRO_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_DEEP_MODEL:-}" ;; + audit) _agentic_gemini_chain="${GEMINI_PRO_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_AUDIT_MODEL:-}" ;; + single) _agentic_gemini_chain="${GEMINI_PRO_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_SINGLE_MODEL:-}" ;; + *) _agentic_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$model}" + _gemini_tier_default="${ENGINE_ACTION_MODEL:-}" ;; esac if [ -n "$_gemini_tier_default" ] && [ "$model" != "$_gemini_tier_default" ]; then _agentic_gemini_chain="$model" From 49e38472591500b175b0e11a3c7becaa5bc7147e Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:40:45 +0000 Subject: [PATCH 3/3] fix(reviews): address review comments [skip ci-relay] --- scripts/engine.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index b236e743d..d1f9d60dd 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -118,7 +118,7 @@ set_engine_config() { # not a typo for o1-mini or gpt-4o-mini. COPILOT_API_MODEL="${COPILOT_API_MODEL:-openai/o4-mini}" export COPILOT_API_MODEL - ENGINE_LABEL="triage: o4-mini → deep: o4-mini + duck: gemini-2.0-flash → audit: o4-mini (GitHub Models API)" + ENGINE_LABEL="triage: o4-mini → deep: o4-mini + duck: gemini-3.5-flash → audit: o4-mini (GitHub Models API)" ENGINE_SINGLE_LABEL="single-reviewer mode: o4-mini (GitHub Models API)" # Cross-engine rubber duck: use Gemini when Copilot is primary DUCK_ENGINE="gemini" @@ -745,8 +745,9 @@ run_triage() { gemini) local _triage_gemini_chain="${GEMINI_FLASH_MODEL_CHAIN:-$ENGINE_TRIAGE_MODEL}" if [ -n "$_tok_tmp" ]; then + _GEMINI_CHAIN_MODEL_USED="" _gemini_chain_invoke "$_triage_gemini_chain" "$prompt_file" "$TRIAGE_TIMEOUT_SEC" \ - --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + --approval-mode auto_edit > >(tee "$_tok_tmp") || rc=$? else _gemini_chain_invoke "$_triage_gemini_chain" "$prompt_file" "$TRIAGE_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$? @@ -865,8 +866,9 @@ run_agentic() { _agentic_gemini_chain="$model" fi if [ -n "$_tok_tmp" ]; then + _GEMINI_CHAIN_MODEL_USED="" _gemini_chain_invoke "$_agentic_gemini_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \ - --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + --approval-mode auto_edit > >(tee "$_tok_tmp") || rc=$? else _gemini_chain_invoke "$_agentic_gemini_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$? @@ -1131,8 +1133,9 @@ run_writer() { _writer_gemini_chain="$model" fi if [ -n "$_tmp" ]; then + _GEMINI_CHAIN_MODEL_USED="" _gemini_chain_invoke "$_writer_gemini_chain" "$prompt_file" "$ACTION_TIMEOUT_SEC" \ - --approval-mode auto_edit 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} + --approval-mode auto_edit > >(tee "$_tmp") 2>&1 || rc=$? else _gemini_chain_invoke "$_writer_gemini_chain" "$prompt_file" "$ACTION_TIMEOUT_SEC" \ --approval-mode auto_edit || rc=$?