Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 52 additions & 4 deletions scripts/engine.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,18 @@ DUCK_TIMEOUT_SEC="${DUCK_TIMEOUT_SEC:-300}"
RETRY_MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-2}" # total attempts including first
RETRY_BASE_DELAY_SEC="${RETRY_BASE_DELAY_SEC:-5}"

# Opt-in MCP (Model Context Protocol) config for the agentic review tiers.
# REVIEW_MCP_CONFIG — path to an MCP-servers JSON config file. When unset/empty
# or not readable, the claude invocations gain NO new flags and behavior is
# byte-for-byte unchanged for every existing adopter.
# REVIEW_MCP_ALLOWED_TOOLS — comma-separated MCP tool names (e.g.
# "mcp__context7__*") merged into the per-tier --allowed-tools list so the
# configured MCP tools are actually permitted. Default empty.
# Only the deep (run_agentic) and rubber-duck (run_duck) claude tiers receive
# this — the triage tier stays fast/restricted (--disallowed-tools only).
REVIEW_MCP_CONFIG="${REVIEW_MCP_CONFIG:-}"
REVIEW_MCP_ALLOWED_TOOLS="${REVIEW_MCP_ALLOWED_TOOLS:-}"

set_engine_config() {
case "$REVIEW_ENGINE" in
claude)
Expand Down Expand Up @@ -582,6 +594,34 @@ _record_engine_tokens() {
"$input_tokens" "$cache_read_tokens" "$output_tokens" "$context" "$cache_write_tokens" || true
}

# _mcp_review_flags <base_allowed_tools>
# Threads the opt-in MCP config into the claude agentic/duck tiers. Populates two
# globals for the caller to splice into the claude --print invocation:
# _MCP_ALLOWED_TOOLS — base_allowed_tools with REVIEW_MCP_ALLOWED_TOOLS merged
# in (comma-separated). Equals base when no MCP tools set.
# _MCP_FLAGS — array: (--mcp-config <file> --strict-mcp-config) when
# REVIEW_MCP_CONFIG points to a readable file; empty array
# otherwise.
# Verified against @anthropic-ai/claude-code 2.1.138 (claude --help):
# --mcp-config <configs...> Load MCP servers from JSON files or strings
# --strict-mcp-config Only use MCP servers from --mcp-config, ignoring
# all other MCP configurations
# When REVIEW_MCP_CONFIG is unset/empty/unreadable, _MCP_FLAGS stays empty and
# _MCP_ALLOWED_TOOLS == base_allowed_tools → no new flags, behavior unchanged.
_mcp_review_flags() {
local base="$1"
_MCP_FLAGS=()
_MCP_ALLOWED_TOOLS="$base"
if [ -n "${REVIEW_MCP_CONFIG:-}" ] && [ -f "${REVIEW_MCP_CONFIG}" ] && [ -r "${REVIEW_MCP_CONFIG}" ]; then
_MCP_FLAGS=(--mcp-config "$REVIEW_MCP_CONFIG" --strict-mcp-config)
if [ -n "${REVIEW_MCP_ALLOWED_TOOLS:-}" ]; then
_MCP_ALLOWED_TOOLS="${base},${REVIEW_MCP_ALLOWED_TOOLS}"
fi
elif [ -n "${REVIEW_MCP_CONFIG:-}" ]; then
echo " [mcp] REVIEW_MCP_CONFIG='$REVIEW_MCP_CONFIG' is not a readable file — skipping MCP flags" >&2
fi
}

# run_triage <prompt_file>
# Used by: review-one-pr.sh only (not the dev-lead writer pipeline).
# No-tool mode. The prompt file already has all PR context inlined by the
Expand Down Expand Up @@ -706,15 +746,19 @@ run_agentic() {
if [ -n "$_tier_default" ] && [ "$model" != "$_tier_default" ]; then
_agentic_chain="$model"
fi
# Thread the opt-in MCP config (no-op when REVIEW_MCP_CONFIG is unset).
_mcp_review_flags "Bash,Read,Grep,Glob"
if [ -n "$_tok_tmp" ]; then
_claude_chain_invoke "$_agentic_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \
--permission-mode acceptEdits \
--allowed-tools "Bash,Read,Grep,Glob" \
--allowed-tools "$_MCP_ALLOWED_TOOLS" \
${_MCP_FLAGS[@]+"${_MCP_FLAGS[@]}"} \
| tee "$_tok_tmp" || rc=${PIPESTATUS[0]}
else
_claude_chain_invoke "$_agentic_chain" "$prompt_file" "$DEEP_TIMEOUT_SEC" \
--permission-mode acceptEdits \
--allowed-tools "Bash,Read,Grep,Glob" \
--allowed-tools "$_MCP_ALLOWED_TOOLS" \
${_MCP_FLAGS[@]+"${_MCP_FLAGS[@]}"} \
|| rc=$?
fi
Comment thread
don-petry marked this conversation as resolved.
Resolved
;;
Expand Down Expand Up @@ -808,19 +852,23 @@ run_duck() {
claude)
unset COPILOT_GITHUB_TOKEN 2>/dev/null || true
unset GOOGLE_API_KEY 2>/dev/null || true
# Thread the opt-in MCP config (no-op when REVIEW_MCP_CONFIG is unset).
_mcp_review_flags "Bash,Read,Grep,Glob"
# Route through the chain helper so real token usage (incl. cache) is captured
# when logging is on; a single-model "chain" preserves prior behaviour.
if [ -n "$_tok_tmp" ]; then
_claude_chain_invoke "$model" "$prompt_file" "$DUCK_TIMEOUT_SEC" \
--permission-mode acceptEdits \
--allowed-tools "Bash,Read,Grep,Glob" \
--allowed-tools "$_MCP_ALLOWED_TOOLS" \
--max-turns 25 \
${_MCP_FLAGS[@]+"${_MCP_FLAGS[@]}"} \
| tee "$_tok_tmp" || rc=${PIPESTATUS[0]}
else
_claude_chain_invoke "$model" "$prompt_file" "$DUCK_TIMEOUT_SEC" \
--permission-mode acceptEdits \
--allowed-tools "Bash,Read,Grep,Glob" \
--allowed-tools "$_MCP_ALLOWED_TOOLS" \
--max-turns 25 \
${_MCP_FLAGS[@]+"${_MCP_FLAGS[@]}"} \
|| rc=$?
fi
Comment thread
don-petry marked this conversation as resolved.
Resolved
;;
Expand Down
5 changes: 5 additions & 0 deletions tests/dev-lead/fixtures/engines/stub-claude
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@
# STUB_CLAUDE_USAGE — "input cache_read cache_write output" emitted as the
# .usage block when --output-format json is requested.
# Default "100 20 30 40".
# STUB_ENGINE_RECORD_ARGS — if set to a writable path, the full argv of every
# invocation is appended as one line for inspection.
if [ -n "${STUB_ENGINE_RECORD_ARGS:-}" ]; then
printf '%s\n' "$*" >> "$STUB_ENGINE_RECORD_ARGS" 2>/dev/null || true
fi
model=""
output_format="text"
while [[ $# -gt 0 ]]; do
Expand Down
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_engine_chain.bats
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# ── _claude_chain_invoke unit tests ───────────────────────────────────────────
Expand Down
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_engine_fallback.bats
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# Helper: create a stub that always returns a given exit code
Expand Down
167 changes: 167 additions & 0 deletions tests/dev-lead/unit/test_engine_mcp.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
#!/usr/bin/env bats
# Unit tests for engine.sh — opt-in MCP config threading (issue #677).
#
# Verifies that REVIEW_MCP_CONFIG / REVIEW_MCP_ALLOWED_TOOLS thread the MCP
# flags (--mcp-config <file> --strict-mcp-config) and merged --allowed-tools
# into ONLY the claude branch of the agentic (run_agentic) and rubber-duck
# (run_duck) tiers — never the triage tier — and that the default (knob unset)
# behavior is byte-for-byte unchanged.

SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)"
ENGINE_SCRIPT="$SCRIPT_DIR/scripts/engine.sh"
STUB_ENGINES_DIR="$SCRIPT_DIR/tests/dev-lead/fixtures/engines"

setup() {
export GITHUB_ENV="$(mktemp)"
export GITHUB_OUTPUT="$(mktemp)"

unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN
unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN
unset REVIEW_MCP_CONFIG REVIEW_MCP_ALLOWED_TOOLS

STUB_BIN_DIR="$(mktemp -d)"
cp "$STUB_ENGINES_DIR/stub-claude" "$STUB_BIN_DIR/claude"
cp "$STUB_ENGINES_DIR/stub-gemini" "$STUB_BIN_DIR/gemini"
chmod +x "$STUB_BIN_DIR/claude" "$STUB_BIN_DIR/gemini"
export PATH="$STUB_BIN_DIR:$PATH"
export STUB_BIN_DIR

TEST_PROMPT="$(mktemp)"
echo "test prompt content" > "$TEST_PROMPT"
export TEST_PROMPT

ARGS_RECORD="$(mktemp)"
export STUB_ENGINE_RECORD_ARGS="$ARGS_RECORD"

# A readable MCP config file the helper can point at.
MCP_CONFIG_FILE="$(mktemp)"
echo '{"mcpServers":{}}' > "$MCP_CONFIG_FILE"
export MCP_CONFIG_FILE

export DEV_LEAD_DRY_RUN="false"
export STUB_ENGINE_EXIT=0
}

teardown() {
rm -f "$GITHUB_ENV" "$GITHUB_OUTPUT" "$TEST_PROMPT" "$ARGS_RECORD" "$MCP_CONFIG_FILE"
rm -rf "$STUB_BIN_DIR"
unset STUB_ENGINE_RECORD_ARGS STUB_ENGINE_EXIT
unset REVIEW_MCP_CONFIG REVIEW_MCP_ALLOWED_TOOLS
}

_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT"
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

# ── Default (knob unset): behavior unchanged ──────────────────────────────────

@test "agentic: no MCP knob → no MCP flags, default allowed-tools unchanged" {
_source_engine "claude"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
! grep -q -- "--strict-mcp-config" "$ARGS_RECORD"
grep -q -- "--allowed-tools Bash,Read,Grep,Glob" "$ARGS_RECORD"
}

@test "agentic: empty REVIEW_MCP_CONFIG → no MCP flags" {
_source_engine "claude"
export REVIEW_MCP_CONFIG=""
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
}

@test "agentic: REVIEW_MCP_CONFIG pointing to unreadable file → no MCP flags" {
_source_engine "claude"
local unreadable_file
unreadable_file="$(mktemp)"
chmod 000 "$unreadable_file"
export REVIEW_MCP_CONFIG="$unreadable_file"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
chmod 600 "$unreadable_file"
rm -f "$unreadable_file"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
grep -q -- "--allowed-tools Bash,Read,Grep,Glob" "$ARGS_RECORD"
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

@test "agentic: REVIEW_MCP_CONFIG pointing to a directory → no MCP flags" {
_source_engine "claude"
export REVIEW_MCP_CONFIG="$(mktemp -d)"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
rmdir "$REVIEW_MCP_CONFIG"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
grep -q -- "--allowed-tools Bash,Read,Grep,Glob" "$ARGS_RECORD"
}

# ── Knob set: MCP flags threaded into agentic ────────────────────────────────

@test "agentic: REVIEW_MCP_CONFIG set → MCP flags appended to claude call" {
_source_engine "claude"
export REVIEW_MCP_CONFIG="$MCP_CONFIG_FILE"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
[ "$status" -eq 0 ]
grep -q -- "--mcp-config $MCP_CONFIG_FILE" "$ARGS_RECORD"
grep -q -- "--strict-mcp-config" "$ARGS_RECORD"
}

@test "agentic: REVIEW_MCP_ALLOWED_TOOLS merged into --allowed-tools" {
_source_engine "claude"
export REVIEW_MCP_CONFIG="$MCP_CONFIG_FILE"
export REVIEW_MCP_ALLOWED_TOOLS="mcp__context7__*"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
[ "$status" -eq 0 ]
grep -q -- "--allowed-tools Bash,Read,Grep,Glob,mcp__context7__\*" "$ARGS_RECORD"
}

@test "agentic: MCP config set but no allowed-tools knob → base allowed-tools kept" {
_source_engine "claude"
export REVIEW_MCP_CONFIG="$MCP_CONFIG_FILE"
run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep"
[ "$status" -eq 0 ]
grep -q -- "--allowed-tools Bash,Read,Grep,Glob" "$ARGS_RECORD"
! grep -q -- "--allowed-tools Bash,Read,Grep,Glob," "$ARGS_RECORD"
}

# ── Knob set: MCP flags threaded into duck ───────────────────────────────────

@test "duck: REVIEW_MCP_CONFIG set → MCP flags appended to claude duck call" {
_source_engine "claude"
export DUCK_ENGINE="claude"
export REVIEW_MCP_CONFIG="$MCP_CONFIG_FILE"
export REVIEW_MCP_ALLOWED_TOOLS="mcp__context7__*"
run run_duck "$TEST_PROMPT" "claude-sonnet-4-6"
[ "$status" -eq 0 ]
grep -q -- "--mcp-config $MCP_CONFIG_FILE" "$ARGS_RECORD"
grep -q -- "--strict-mcp-config" "$ARGS_RECORD"
grep -q -- "--allowed-tools Bash,Read,Grep,Glob,mcp__context7__\*" "$ARGS_RECORD"
# The duck tier's existing --max-turns flag must still be present.
grep -q -- "--max-turns 25" "$ARGS_RECORD"
}

@test "duck: no MCP knob → no MCP flags on claude duck call" {
_source_engine "claude"
export DUCK_ENGINE="claude"
run run_duck "$TEST_PROMPT" "claude-sonnet-4-6"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
grep -q -- "--allowed-tools Bash,Read,Grep,Glob" "$ARGS_RECORD"
}

# ── Triage tier: never gets MCP flags ────────────────────────────────────────

@test "triage: MCP knob set → triage still uses --disallowed-tools only, no MCP" {
_source_engine "claude"
export REVIEW_MCP_CONFIG="$MCP_CONFIG_FILE"
export REVIEW_MCP_ALLOWED_TOOLS="mcp__context7__*"
run run_triage "$TEST_PROMPT"
[ "$status" -eq 0 ]
! grep -q -- "--mcp-config" "$ARGS_RECORD"
! grep -q -- "--strict-mcp-config" "$ARGS_RECORD"
! grep -q -- "--allowed-tools" "$ARGS_RECORD"
grep -q -- "--disallowed-tools" "$ARGS_RECORD"
}
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_engine_new_models.bats
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# ── Tier default model assignments ───────────────────────────────────────────
Expand Down
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_engine_writer.bats
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# ── run_writer tests ──────────────────────────────────────────────────────────
Expand Down
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_model_dispatch.bats
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# ── triage-tier intents (haiku / lightest model) ──────────────────────────────
Expand Down
2 changes: 1 addition & 1 deletion tests/dev-lead/unit/test_provider_headroom.bats
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ teardown() {
_source_engine() {
local engine="${1:-claude}"
export REVIEW_ENGINE="$engine"
source "$ENGINE_SCRIPT" 2>/dev/null || true
source "$ENGINE_SCRIPT"
}

# Helper: install a curl stub that outputs given headers and exits with given code
Expand Down
Loading