diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 8b4777b55..f3c4bd874 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -74,6 +74,90 @@ jobs: GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} run: bash scripts/fleet_monitor.sh + - name: Summarize token usage by workflow + env: + GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} + REPO: petry-projects/.github-private + LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }} + run: | + set -euo pipefail + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT + jsonl_dir="$workdir/jsonl" + artifacts_file="$workdir/token-artifacts.base64" + mkdir -p "$jsonl_dir" + + cutoff_epoch="$(date -u -d "-${LOOKBACK_DAYS} days" +%s)" + artifact_count=0 + + gh api "repos/$REPO/actions/artifacts" --paginate \ + --jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' \ + > "$artifacts_file" + + while IFS= read -r encoded; do + [ -n "$encoded" ] || continue + item="$(printf '%s' "$encoded" | base64 -d)" + id="$(printf '%s' "$item" | jq -r '.id')" + created_at="$(printf '%s' "$item" | jq -r '.created_at')" + expired="$(printf '%s' "$item" | jq -r '.expired')" + created_epoch="$(date -u -d "$created_at" +%s)" + + if [ "$expired" = "true" ] || [ "$created_epoch" -lt "$cutoff_epoch" ]; then + continue + fi + + zip_file="$workdir/artifact-$id.zip" + unzip_dir="$workdir/unzip-$id" + mkdir -p "$unzip_dir" + gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file" + unzip -q -o "$zip_file" -d "$unzip_dir" + + found_any=false + while IFS= read -r f; do + found_any=true + cp "$f" "$jsonl_dir/${id}-$(basename "$f")" + done < <(find "$unzip_dir" -type f -name '*.jsonl' -print) + + if [ "$found_any" = true ]; then + artifact_count=$((artifact_count + 1)) + fi + done < "$artifacts_file" + + if ! find "$jsonl_dir" -type f -name '*.jsonl' | grep -q .; then + { + echo "## Token usage (${LOOKBACK_DAYS} day lookback)" + echo + echo "No token-usage artifacts found in the lookback window." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + aggregate="$workdir/token-usage-summary.json" + jq -s ' + sort_by(.workflow) | + group_by(.workflow) | + map({ + workflow: (.[0].workflow // "unknown"), + calls: length, + input_tokens: (map(.input_tokens // 0) | add), + cache_read_tokens: (map(.cache_read_tokens // 0) | add), + output_tokens: (map(.output_tokens // 0) | add), + et: (map(.et // 0) | add) + }) | + sort_by(.et) | reverse + ' "$jsonl_dir"/*.jsonl > "$aggregate" + + { + echo "## Token usage (${LOOKBACK_DAYS} day lookback)" + echo + echo "Processed ${artifact_count} token-usage artifact(s)." + echo + echo "| Workflow | Calls | Input Tokens | Cache Read Tokens | Output Tokens | Total ET |" + echo "|---|---:|---:|---:|---:|---:|" + jq -r '.[] | "| \(.workflow) | \(.calls) | \(.input_tokens) | \(.cache_read_tokens) | \(.output_tokens) | \(.et | tostring) |"' "$aggregate" + } >> "$GITHUB_STEP_SUMMARY" + - name: Create issue with findings if: env.HAS_FAILURES == 'true' && hashFiles('fleet_monitor_report.md') != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 88367ebe2..1186dee59 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -250,7 +250,7 @@ jobs: # Triggered when a trusted bot approves a PR. Skips engine install and # pre-flight — just checks reviewDecision and enables auto-merge if APPROVED. - name: Enable auto-merge on bot approval - if: env.INTENT_TYPE == 'enable-auto-merge' + if: env.INTENT_TYPE == 'enable-auto-merge' && env.INTENT_PR_NUMBER != '' env: PR_NUMBER: ${{ env.INTENT_PR_NUMBER }} HEAD_SHA: ${{ env.INTENT_HEAD_SHA }} diff --git a/docs/dev-lead/spec.md b/docs/dev-lead/spec.md index 422a35f59..60c80023a 100644 --- a/docs/dev-lead/spec.md +++ b/docs/dev-lead/spec.md @@ -582,9 +582,9 @@ Per-engine model names are controlled by the same env vars used by the PR Review | Variable | Claude default | Gemini default | Copilot default | |---|---|---|---| | `ENGINE_TRIAGE_MODEL` | `claude-haiku-4-5-20251001` | `gemini-2.0-flash` | `o4-mini` | -| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` | -| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` | -| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-1.5-pro` | `o4-mini` | +| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` | +| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` | +| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-2.5-pro` | `o4-mini` | --- diff --git a/scripts/engine.sh b/scripts/engine.sh index 60810d33b..47ec9f997 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -51,12 +51,12 @@ set_engine_config() { ;; gemini) ENGINE_TRIAGE_MODEL="gemini-2.0-flash" - ENGINE_DEEP_MODEL="gemini-1.5-pro" - ENGINE_AUDIT_MODEL="gemini-1.5-pro" - ENGINE_ACTION_MODEL="gemini-1.5-pro" - ENGINE_SINGLE_MODEL="gemini-1.5-pro" - ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-1.5-pro + duck: sonnet 4.6 → audit: gemini-1.5-pro" - ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-1.5-pro" + ENGINE_DEEP_MODEL="gemini-2.5-pro" + ENGINE_AUDIT_MODEL="gemini-2.5-pro" + ENGINE_ACTION_MODEL="gemini-2.5-pro" + ENGINE_SINGLE_MODEL="gemini-2.5-pro" + ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-2.5-pro + duck: sonnet 4.6 → audit: gemini-2.5-pro" + ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-2.5-pro" # Cross-engine rubber duck: use Claude for diversity DUCK_ENGINE="claude" DUCK_MODEL="claude-sonnet-4-6" @@ -536,7 +536,7 @@ run_writer() { --model "$model" \ --permission-mode acceptEdits \ --allowed-tools "Bash,Read,Write,Edit,Grep,Glob,WebFetch" \ - < "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]} + < "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else timeout "$ACTION_TIMEOUT_SEC" claude --print \ --model "$model" \ @@ -551,7 +551,7 @@ run_writer() { --model "$model" \ --approval-mode auto_edit \ --output-format text \ - < "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]} + < "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else timeout "$ACTION_TIMEOUT_SEC" gemini --prompt "" \ --model "$model" \ @@ -563,7 +563,7 @@ run_writer() { copilot) # Self-sufficient write support via gh copilot --yolo if [ -n "$_tmp" ]; then - copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo | tee "$_tmp" || rc=${PIPESTATUS[0]} + copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo || rc=$? fi @@ -595,6 +595,11 @@ run_writer_with_fallback() { done for engine in "${engines[@]}"; do + if [ "$engine" = "copilot" ] && [[ "${COPILOT_GITHUB_TOKEN:-}" == ghp_* ]]; then + echo "::warning::Skipping copilot fallback: classic PAT in COPILOT_GITHUB_TOKEN is unsupported" >&2 + continue + fi + local saved="$REVIEW_ENGINE" export REVIEW_ENGINE="$engine" # Re-evaluate model names for the new engine diff --git a/tests/dev-lead/unit/test_engine_writer.bats b/tests/dev-lead/unit/test_engine_writer.bats index 29067b3a9..663c62f9f 100644 --- a/tests/dev-lead/unit/test_engine_writer.bats +++ b/tests/dev-lead/unit/test_engine_writer.bats @@ -157,6 +157,21 @@ STUB [ "$status" -eq 2 ] } +@test "writer: run_writer returns exit 2 when gemini writes rate-limit text to stderr" { + _source_engine "gemini" + export DEV_LEAD_DRY_RUN=false + cat > "$STUB_BIN_DIR/gemini" << 'STUB' +#!/usr/bin/env bash +echo "quota exceeded for today" >&2 +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/gemini" + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 2 ] +} + @test "writer: run_writer returns exit 1 (not 2) for non-rate-limit failure" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false @@ -214,6 +229,34 @@ GHEOF [ "$status" -eq 2 ] } +@test "writer: run_writer_with_fallback skips copilot when token is classic PAT" { + _source_engine "claude" + export DEV_LEAD_DRY_RUN=false + export COPILOT_GITHUB_TOKEN="ghp_classic_token" + + # Claude and Gemini both rate-limit. + for engine in claude gemini; do + cat > "$STUB_BIN_DIR/$engine" << 'STUB' +#!/usr/bin/env bash +echo "quota exceeded" >&2 +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/$engine" + done + + # If copilot is invoked, this stub would fail the test via status 1 path. + cat > "$STUB_BIN_DIR/gh" << 'GHEOF' +#!/usr/bin/env bash +echo "copilot should have been skipped" >&2 +exit 1 +GHEOF + chmod +x "$STUB_BIN_DIR/gh" + + run run_writer_with_fallback "$TEST_PROMPT" + + [ "$status" -eq 2 ] +} + @test "writer: run_writer_with_fallback succeeds on second engine if first rate-limited" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false