From f2af5783532770ebe720d83459d9012e337ef952 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 07:14:06 -0500 Subject: [PATCH 1/8] feat(fleet-monitor): add token usage summary by workflow Download recent token-usage artifacts, aggregate JSONL records by workflow, and publish token totals and ET in the workflow step summary. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 8b4777b55..9bb34365b 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -74,6 +74,87 @@ jobs: GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} run: bash scripts/fleet_monitor.sh + - name: Summarize token usage by workflow + env: + GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} + REPO: ${{ github.repository }} + LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }} + run: | + set -euo pipefail + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT + jsonl_dir="$workdir/jsonl" + mkdir -p "$jsonl_dir" + + cutoff_epoch="$(date -u -d "-${LOOKBACK_DAYS} days" +%s)" + artifact_count=0 + + while IFS= read -r encoded; do + [ -n "$encoded" ] || continue + item="$(printf '%s' "$encoded" | base64 -d)" + id="$(printf '%s' "$item" | jq -r '.id')" + created_at="$(printf '%s' "$item" | jq -r '.created_at')" + expired="$(printf '%s' "$item" | jq -r '.expired')" + created_epoch="$(date -u -d "$created_at" +%s)" + + if [ "$expired" = "true" ] || [ "$created_epoch" -lt "$cutoff_epoch" ]; then + continue + fi + + zip_file="$workdir/artifact-$id.zip" + unzip_dir="$workdir/unzip-$id" + mkdir -p "$unzip_dir" + gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file" + unzip -q -o "$zip_file" -d "$unzip_dir" + + found_any=false + while IFS= read -r f; do + found_any=true + cp "$f" "$jsonl_dir/" + done < <(find "$unzip_dir" -type f -name '*.jsonl' -print) + + if [ "$found_any" = true ]; then + artifact_count=$((artifact_count + 1)) + fi + done < <( + gh api "repos/$REPO/actions/artifacts" --paginate \ + --jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' + ) + + if ! find "$jsonl_dir" -type f -name '*.jsonl' | grep -q .; then + { + echo "## Token usage (${LOOKBACK_DAYS} day lookback)" + echo + echo "No token-usage artifacts found in the lookback window." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + aggregate="$workdir/token-usage-summary.json" + jq -s ' + group_by(.workflow) | + map({ + workflow: (.[0].workflow // "unknown"), + calls: length, + input_tokens: (map(.input_tokens // 0) | add), + cache_read_tokens: (map(.cache_read_tokens // 0) | add), + output_tokens: (map(.output_tokens // 0) | add), + et: (map(.et // 0) | add) + }) | + sort_by(.et) | reverse + ' "$jsonl_dir"/*.jsonl > "$aggregate" + + { + echo "## Token usage (${LOOKBACK_DAYS} day lookback)" + echo + echo "Processed ${artifact_count} token-usage artifact(s)." + echo + echo "| Workflow | Calls | Input Tokens | Cache Read Tokens | Output Tokens | Total ET |" + echo "|---|---:|---:|---:|---:|---:|" + jq -r '.[] | "| \(.workflow) | \(.calls) | \(.input_tokens) | \(.cache_read_tokens) | \(.output_tokens) | \(.et | tostring) |"' "$aggregate" + } >> "$GITHUB_STEP_SUMMARY" + - name: Create issue with findings if: env.HAS_FAILURES == 'true' && hashFiles('fleet_monitor_report.md') != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 From 8683edd31db06a3edec935f851d4e7c33d18e5cf Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 07:39:17 -0500 Subject: [PATCH 2/8] fix fleet monitor token summary aggregation Avoid JSONL filename collisions by staging per-artifact directories and sort by workflow before jq group_by to ensure correct totals. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 9bb34365b..7c28fc2e5 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -104,14 +104,16 @@ jobs: zip_file="$workdir/artifact-$id.zip" unzip_dir="$workdir/unzip-$id" + artifact_jsonl_dir="$jsonl_dir/$id" mkdir -p "$unzip_dir" + mkdir -p "$artifact_jsonl_dir" gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file" unzip -q -o "$zip_file" -d "$unzip_dir" found_any=false while IFS= read -r f; do found_any=true - cp "$f" "$jsonl_dir/" + cp "$f" "$artifact_jsonl_dir/" done < <(find "$unzip_dir" -type f -name '*.jsonl' -print) if [ "$found_any" = true ]; then @@ -133,6 +135,7 @@ jobs: aggregate="$workdir/token-usage-summary.json" jq -s ' + sort_by(.workflow) | group_by(.workflow) | map({ workflow: (.[0].workflow // "unknown"), From 5fa36d2a32306045bf4cee66270c10aaf039f150 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 07:52:42 -0500 Subject: [PATCH 3/8] fix token artifact aggregation file matching Prefix copied JSONL filenames with artifact IDs to avoid collisions while keeping flat aggregation input for jq. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 7c28fc2e5..2873b1926 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -104,16 +104,14 @@ jobs: zip_file="$workdir/artifact-$id.zip" unzip_dir="$workdir/unzip-$id" - artifact_jsonl_dir="$jsonl_dir/$id" mkdir -p "$unzip_dir" - mkdir -p "$artifact_jsonl_dir" gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file" unzip -q -o "$zip_file" -d "$unzip_dir" found_any=false while IFS= read -r f; do found_any=true - cp "$f" "$artifact_jsonl_dir/" + cp "$f" "$jsonl_dir/${id}-$(basename "$f")" done < <(find "$unzip_dir" -type f -name '*.jsonl' -print) if [ "$found_any" = true ]; then From 33259eb277ded367df3b08a0df455824a2c11e03 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 08:08:12 -0500 Subject: [PATCH 4/8] fix fleet monitor token artifact collection robustness Use explicit artifact-list capture (no process substitution masking), and pin artifact source repo to .github-private for workflow_call correctness. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 2873b1926..f3c4bd874 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -77,7 +77,7 @@ jobs: - name: Summarize token usage by workflow env: GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} - REPO: ${{ github.repository }} + REPO: petry-projects/.github-private LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }} run: | set -euo pipefail @@ -85,11 +85,16 @@ jobs: workdir="$(mktemp -d)" trap 'rm -rf "$workdir"' EXIT jsonl_dir="$workdir/jsonl" + artifacts_file="$workdir/token-artifacts.base64" mkdir -p "$jsonl_dir" cutoff_epoch="$(date -u -d "-${LOOKBACK_DAYS} days" +%s)" artifact_count=0 + gh api "repos/$REPO/actions/artifacts" --paginate \ + --jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' \ + > "$artifacts_file" + while IFS= read -r encoded; do [ -n "$encoded" ] || continue item="$(printf '%s' "$encoded" | base64 -d)" @@ -117,10 +122,7 @@ jobs: if [ "$found_any" = true ]; then artifact_count=$((artifact_count + 1)) fi - done < <( - gh api "repos/$REPO/actions/artifacts" --paginate \ - --jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' - ) + done < "$artifacts_file" if ! find "$jsonl_dir" -type f -name '*.jsonl' | grep -q .; then { From 1c972fa73aa97a3e40bb576eb0600080363054ad Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 08:22:44 -0500 Subject: [PATCH 5/8] fix gemini fallback model for dev-lead Switch gemini deep/audit/action/single models to gemini-2.5-pro to avoid ModelNotFound failures during rate-limit fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/dev-lead/spec.md | 6 +++--- scripts/engine.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/dev-lead/spec.md b/docs/dev-lead/spec.md index 422a35f59..60c80023a 100644 --- a/docs/dev-lead/spec.md +++ b/docs/dev-lead/spec.md @@ -582,9 +582,9 @@ Per-engine model names are controlled by the same env vars used by the PR Review | Variable | Claude default | Gemini default | Copilot default | |---|---|---|---| | `ENGINE_TRIAGE_MODEL` | `claude-haiku-4-5-20251001` | `gemini-2.0-flash` | `o4-mini` | -| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` | -| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` | -| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-1.5-pro` | `o4-mini` | +| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` | +| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` | +| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-2.5-pro` | `o4-mini` | --- diff --git a/scripts/engine.sh b/scripts/engine.sh index 60810d33b..1e3cc5664 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -51,12 +51,12 @@ set_engine_config() { ;; gemini) ENGINE_TRIAGE_MODEL="gemini-2.0-flash" - ENGINE_DEEP_MODEL="gemini-1.5-pro" - ENGINE_AUDIT_MODEL="gemini-1.5-pro" - ENGINE_ACTION_MODEL="gemini-1.5-pro" - ENGINE_SINGLE_MODEL="gemini-1.5-pro" - ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-1.5-pro + duck: sonnet 4.6 → audit: gemini-1.5-pro" - ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-1.5-pro" + ENGINE_DEEP_MODEL="gemini-2.5-pro" + ENGINE_AUDIT_MODEL="gemini-2.5-pro" + ENGINE_ACTION_MODEL="gemini-2.5-pro" + ENGINE_SINGLE_MODEL="gemini-2.5-pro" + ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-2.5-pro + duck: sonnet 4.6 → audit: gemini-2.5-pro" + ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-2.5-pro" # Cross-engine rubber duck: use Claude for diversity DUCK_ENGINE="claude" DUCK_MODEL="claude-sonnet-4-6" From 6aaee066cfd87ab4a660f514eb943e923942fc2a Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 15:29:10 -0500 Subject: [PATCH 6/8] fix dev-lead auto-merge guard for skip intent Require INTENT_PR_NUMBER for enable-auto-merge step so self-review skip events don't invoke fix-reviews with empty PR context. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/dev-lead-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 88367ebe2..1186dee59 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -250,7 +250,7 @@ jobs: # Triggered when a trusted bot approves a PR. Skips engine install and # pre-flight — just checks reviewDecision and enables auto-merge if APPROVED. - name: Enable auto-merge on bot approval - if: env.INTENT_TYPE == 'enable-auto-merge' + if: env.INTENT_TYPE == 'enable-auto-merge' && env.INTENT_PR_NUMBER != '' env: PR_NUMBER: ${{ env.INTENT_PR_NUMBER }} HEAD_SHA: ${{ env.INTENT_HEAD_SHA }} From 5422b3049ea6372febfef210a0a3033a44cbb920 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 15:32:35 -0500 Subject: [PATCH 7/8] fix fallback detection for stderr rate-limit errors Capture stderr in run_writer token logs so quota/rate-limit responses from Gemini and Copilot are classified as retryable (exit 2) and can fall through to next engine; add regression test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/engine.sh | 6 +++--- tests/dev-lead/unit/test_engine_writer.bats | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index 1e3cc5664..2eea7a777 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -536,7 +536,7 @@ run_writer() { --model "$model" \ --permission-mode acceptEdits \ --allowed-tools "Bash,Read,Write,Edit,Grep,Glob,WebFetch" \ - < "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]} + < "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else timeout "$ACTION_TIMEOUT_SEC" claude --print \ --model "$model" \ @@ -551,7 +551,7 @@ run_writer() { --model "$model" \ --approval-mode auto_edit \ --output-format text \ - < "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]} + < "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else timeout "$ACTION_TIMEOUT_SEC" gemini --prompt "" \ --model "$model" \ @@ -563,7 +563,7 @@ run_writer() { copilot) # Self-sufficient write support via gh copilot --yolo if [ -n "$_tmp" ]; then - copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo | tee "$_tmp" || rc=${PIPESTATUS[0]} + copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo || rc=$? fi diff --git a/tests/dev-lead/unit/test_engine_writer.bats b/tests/dev-lead/unit/test_engine_writer.bats index 29067b3a9..668f67a78 100644 --- a/tests/dev-lead/unit/test_engine_writer.bats +++ b/tests/dev-lead/unit/test_engine_writer.bats @@ -157,6 +157,21 @@ STUB [ "$status" -eq 2 ] } +@test "writer: run_writer returns exit 2 when gemini writes rate-limit text to stderr" { + _source_engine "gemini" + export DEV_LEAD_DRY_RUN=false + cat > "$STUB_BIN_DIR/gemini" << 'STUB' +#!/usr/bin/env bash +echo "quota exceeded for today" >&2 +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/gemini" + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 2 ] +} + @test "writer: run_writer returns exit 1 (not 2) for non-rate-limit failure" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false From 6ec277a65c9514d8cf0f5bcb38b0769c1c86095c Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 21 May 2026 15:35:07 -0500 Subject: [PATCH 8/8] fix fallback when copilot token is classic PAT Skip copilot fallback when COPILOT_GITHUB_TOKEN is a classic ghp token (unsupported by Copilot CLI), preserving rate-limit semantics and preventing hard failures after claude/gemini exhaustion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/engine.sh | 5 ++++ tests/dev-lead/unit/test_engine_writer.bats | 28 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/scripts/engine.sh b/scripts/engine.sh index 2eea7a777..47ec9f997 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -595,6 +595,11 @@ run_writer_with_fallback() { done for engine in "${engines[@]}"; do + if [ "$engine" = "copilot" ] && [[ "${COPILOT_GITHUB_TOKEN:-}" == ghp_* ]]; then + echo "::warning::Skipping copilot fallback: classic PAT in COPILOT_GITHUB_TOKEN is unsupported" >&2 + continue + fi + local saved="$REVIEW_ENGINE" export REVIEW_ENGINE="$engine" # Re-evaluate model names for the new engine diff --git a/tests/dev-lead/unit/test_engine_writer.bats b/tests/dev-lead/unit/test_engine_writer.bats index 668f67a78..663c62f9f 100644 --- a/tests/dev-lead/unit/test_engine_writer.bats +++ b/tests/dev-lead/unit/test_engine_writer.bats @@ -229,6 +229,34 @@ GHEOF [ "$status" -eq 2 ] } +@test "writer: run_writer_with_fallback skips copilot when token is classic PAT" { + _source_engine "claude" + export DEV_LEAD_DRY_RUN=false + export COPILOT_GITHUB_TOKEN="ghp_classic_token" + + # Claude and Gemini both rate-limit. + for engine in claude gemini; do + cat > "$STUB_BIN_DIR/$engine" << 'STUB' +#!/usr/bin/env bash +echo "quota exceeded" >&2 +exit 1 +STUB + chmod +x "$STUB_BIN_DIR/$engine" + done + + # If copilot is invoked, this stub would fail the test via status 1 path. + cat > "$STUB_BIN_DIR/gh" << 'GHEOF' +#!/usr/bin/env bash +echo "copilot should have been skipped" >&2 +exit 1 +GHEOF + chmod +x "$STUB_BIN_DIR/gh" + + run run_writer_with_fallback "$TEST_PROMPT" + + [ "$status" -eq 2 ] +} + @test "writer: run_writer_with_fallback succeeds on second engine if first rate-limited" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false