Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions .github/workflows/actions-fleet-monitor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,90 @@ jobs:
GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }}
run: bash scripts/fleet_monitor.sh

- name: Summarize token usage by workflow
env:
GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }}
REPO: petry-projects/.github-private
LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }}
run: |
set -euo pipefail

workdir="$(mktemp -d)"
trap 'rm -rf "$workdir"' EXIT
jsonl_dir="$workdir/jsonl"
artifacts_file="$workdir/token-artifacts.base64"
mkdir -p "$jsonl_dir"

cutoff_epoch="$(date -u -d "-${LOOKBACK_DAYS} days" +%s)"
artifact_count=0

gh api "repos/$REPO/actions/artifacts" --paginate \
--jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' \
> "$artifacts_file"

while IFS= read -r encoded; do
[ -n "$encoded" ] || continue
item="$(printf '%s' "$encoded" | base64 -d)"
id="$(printf '%s' "$item" | jq -r '.id')"
created_at="$(printf '%s' "$item" | jq -r '.created_at')"
expired="$(printf '%s' "$item" | jq -r '.expired')"
created_epoch="$(date -u -d "$created_at" +%s)"

if [ "$expired" = "true" ] || [ "$created_epoch" -lt "$cutoff_epoch" ]; then
continue
fi

zip_file="$workdir/artifact-$id.zip"
unzip_dir="$workdir/unzip-$id"
mkdir -p "$unzip_dir"
gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file"
unzip -q -o "$zip_file" -d "$unzip_dir"

found_any=false
while IFS= read -r f; do
found_any=true
cp "$f" "$jsonl_dir/${id}-$(basename "$f")"
done < <(find "$unzip_dir" -type f -name '*.jsonl' -print)

if [ "$found_any" = true ]; then
artifact_count=$((artifact_count + 1))
fi
done < "$artifacts_file"

if ! find "$jsonl_dir" -type f -name '*.jsonl' | grep -q .; then
{
echo "## Token usage (${LOOKBACK_DAYS} day lookback)"
echo
echo "No token-usage artifacts found in the lookback window."
} >> "$GITHUB_STEP_SUMMARY"
exit 0
fi

aggregate="$workdir/token-usage-summary.json"
jq -s '
sort_by(.workflow) |
group_by(.workflow) |
map({
workflow: (.[0].workflow // "unknown"),
calls: length,
input_tokens: (map(.input_tokens // 0) | add),
cache_read_tokens: (map(.cache_read_tokens // 0) | add),
output_tokens: (map(.output_tokens // 0) | add),
et: (map(.et // 0) | add)
}) |
sort_by(.et) | reverse
' "$jsonl_dir"/*.jsonl > "$aggregate"
Comment thread
don-petry marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Comment thread
don-petry marked this conversation as resolved.
Resolved

{
echo "## Token usage (${LOOKBACK_DAYS} day lookback)"
echo
echo "Processed ${artifact_count} token-usage artifact(s)."
echo
echo "| Workflow | Calls | Input Tokens | Cache Read Tokens | Output Tokens | Total ET |"
echo "|---|---:|---:|---:|---:|---:|"
jq -r '.[] | "| \(.workflow) | \(.calls) | \(.input_tokens) | \(.cache_read_tokens) | \(.output_tokens) | \(.et | tostring) |"' "$aggregate"
} >> "$GITHUB_STEP_SUMMARY"

- name: Create issue with findings
if: env.HAS_FAILURES == 'true' && hashFiles('fleet_monitor_report.md') != ''
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dev-lead-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ jobs:
# Triggered when a trusted bot approves a PR. Skips engine install and
# pre-flight — just checks reviewDecision and enables auto-merge if APPROVED.
- name: Enable auto-merge on bot approval
if: env.INTENT_TYPE == 'enable-auto-merge'
if: env.INTENT_TYPE == 'enable-auto-merge' && env.INTENT_PR_NUMBER != ''
env:
PR_NUMBER: ${{ env.INTENT_PR_NUMBER }}
HEAD_SHA: ${{ env.INTENT_HEAD_SHA }}
Expand Down
6 changes: 3 additions & 3 deletions docs/dev-lead/spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -582,9 +582,9 @@ Per-engine model names are controlled by the same env vars used by the PR Review
| Variable | Claude default | Gemini default | Copilot default |
|---|---|---|---|
| `ENGINE_TRIAGE_MODEL` | `claude-haiku-4-5-20251001` | `gemini-2.0-flash` | `o4-mini` |
| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` |
| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-1.5-pro` | `o4-mini` |
| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-1.5-pro` | `o4-mini` |
| `ENGINE_ACTION_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` |
| `ENGINE_DEEP_MODEL` | `claude-sonnet-4-6` | `gemini-2.5-pro` | `o4-mini` |
| `ENGINE_SINGLE_MODEL` | `claude-opus-4-7` | `gemini-2.5-pro` | `o4-mini` |

---

Expand Down
23 changes: 14 additions & 9 deletions scripts/engine.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@ set_engine_config() {
;;
gemini)
ENGINE_TRIAGE_MODEL="gemini-2.0-flash"
ENGINE_DEEP_MODEL="gemini-1.5-pro"
ENGINE_AUDIT_MODEL="gemini-1.5-pro"
ENGINE_ACTION_MODEL="gemini-1.5-pro"
ENGINE_SINGLE_MODEL="gemini-1.5-pro"
ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-1.5-pro + duck: sonnet 4.6 → audit: gemini-1.5-pro"
ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-1.5-pro"
ENGINE_DEEP_MODEL="gemini-2.5-pro"
ENGINE_AUDIT_MODEL="gemini-2.5-pro"
ENGINE_ACTION_MODEL="gemini-2.5-pro"
ENGINE_SINGLE_MODEL="gemini-2.5-pro"
ENGINE_LABEL="triage: gemini-2.0-flash → deep: gemini-2.5-pro + duck: sonnet 4.6 → audit: gemini-2.5-pro"
ENGINE_SINGLE_LABEL="single-reviewer mode: gemini-2.5-pro"
# Cross-engine rubber duck: use Claude for diversity
DUCK_ENGINE="claude"
DUCK_MODEL="claude-sonnet-4-6"
Expand Down Expand Up @@ -536,7 +536,7 @@ run_writer() {
--model "$model" \
--permission-mode acceptEdits \
--allowed-tools "Bash,Read,Write,Edit,Grep,Glob,WebFetch" \
< "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]}
< "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]}
else
timeout "$ACTION_TIMEOUT_SEC" claude --print \
--model "$model" \
Expand All @@ -551,7 +551,7 @@ run_writer() {
--model "$model" \
--approval-mode auto_edit \
--output-format text \
< "$prompt_file" | tee "$_tmp" || rc=${PIPESTATUS[0]}
< "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]}
else
timeout "$ACTION_TIMEOUT_SEC" gemini --prompt "" \
--model "$model" \
Expand All @@ -563,7 +563,7 @@ run_writer() {
copilot)
# Self-sufficient write support via gh copilot --yolo
if [ -n "$_tmp" ]; then
copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo | tee "$_tmp" || rc=${PIPESTATUS[0]}
copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]}
else
copilot_chat "$prompt_file" "$ACTION_TIMEOUT_SEC" --yolo || rc=$?
fi
Expand Down Expand Up @@ -595,6 +595,11 @@ run_writer_with_fallback() {
done

for engine in "${engines[@]}"; do
if [ "$engine" = "copilot" ] && [[ "${COPILOT_GITHUB_TOKEN:-}" == ghp_* ]]; then
echo "::warning::Skipping copilot fallback: classic PAT in COPILOT_GITHUB_TOKEN is unsupported" >&2
continue
fi

local saved="$REVIEW_ENGINE"
export REVIEW_ENGINE="$engine"
# Re-evaluate model names for the new engine
Expand Down
43 changes: 43 additions & 0 deletions tests/dev-lead/unit/test_engine_writer.bats
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,21 @@ STUB
[ "$status" -eq 2 ]
}

@test "writer: run_writer returns exit 2 when gemini writes rate-limit text to stderr" {
_source_engine "gemini"
export DEV_LEAD_DRY_RUN=false
cat > "$STUB_BIN_DIR/gemini" << 'STUB'
#!/usr/bin/env bash
echo "quota exceeded for today" >&2
exit 1
STUB
chmod +x "$STUB_BIN_DIR/gemini"

run run_writer "$TEST_PROMPT"

[ "$status" -eq 2 ]
}

@test "writer: run_writer returns exit 1 (not 2) for non-rate-limit failure" {
_source_engine "claude"
export DEV_LEAD_DRY_RUN=false
Expand Down Expand Up @@ -214,6 +229,34 @@ GHEOF
[ "$status" -eq 2 ]
}

@test "writer: run_writer_with_fallback skips copilot when token is classic PAT" {
_source_engine "claude"
export DEV_LEAD_DRY_RUN=false
export COPILOT_GITHUB_TOKEN="ghp_classic_token"

# Claude and Gemini both rate-limit.
for engine in claude gemini; do
cat > "$STUB_BIN_DIR/$engine" << 'STUB'
#!/usr/bin/env bash
echo "quota exceeded" >&2
exit 1
STUB
chmod +x "$STUB_BIN_DIR/$engine"
done

# If copilot is invoked, this stub would fail the test via status 1 path.
cat > "$STUB_BIN_DIR/gh" << 'GHEOF'
#!/usr/bin/env bash
echo "copilot should have been skipped" >&2
exit 1
GHEOF
chmod +x "$STUB_BIN_DIR/gh"

run run_writer_with_fallback "$TEST_PROMPT"

[ "$status" -eq 2 ]
}

@test "writer: run_writer_with_fallback succeeds on second engine if first rate-limited" {
_source_engine "claude"
export DEV_LEAD_DRY_RUN=false
Expand Down
Loading