Repository navigation
fix(dev-lead): rate-limit detection, separate status, and retry cron (#192) #196
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
fd63d11
fcedcc7
041f318
d2c0aed
0a8c732
3b50d87
55a6159
b655931
969f42f
02bae5f
0c77700
f2ee13f
aeb0582
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| name: Dev-Lead Retry (rate-limited) | ||
| # Scans open PRs across the org for status=rate-limited markers and | ||
| # re-dispatches the appropriate dev-lead event once the rate limit clears. | ||
| # | ||
| # Runs every 2 hours (single schedule, no staggered offsets needed since the | ||
| # concurrency group serialises any overlap from GitHub scheduler jitter). | ||
| # Each repo is staggered by 30s during the scan to avoid dispatching all | ||
| # repos simultaneously and hitting the same org-level rate limit again. | ||
| # | ||
| # Phase 2 (fix-ci retry) and Phase 3 (fix-reviews/human-pr/rebase retry) are | ||
| # both handled here since the retry logic is the same for all retryable intents. | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: "15 */2 * * *" | ||
| workflow_dispatch: | ||
| inputs: | ||
| dry_run: | ||
| description: "If true, log retries but do not dispatch" | ||
| required: false | ||
| default: "false" | ||
| type: string | ||
| dispatch_delay_sec: | ||
| description: "Seconds between repo dispatches (default: 30)" | ||
| required: false | ||
| default: "30" | ||
| type: string | ||
|
|
||
| # contents: write is required to create repository_dispatch events. | ||
| permissions: | ||
| contents: write | ||
|
|
||
| concurrency: | ||
| group: dev-lead-retry | ||
| cancel-in-progress: false | ||
|
|
||
| env: | ||
| GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS || secrets.GITHUB_TOKEN }} | ||
| TARGET_ORG: ${{ vars.TARGET_ORG || 'petry-projects' }} | ||
| DRY_RUN: ${{ inputs.dry_run || 'false' }} | ||
| DISPATCH_DELAY_SEC: ${{ inputs.dispatch_delay_sec || '30' }} | ||
|
|
||
| jobs: | ||
| retry: | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 30 | ||
| steps: | ||
| - name: Checkout agent repo | ||
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | ||
| with: | ||
| token: ${{ secrets.GH_PAT_WORKFLOWS || secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Scan and retry rate-limited PRs | ||
| run: bash scripts/dev-lead-retry.sh |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,12 +20,14 @@ MARKER_PREFIX="<!-- dev-lead-fix-ci sha=" | |
| EXHAUSTION_MARKER="<!-- dev-lead-fix-ci pr=${PR_NUMBER} status=exhausted -->" | ||
| export PROMPTS_DIR="${PROMPTS_DIR:-prompts/dev-lead}" | ||
|
|
||
| # check_idempotency: returns 0 (skip) if this exact SHA was already attempted, | ||
| # OR if a PR-level exhaustion marker exists (blocks all future SHAs). | ||
| # check_idempotency: returns 0 (skip) if this exact SHA was already TERMINALLY | ||
| # handled, OR if a PR-level exhaustion marker exists (blocks all future SHAs). | ||
| # status=rate-limited is NOT terminal — those runs must be retriable. | ||
| check_idempotency() { | ||
| local comments | ||
| # Use standalone jq so the mock stub in tests can return raw JSON | ||
| comments=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \ | ||
| # Use standalone jq so the mock stub in tests can return raw JSON. | ||
| # --paginate ensures markers on busy PRs (>30 comments) are not missed. | ||
| comments=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ | ||
| | jq -r '.[].body' 2>/dev/null || true) | ||
|
|
||
| # PR-level exhaustion — blocks regardless of SHA | ||
|
|
@@ -34,22 +36,40 @@ check_idempotency() { | |
| return 0 | ||
| fi | ||
|
|
||
| # SHA-level idempotency — skip if this exact SHA was already attempted | ||
| if echo "$comments" | grep -qF "${MARKER_PREFIX}${HEAD_SHA}"; then | ||
| # SHA-level idempotency — skip only for terminal statuses (applied, failed, no-changes). | ||
| # status=rate-limited is retriable: do not skip on it so the retry cron can re-run. | ||
| if echo "$comments" | grep -qE "${MARKER_PREFIX}${HEAD_SHA} status=(applied|failed|no-changes)"; then | ||
| echo "::notice::Already handled CI failure at SHA ${HEAD_SHA} with terminal status — skipping" | ||
| return 0 | ||
| fi | ||
|
|
||
| return 1 | ||
| } | ||
|
|
||
| # count_recent_failures: count status=failed markers on this PR (any SHA) | ||
| # count_recent_failures: count status=failed markers on this PR (any SHA). | ||
| # Intentionally excludes status=rate-limited so rate limit events do not count | ||
| # toward the exhaustion threshold — rate limits are temporary infrastructure | ||
| # events, not evidence of repeated engine errors on this PR's content. | ||
| # --paginate ensures an accurate count even on PRs with >30 comments. | ||
| count_recent_failures() { | ||
| local pattern="${MARKER_PREFIX}[a-f0-9A-F]* status=failed" | ||
| gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \ | ||
| gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ | ||
| | jq "[.[] | select(.body | test(\"${pattern}\"))] | length" 2>/dev/null \ | ||
| || echo "0" | ||
| } | ||
|
|
||
| # has_rate_limited_marker: returns 0 if a status=rate-limited marker already | ||
| # exists for HEAD_SHA on this PR. Used to avoid duplicate rate-limited comments. | ||
| # --paginate ensures the dedup check is accurate on PRs with >30 comments. | ||
| has_rate_limited_marker() { | ||
| local pattern="${MARKER_PREFIX}${HEAD_SHA} status=rate-limited" | ||
| local count | ||
| count=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ | ||
| | jq "[.[] | select(.body | test(\"${pattern}\"))] | length" 2>/dev/null \ | ||
| || echo "0") | ||
| [ "${count:-0}" -gt 0 ] | ||
| } | ||
|
|
||
| collect_logs() { | ||
| local check_name="$1" details_url="$2" | ||
| local run_id | ||
|
|
@@ -114,6 +134,46 @@ To re-enable, delete this comment or push a new commit with a substantially diff | |
| fi | ||
| } | ||
|
|
||
| # post_rate_limited: posts a status=rate-limited marker with embedded reset | ||
| # time and check name. Skips if a marker already exists for HEAD_SHA (dedup). | ||
| # The check= field lets the retry cron look up current check-run details when | ||
| # re-dispatching, giving the engine full failure logs/annotations on retry. | ||
| post_rate_limited() { | ||
| # Dedup: don't accumulate multiple rate-limited markers for the same SHA | ||
| if has_rate_limited_marker; then | ||
| echo "::notice::rate-limited marker already posted for SHA ${HEAD_SHA} — skipping duplicate" | ||
| return 0 | ||
| fi | ||
|
|
||
| local reset_time | ||
| reset_time=$(cat /tmp/dev-lead-rate-limit-reset 2>/dev/null || true) | ||
| local reset_detail="" | ||
| if [ -n "$reset_time" ]; then | ||
| reset_detail=" reset=${reset_time}" | ||
| fi | ||
|
|
||
| # Embed check name so the retry cron can look up the current check run for logs | ||
| local check_name | ||
| check_name=$(echo "${CHECKS_JSON:-[]}" | jq -r '.[0].name // "CI failure"' 2>/dev/null || echo "CI failure") | ||
|
|
||
| local marker="${MARKER_PREFIX}${HEAD_SHA} status=rate-limited${reset_detail} check=${check_name} -->" | ||
|
Comment on lines
+157
to
+159
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
For failed checks whose names contain spaces or punctuation, this embeds the raw name as Useful? React with 👍 / 👎. |
||
| local details="All engines were rate-limited. The retry cron will re-attempt automatically." | ||
| if [ -n "$reset_time" ]; then | ||
| details="${details} Rate limit resets at: \`${reset_time}\`" | ||
| fi | ||
| local body="${marker} | ||
| ## Dev-Lead Fix CI — rate-limited | ||
| **PR:** #${PR_NUMBER} | **SHA:** \`${HEAD_SHA}\` | ||
| ${details}" | ||
|
|
||
| if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then | ||
| echo "[dry-run] would post PR comment: rate-limited" | ||
| echo "$body" | ||
| else | ||
| gh pr comment "$PR_NUMBER" --repo "$REPO" --body "$body" | ||
| fi | ||
| } | ||
|
|
||
| main() { | ||
| if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_SHA" ]; then | ||
| echo "::error::PR_NUMBER and HEAD_SHA are required" | ||
|
|
@@ -145,7 +205,15 @@ main() { | |
| run_writer_with_fallback "$prompt_file" || engine_rc=$? | ||
|
|
||
| if [ "$engine_rc" -ne 0 ]; then | ||
| # Classify the failure: timeout (124) is a hard exhaustion candidate | ||
| if [ "$engine_rc" -eq 2 ]; then | ||
| # Rate-limited (all engines exhausted) — not a real failure; do not | ||
| # count toward exhaustion threshold. Post rate-limited marker for retry cron. | ||
| echo "::warning::All engines rate-limited — posting rate-limited marker for retry cron" | ||
| post_rate_limited | ||
| exit 2 | ||
| fi | ||
|
|
||
| # Real engine failure (exit 1 or 124) — counts toward exhaustion threshold | ||
| local reason="Engine invocation failed (exit ${engine_rc})" | ||
| [ "$engine_rc" -eq 124 ] && reason="Engine timed out — PR may be too large for automated fixing" | ||
| post_summary "failed" "$reason" | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
On PRs whose comments span multiple pages,
gh api --paginatefeeds one JSON array per page into this jq filter, socountbecomes multiple newline-separated numbers such as1\n0. The subsequent[ "${count:-0}" -gt 0 ]is not a valid numeric comparison and returns false, so an existing rate-limited marker on an earlier page is not deduped and repeated rate-limit runs keep posting duplicate comments.Useful? React with 👍 / 👎.