Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 0 additions & 68 deletions .github/workflows/claude.yml

This file was deleted.

83 changes: 0 additions & 83 deletions .github/workflows/daily-pr-review-health.yml

This file was deleted.

54 changes: 54 additions & 0 deletions .github/workflows/dev-lead-retry.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
name: Dev-Lead Retry (rate-limited)
# Scans open PRs across the org for status=rate-limited markers and
# re-dispatches the appropriate dev-lead event once the rate limit clears.
#
# Runs every 2 hours (single schedule, no staggered offsets needed since the
# concurrency group serialises any overlap from GitHub scheduler jitter).
# Each repo is staggered by 30s during the scan to avoid dispatching all
# repos simultaneously and hitting the same org-level rate limit again.
#
# Phase 2 (fix-ci retry) and Phase 3 (fix-reviews/human-pr/rebase retry) are
# both handled here since the retry logic is the same for all retryable intents.

on:
schedule:
- cron: "15 */2 * * *"
workflow_dispatch:
inputs:
dry_run:
description: "If true, log retries but do not dispatch"
required: false
default: "false"
type: string
dispatch_delay_sec:
description: "Seconds between repo dispatches (default: 30)"
required: false
default: "30"
type: string

# contents: write is required to create repository_dispatch events.
permissions:
contents: write

concurrency:
group: dev-lead-retry
cancel-in-progress: false

env:
GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS || secrets.GITHUB_TOKEN }}
TARGET_ORG: ${{ vars.TARGET_ORG || 'petry-projects' }}
DRY_RUN: ${{ inputs.dry_run || 'false' }}
DISPATCH_DELAY_SEC: ${{ inputs.dispatch_delay_sec || '30' }}

jobs:
retry:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout agent repo
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
token: ${{ secrets.GH_PAT_WORKFLOWS || secrets.GITHUB_TOKEN }}

- name: Scan and retry rate-limited PRs
run: bash scripts/dev-lead-retry.sh
4 changes: 2 additions & 2 deletions .github/workflows/dev-lead.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ on:
check_run:
types: [completed]
repository_dispatch:
types: [dev-lead-ci-failure]
types: [dev-lead-ci-failure, dev-lead-reviews-retry]

permissions:
contents: write
Expand All @@ -29,7 +29,7 @@ concurrency:
group: >-
${{
github.event_name == 'check_run' && format('dev-lead-ci-relay-{0}', github.event.check_run.head_sha) ||
github.event_name == 'repository_dispatch' && format('dev-lead-fix-ci-{0}', github.event.client_payload.pr_number) ||
github.event_name == 'repository_dispatch' && format('dev-lead-pr-{0}', github.event.client_payload.pr_number) ||
github.event_name == 'issues' && format('dev-lead-issue-{0}', github.event.issue.number) ||
format('dev-lead-pr-{0}', github.event.pull_request.number || github.event.issue.number || '0')
}}
Expand Down
6 changes: 4 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c

### Workflow Files

- Do **not** modify `.github/workflows/claude.yml` or `.github/workflows/agent-shield.yml` —
these are exempted from agent modification per
- Do **not** modify `.github/workflows/agent-shield.yml` — this is exempted from agent modification per
[`standards/agent-standards.md`](https://github.com/petry-projects/.github/blob/main/standards/agent-standards.md).
- `.github/workflows/dev-lead.yml` is this repo's primary AI automation workflow (replaced `claude.yml` 2026-05).
It runs inline steps directly. To change behavior for this repo only, edit `dev-lead.yml`.
To change behavior across all org repos (via the cross-repo reusable), edit `dev-lead-reusable.yml`.
- All other workflow changes must use templates from
[`standards/workflows/`](https://github.com/petry-projects/.github/tree/main/standards/workflows) verbatim.

Expand Down
86 changes: 77 additions & 9 deletions scripts/dev-lead-fix-ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ MARKER_PREFIX="<!-- dev-lead-fix-ci sha="
EXHAUSTION_MARKER="<!-- dev-lead-fix-ci pr=${PR_NUMBER} status=exhausted -->"
export PROMPTS_DIR="${PROMPTS_DIR:-prompts/dev-lead}"

# check_idempotency: returns 0 (skip) if this exact SHA was already attempted,
# OR if a PR-level exhaustion marker exists (blocks all future SHAs).
# check_idempotency: returns 0 (skip) if this exact SHA was already TERMINALLY
# handled, OR if a PR-level exhaustion marker exists (blocks all future SHAs).
# status=rate-limited is NOT terminal — those runs must be retriable.
check_idempotency() {
local comments
# Use standalone jq so the mock stub in tests can return raw JSON
comments=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
# Use standalone jq so the mock stub in tests can return raw JSON.
# --paginate ensures markers on busy PRs (>30 comments) are not missed.
comments=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \
| jq -r '.[].body' 2>/dev/null || true)

# PR-level exhaustion — blocks regardless of SHA
Expand All @@ -34,22 +36,40 @@ check_idempotency() {
return 0
fi

# SHA-level idempotency — skip if this exact SHA was already attempted
if echo "$comments" | grep -qF "${MARKER_PREFIX}${HEAD_SHA}"; then
# SHA-level idempotency — skip only for terminal statuses (applied, failed, no-changes).
# status=rate-limited is retriable: do not skip on it so the retry cron can re-run.
if echo "$comments" | grep -qE "${MARKER_PREFIX}${HEAD_SHA} status=(applied|failed|no-changes)"; then
echo "::notice::Already handled CI failure at SHA ${HEAD_SHA} with terminal status — skipping"
return 0
fi

return 1
}

# count_recent_failures: count status=failed markers on this PR (any SHA)
# count_recent_failures: count status=failed markers on this PR (any SHA).
# Intentionally excludes status=rate-limited so rate limit events do not count
# toward the exhaustion threshold — rate limits are temporary infrastructure
# events, not evidence of repeated engine errors on this PR's content.
# --paginate ensures an accurate count even on PRs with >30 comments.
count_recent_failures() {
local pattern="${MARKER_PREFIX}[a-f0-9A-F]* status=failed"
gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \
| jq "[.[] | select(.body | test(\"${pattern}\"))] | length" 2>/dev/null \
|| echo "0"
}

# has_rate_limited_marker: returns 0 if a status=rate-limited marker already
# exists for HEAD_SHA on this PR. Used to avoid duplicate rate-limited comments.
# --paginate ensures the dedup check is accurate on PRs with >30 comments.
has_rate_limited_marker() {
local pattern="${MARKER_PREFIX}${HEAD_SHA} status=rate-limited"
local count
count=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \
| jq "[.[] | select(.body | test(\"${pattern}\"))] | length" 2>/dev/null \
|| echo "0")
[ "${count:-0}" -gt 0 ]
Comment on lines +67 to +70

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Combine paginated counts before numeric tests

On PRs whose comments span multiple pages, gh api --paginate feeds one JSON array per page into this jq filter, so count becomes multiple newline-separated numbers such as 1\n0. The subsequent [ "${count:-0}" -gt 0 ] is not a valid numeric comparison and returns false, so an existing rate-limited marker on an earlier page is not deduped and repeated rate-limit runs keep posting duplicate comments.

Useful? React with 👍 / 👎.

}

collect_logs() {
local check_name="$1" details_url="$2"
local run_id
Expand Down Expand Up @@ -114,6 +134,46 @@ To re-enable, delete this comment or push a new commit with a substantially diff
fi
}

# post_rate_limited: posts a status=rate-limited marker with embedded reset
# time and check name. Skips if a marker already exists for HEAD_SHA (dedup).
# The check= field lets the retry cron look up current check-run details when
# re-dispatching, giving the engine full failure logs/annotations on retry.
post_rate_limited() {
# Dedup: don't accumulate multiple rate-limited markers for the same SHA
if has_rate_limited_marker; then
echo "::notice::rate-limited marker already posted for SHA ${HEAD_SHA} — skipping duplicate"
return 0
fi

local reset_time
reset_time=$(cat /tmp/dev-lead-rate-limit-reset 2>/dev/null || true)
local reset_detail=""
if [ -n "$reset_time" ]; then
reset_detail=" reset=${reset_time}"
fi

# Embed check name so the retry cron can look up the current check run for logs
local check_name
check_name=$(echo "${CHECKS_JSON:-[]}" | jq -r '.[0].name // "CI failure"' 2>/dev/null || echo "CI failure")

local marker="${MARKER_PREFIX}${HEAD_SHA} status=rate-limited${reset_detail} check=${check_name} -->"
Comment on lines +157 to +159

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Encode check names before storing them in markers

For failed checks whose names contain spaces or punctuation, this embeds the raw name as check=${check_name} in the HTML marker, but the retry scanner later extracts only a single non-whitespace token with check=(...). Fresh evidence since the prior CI-context finding: even after adding the lookup path, a common name like Test Dev-Lead Agent is stored in a form that can only be recovered as Test, so the check-run lookup misses and the retried CI fix again loses logs/annotations.

Useful? React with 👍 / 👎.

local details="All engines were rate-limited. The retry cron will re-attempt automatically."
if [ -n "$reset_time" ]; then
details="${details} Rate limit resets at: \`${reset_time}\`"
fi
local body="${marker}
## Dev-Lead Fix CI — rate-limited
**PR:** #${PR_NUMBER} | **SHA:** \`${HEAD_SHA}\`
${details}"

if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then
echo "[dry-run] would post PR comment: rate-limited"
echo "$body"
else
gh pr comment "$PR_NUMBER" --repo "$REPO" --body "$body"
fi
}

main() {
if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_SHA" ]; then
echo "::error::PR_NUMBER and HEAD_SHA are required"
Expand Down Expand Up @@ -145,7 +205,15 @@ main() {
run_writer_with_fallback "$prompt_file" || engine_rc=$?

if [ "$engine_rc" -ne 0 ]; then
# Classify the failure: timeout (124) is a hard exhaustion candidate
if [ "$engine_rc" -eq 2 ]; then
# Rate-limited (all engines exhausted) — not a real failure; do not
# count toward exhaustion threshold. Post rate-limited marker for retry cron.
echo "::warning::All engines rate-limited — posting rate-limited marker for retry cron"
post_rate_limited
exit 2
fi

# Real engine failure (exit 1 or 124) — counts toward exhaustion threshold
local reason="Engine invocation failed (exit ${engine_rc})"
[ "$engine_rc" -eq 124 ] && reason="Engine timed out — PR may be too large for automated fixing"
post_summary "failed" "$reason"
Expand Down
Loading