From 329a6240833c593c6e0478b6c2c3dbc4e4218d4f Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Sat, 13 Jun 2026 17:34:44 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20implement=20issue=20#645=20?= =?UTF-8?q?=E2=80=94=20no=20fix=20when=20auto=20rebase=20workflow=20fails?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/auto-rebase-retry.yml | 57 ++++++++ .github/workflows/test-dev-lead.yml | 4 + AGENTS.md | 6 + scripts/auto-rebase-retry.sh | 97 ++++++++++++++ .../dev-lead/unit/test_auto_rebase_retry.bats | 126 ++++++++++++++++++ 5 files changed, 290 insertions(+) create mode 100644 .github/workflows/auto-rebase-retry.yml create mode 100755 scripts/auto-rebase-retry.sh create mode 100644 tests/dev-lead/unit/test_auto_rebase_retry.bats diff --git a/.github/workflows/auto-rebase-retry.yml b/.github/workflows/auto-rebase-retry.yml new file mode 100644 index 000000000..1e19737f2 --- /dev/null +++ b/.github/workflows/auto-rebase-retry.yml @@ -0,0 +1,57 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Auto-rebase retry — self-heals a FAILED Auto-rebase workflow run. +# +# REPO-SPECIFIC WORKFLOW (no central template). See AGENTS.md +# §"Workflow Files" exception note. Documented gap: the Auto-rebase reusable +# handles merge *conflicts* gracefully (sentinel comment → dev-lead `rebase` +# intent), but a failed *run* (transient API error, rate-limit, infra hiccup) +# posts no sentinel and nothing retries it. This handler re-runs the failed +# jobs, bounded by the run's attempt counter (MAX_ATTEMPTS), and surfaces a +# warning once retries are exhausted. +# +# workflow_run handlers run from the default branch only, so changes here take +# effect after merge to main. +# ───────────────────────────────────────────────────────────────────────────── +name: Auto-rebase retry + +on: + workflow_run: + workflows: ["Auto-rebase non-Dependabot PRs"] + types: [completed] + +permissions: {} + +concurrency: + group: auto-rebase-retry-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + retry: + # Only act on failed runs; the script re-checks but gating here avoids + # spinning up a runner for every successful/cancelled Auto-rebase run. + if: github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write # gh run rerun --failed + steps: + - name: Checkout agent repo + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Retry failed Auto-rebase run + env: + # Prefer a PAT: a re-run requested via GITHUB_TOKEN does not re-fire + # workflow_run events (GitHub's recursion guard), so only GH_PAT_WORKFLOWS + # lets attempts past the first chain back into this handler toward + # MAX_ATTEMPTS. With github.token alone, a single retry still fires. + GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS || github.token }} + REPO: ${{ github.repository }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + HTML_URL: ${{ github.event.workflow_run.html_url }} + MAX_ATTEMPTS: "3" + run: bash scripts/auto-rebase-retry.sh diff --git a/.github/workflows/test-dev-lead.yml b/.github/workflows/test-dev-lead.yml index a8cad7c13..49d8895b5 100644 --- a/.github/workflows/test-dev-lead.yml +++ b/.github/workflows/test-dev-lead.yml @@ -3,6 +3,7 @@ on: pull_request: paths: - 'scripts/dev-lead*' + - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' - '.github/workflows/dev-lead*.yml' @@ -10,12 +11,14 @@ on: - '.github/workflows/issue-triage-runner.yml' - '.github/workflows/dependency-audit.yml' - '.github/workflows/auto-rebase.yml' + - '.github/workflows/auto-rebase-retry.yml' - 'tests/dev-lead/**' - 'prompts/dev-lead/**' push: branches: [main] paths: - 'scripts/dev-lead*' + - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' - 'tests/dev-lead/**' @@ -28,6 +31,7 @@ on: - '.github/workflows/issue-triage-runner.yml' - '.github/workflows/dependency-audit.yml' - '.github/workflows/auto-rebase.yml' + - '.github/workflows/auto-rebase-retry.yml' workflow_dispatch: {} permissions: diff --git a/AGENTS.md b/AGENTS.md index 99a861c4a..85cdd81cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,12 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c - **Exception:** The `bats` test list in `lint.yml` is extended with repo-specific test files (e.g. `tests/test_push_protection.bats`) beyond the org template baseline. When adding new test files to this repo, add them to this list. Template syncs must not reset it to the base template list. +- **Exception:** `auto-rebase-retry.yml` is a documented repo-specific workflow with no corresponding org + template in `standards/workflows/`. It is a `workflow_run` handler that self-heals a *failed* Auto-rebase + run by re-running its failed jobs (bounded by `run_attempt`) — the gap left by the conflict-only sentinel + path, which the thin-caller `auto-rebase.yml` and central reusable cannot cover. It must not be removed by + template syncs. If the org template gains a retry equivalent, remove this exception and defer to the + template instead. ### Agent Profiles (`agents/*.md`) diff --git a/scripts/auto-rebase-retry.sh b/scripts/auto-rebase-retry.sh new file mode 100755 index 000000000..750265e43 --- /dev/null +++ b/scripts/auto-rebase-retry.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail +# auto-rebase-retry.sh — self-heal a failed Auto-rebase workflow run. +# +# The Auto-rebase workflow handles *merge conflicts* gracefully (it posts a +# sentinel comment that dev-lead's `rebase` intent then resolves). But when the +# run itself fails — a transient GitHub API error, a rate-limit, an infra +# hiccup — no sentinel is posted and nothing retries it. This handler closes +# that gap: on a failed run it re-runs the failed jobs, bounded by the run's +# attempt counter so a deterministically-broken run cannot loop forever. When +# the attempt cap is reached it stops retrying and surfaces a warning so a +# human can intervene. +# +# Bounding works without external state: `gh run rerun --failed` increments the +# run's attempt counter, and each completed re-run fires another workflow_run +# event, so RUN_ATTEMPT naturally climbs toward MAX_ATTEMPTS. (Chaining past the +# first retry requires GH_PAT_WORKFLOWS — re-runs requested via the default +# GITHUB_TOKEN do not re-fire workflow_run events.) +# +# Env inputs: +# CONCLUSION — github.event.workflow_run.conclusion +# RUN_ID — github.event.workflow_run.id +# RUN_ATTEMPT — github.event.workflow_run.run_attempt (defaults to 1) +# WORKFLOW_NAME — github.event.workflow_run.name (logging only) +# HTML_URL — github.event.workflow_run.html_url (logging only) +# REPO — owner/repo (defaults to GITHUB_REPOSITORY) +# MAX_ATTEMPTS — give up after this many attempts (default: 3) +# DRY_RUN — if "true", log intent but do not call gh (default: false) +# GITHUB_STEP_SUMMARY — path for the run summary (optional) +# +# Always exits 0: a handler failure would itself be noise. Outcomes are surfaced +# via ::notice:: / ::warning:: annotations and the step summary instead. + +CONCLUSION="${CONCLUSION:-}" +RUN_ID="${RUN_ID:-}" +RUN_ATTEMPT="${RUN_ATTEMPT:-1}" +WORKFLOW_NAME="${WORKFLOW_NAME:-Auto-rebase}" +HTML_URL="${HTML_URL:-}" +REPO="${REPO:-${GITHUB_REPOSITORY:-}}" +MAX_ATTEMPTS="${MAX_ATTEMPTS:-3}" +DRY_RUN="${DRY_RUN:-false}" +GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" + +summary() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; } + +# Default any non-integer attempt to 1 so a malformed payload still retries once +# rather than silently skipping or crashing under `set -e`. +if ! [[ "$RUN_ATTEMPT" =~ ^[0-9]+$ ]]; then + echo "::notice::RUN_ATTEMPT '${RUN_ATTEMPT}' is not an integer — defaulting to 1" + RUN_ATTEMPT=1 +fi +if ! [[ "$MAX_ATTEMPTS" =~ ^[0-9]+$ ]] || [ "$MAX_ATTEMPTS" -lt 1 ]; then + MAX_ATTEMPTS=3 +fi + +# ── guard: only act on failed runs ──────────────────────────────────────────── +if [ "$CONCLUSION" != "failure" ]; then + echo "::notice::${WORKFLOW_NAME} run concluded '${CONCLUSION:-unknown}' (not failure) — nothing to retry" + exit 0 +fi + +# ── guard: need a run id to retry ───────────────────────────────────────────── +if [ -z "$RUN_ID" ]; then + echo "::warning::No RUN_ID provided — cannot retry the failed ${WORKFLOW_NAME} run" + exit 0 +fi + +# ── give up once the attempt cap is reached ─────────────────────────────────── +if [ "$RUN_ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then + echo "::warning::${WORKFLOW_NAME} run ${RUN_ID} failed after ${RUN_ATTEMPT} attempt(s) (cap ${MAX_ATTEMPTS}) — giving up, manual intervention required" + summary "### Auto-rebase retry exhausted" + summary "" + summary "Run [\`${RUN_ID}\`](${HTML_URL}) failed after **${RUN_ATTEMPT}** attempt(s) (cap ${MAX_ATTEMPTS}). Automated retries are exhausted — a maintainer should investigate manually." + exit 0 +fi + +next_attempt=$(( RUN_ATTEMPT + 1 )) +echo "::notice::${WORKFLOW_NAME} run ${RUN_ID} failed on attempt ${RUN_ATTEMPT} — re-running failed jobs (attempt ${next_attempt}/${MAX_ATTEMPTS})" + +if [ "$DRY_RUN" = "true" ]; then + echo " [dry-run] would run: gh run rerun ${RUN_ID} --failed (repo ${REPO})" + summary "### Auto-rebase retry (dry-run)" + summary "Would re-run failed jobs of run \`${RUN_ID}\` (attempt ${next_attempt}/${MAX_ATTEMPTS})." + exit 0 +fi + +if gh run rerun "$RUN_ID" --failed --repo "$REPO"; then + echo "::notice::Re-run of run ${RUN_ID} requested (attempt ${next_attempt}/${MAX_ATTEMPTS})" + summary "### Auto-rebase retry dispatched" + summary "Re-ran failed jobs of run [\`${RUN_ID}\`](${HTML_URL}) — attempt ${next_attempt}/${MAX_ATTEMPTS}." +else + echo "::warning::Failed to request re-run of run ${RUN_ID} — manual intervention may be required" + summary "### Auto-rebase retry failed to dispatch" + summary "Could not re-run run [\`${RUN_ID}\`](${HTML_URL}). A maintainer should re-run it manually." +fi + +exit 0 diff --git a/tests/dev-lead/unit/test_auto_rebase_retry.bats b/tests/dev-lead/unit/test_auto_rebase_retry.bats new file mode 100644 index 000000000..8996a5aa4 --- /dev/null +++ b/tests/dev-lead/unit/test_auto_rebase_retry.bats @@ -0,0 +1,126 @@ +#!/usr/bin/env bats +# Unit tests for scripts/auto-rebase-retry.sh +# Covers the workflow_run failure handler that self-heals a failed Auto-rebase +# run by re-running its failed jobs, bounded by run_attempt, and surfacing a +# warning to humans once retries are exhausted. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +RETRY_SCRIPT="$SCRIPT_DIR/scripts/auto-rebase-retry.sh" + +setup() { + STUB_BIN_DIR="$(mktemp -d)" + RERUN_MARKER="$(mktemp -u)" + export RERUN_MARKER + # Mock gh: record any `run rerun` invocation to RERUN_MARKER. + cat > "$STUB_BIN_DIR/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run rerun"*) + echo "$*" > "$RERUN_MARKER" + [ "${GH_STUB_RERUN_FAILS:-0}" = "1" ] && exit 1 + echo "Requested rerun of run" + ;; + *) + echo "{}" + ;; +esac +GHEOF + chmod +x "$STUB_BIN_DIR/gh" + export PATH="$STUB_BIN_DIR:$PATH" + + export GITHUB_STEP_SUMMARY + GITHUB_STEP_SUMMARY="$(mktemp)" + + # Sensible defaults for a failing run on its first attempt. + export REPO="petry-projects/.github-private" + export RUN_ID="123456" + export RUN_ATTEMPT="1" + export CONCLUSION="failure" + export WORKFLOW_NAME="Auto-rebase non-Dependabot PRs" + export HTML_URL="https://github.com/petry-projects/.github-private/actions/runs/123456" + export MAX_ATTEMPTS="3" + unset GH_STUB_RERUN_FAILS DRY_RUN +} + +teardown() { + rm -rf "$STUB_BIN_DIR" + rm -f "$RERUN_MARKER" "$GITHUB_STEP_SUMMARY" +} + +_reran() { [ -f "$RERUN_MARKER" ]; } + +@test "auto-rebase-retry: failure on first attempt → reruns failed jobs" { + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + _reran + grep -q "rerun" "$RERUN_MARKER" + grep -q "$RUN_ID" "$RERUN_MARKER" +} + +@test "auto-rebase-retry: success conclusion → noop, no rerun" { + export CONCLUSION="success" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran +} + +@test "auto-rebase-retry: cancelled conclusion → noop, no rerun" { + export CONCLUSION="cancelled" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran +} + +@test "auto-rebase-retry: attempt at cap → no rerun, gives up with warning" { + export RUN_ATTEMPT="3" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran + [[ "$output" == *"::warning::"* ]] + grep -qi "exhaust\|gave up\|giving up\|manual" "$GITHUB_STEP_SUMMARY" +} + +@test "auto-rebase-retry: attempt past cap → no rerun" { + export RUN_ATTEMPT="9" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran +} + +@test "auto-rebase-retry: last allowed attempt (cap-1) → reruns" { + export RUN_ATTEMPT="2" + export MAX_ATTEMPTS="3" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + _reran +} + +@test "auto-rebase-retry: missing RUN_ID → noop, no rerun" { + export RUN_ID="" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran +} + +@test "auto-rebase-retry: non-numeric RUN_ATTEMPT defaults to 1 → reruns" { + export RUN_ATTEMPT="notanumber" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + _reran +} + +@test "auto-rebase-retry: DRY_RUN → logs intent, does not rerun" { + export DRY_RUN="true" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + ! _reran + [[ "$output" == *"dry-run"* ]] +} + +@test "auto-rebase-retry: rerun API failure → warns but exits 0" { + export GH_STUB_RERUN_FAILS="1" + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + _reran + [[ "$output" == *"::warning::"* ]] +} From c366c625ec40ade8495d5ddab67b4f3088cc0e35 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sat, 13 Jun 2026 17:44:15 +0000 Subject: [PATCH 2/2] fix(reviews): address review comments [skip ci-relay] --- scripts/auto-rebase-retry.sh | 6 +++--- tests/dev-lead/unit/test_auto_rebase_retry.bats | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/auto-rebase-retry.sh b/scripts/auto-rebase-retry.sh index 750265e43..5eff3cb89 100755 --- a/scripts/auto-rebase-retry.sh +++ b/scripts/auto-rebase-retry.sh @@ -41,7 +41,7 @@ MAX_ATTEMPTS="${MAX_ATTEMPTS:-3}" DRY_RUN="${DRY_RUN:-false}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" -summary() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; } +summary() { echo "$1" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true; } # Default any non-integer attempt to 1 so a malformed payload still retries once # rather than silently skipping or crashing under `set -e`. @@ -54,7 +54,7 @@ if ! [[ "$MAX_ATTEMPTS" =~ ^[0-9]+$ ]] || [ "$MAX_ATTEMPTS" -lt 1 ]; then fi # ── guard: only act on failed runs ──────────────────────────────────────────── -if [ "$CONCLUSION" != "failure" ]; then +if [[ "$CONCLUSION" != "failure" ]]; then echo "::notice::${WORKFLOW_NAME} run concluded '${CONCLUSION:-unknown}' (not failure) — nothing to retry" exit 0 fi @@ -84,7 +84,7 @@ if [ "$DRY_RUN" = "true" ]; then exit 0 fi -if gh run rerun "$RUN_ID" --failed --repo "$REPO"; then +if gh run rerun "$RUN_ID" --failed ${REPO:+--repo "$REPO"}; then echo "::notice::Re-run of run ${RUN_ID} requested (attempt ${next_attempt}/${MAX_ATTEMPTS})" summary "### Auto-rebase retry dispatched" summary "Re-ran failed jobs of run [\`${RUN_ID}\`](${HTML_URL}) — attempt ${next_attempt}/${MAX_ATTEMPTS}." diff --git a/tests/dev-lead/unit/test_auto_rebase_retry.bats b/tests/dev-lead/unit/test_auto_rebase_retry.bats index 8996a5aa4..4fd2fbf4c 100644 --- a/tests/dev-lead/unit/test_auto_rebase_retry.bats +++ b/tests/dev-lead/unit/test_auto_rebase_retry.bats @@ -124,3 +124,12 @@ _reran() { [ -f "$RERUN_MARKER" ]; } _reran [[ "$output" == *"::warning::"* ]] } + +@test "auto-rebase-retry: empty REPO → reruns without --repo flag" { + export REPO="" + unset GITHUB_REPOSITORY + run bash "$RETRY_SCRIPT" + [ "$status" -eq 0 ] + _reran + ! grep -q -- "--repo" "$RERUN_MARKER" +}