Skip to content
Merged
53 changes: 16 additions & 37 deletions scripts/test-pr-review-workflow.sh
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
#!/usr/bin/env bash
# Regression guard: assert that pr-review.yml serializes redundant runs for the
# same PR head commit via a top-level `concurrency:` block. Its overlapping
# triggers (check_suite:completed, pull_request:synchronize,
# pull_request_review) all fire for one commit; without concurrency they launch
# parallel duplicate reviews, inflating duration and producing cancelled runs
# (Fleet Monitor #374). Mirrors the convention of every sibling caller
# (ci-failure-analyst.yml, add-to-project.yml, ...).
# Regression guard: assert that pr-review.yml matches the invariants of the org
# template it is synced from (petry-projects/.github
# standards/workflows/pr-review.yml): a review job that skips Dependabot-
# triggered events (#465).
#
# Run: bash scripts/test-pr-review-workflow.sh
#
Expand Down Expand Up @@ -33,40 +30,22 @@
fi
echo "PASS: $WORKFLOW exists"

# ── Check 2: a top-level concurrency block is present ──────────────────────
group=$(yq '.concurrency.group' "$WORKFLOW" 2>/dev/null)
if [[ "$group" == "null" || -z "$group" ]]; then
echo "FAIL: no top-level 'concurrency.group' block in $WORKFLOW"
echo " Add one to serialize duplicate reviews of the same head SHA."
PASS=false
# ── Check 2: top-level concurrency block (report only) ─────────────────────
# The org template (petry-projects/.github standards/workflows/pr-review.yml)
# has no stub-level block (the engine owns concurrency), while this repo's
# stub may still carry one until the standards sync lands. Both are accepted;
# the block is reported but never fails the guard (#533).
has_concurrency=$(yq 'has("concurrency")' "$WORKFLOW" 2>/dev/null || echo "error")

Check warning on line 38 in scripts/test-pr-review-workflow.sh

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Redirect this error message to stderr (>&2).

See more on https://sonarcloud.io/project/issues?id=petry-projects_TalkTerm&issues=AaEpC93hlUioR2U_tliF&open=AaEpC93hlUioR2U_tliF&pullRequest=535
Comment thread
don-petry marked this conversation as resolved.
Resolved
if [[ "$has_concurrency" == "false" ]]; then
echo "PASS: no stub-level 'concurrency' block in $WORKFLOW"
elif [[ "$has_concurrency" == "true" ]]; then
echo "INFO: top-level 'concurrency' block present in $WORKFLOW (allowed until the standards sync)"
else
echo "PASS: 'concurrency.group' block present in $WORKFLOW"
fi

# ── Check 3: the group is keyed on the PR head commit ──────────────────────
# Grouping per head SHA lets a new commit start its own run while redundant
# events for the *same* commit share a lane.
if [[ "$PASS" == "true" ]]; then
if [[ ! "$group" =~ head\.sha && ! "$group" =~ head_sha ]]; then
echo "FAIL: concurrency group ($group) is not keyed on the PR head SHA in $WORKFLOW"
PASS=false
else
echo "PASS: concurrency group keyed on head SHA in $WORKFLOW"
fi
fi

# ── Check 4: cancel-in-progress is false ───────────────────────────────────
# An in-flight review must never be interrupted; redundant same-SHA runs queue
# and the reusable's idempotency no-ops them.
cancel=$(yq '.concurrency.cancel-in-progress' "$WORKFLOW" 2>/dev/null)
if [[ "$cancel" != "false" ]]; then
echo "FAIL: 'concurrency.cancel-in-progress' is not 'false' (found: '$cancel') in $WORKFLOW"
echo "FAIL: could not parse $WORKFLOW to check for a 'concurrency' block"
PASS=false
else
echo "PASS: 'concurrency.cancel-in-progress: false' present in $WORKFLOW"
fi

# ── Check 5: the review job skips Dependabot-triggered events ──────────────
# ── Check 3: the review job skips Dependabot-triggered events ──────────────
# Dependabot-triggered runs read the separate Dependabot secret store, so
# `secrets: inherit` forwards empty PAT/OAuth secrets and the reusable's
# "Verify auth scopes" step fails outright (Fleet Monitor #465). Guarding the
Expand Down
136 changes: 110 additions & 26 deletions scripts/test-pr-review-workflow.test.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
#!/usr/bin/env bash
# test-pr-review-workflow.test.sh — portable tests for the pr-review.yml
# regression guard (scripts/test-pr-review-workflow.sh). The guard asserts the
# invariants that keep the PR Review workflow's Fleet Monitor failure rate low:
# a head-SHA concurrency lane with cancel-in-progress: false (#374) AND a
# review-job guard that skips Dependabot-triggered runs (#465). Dependabot
# events read the separate Dependabot secret store, so `secrets: inherit`
# forwards empty PAT/OAuth secrets and the reusable's auth-scope check fails;
# skipping the job makes those runs neutral instead of failed.
# invariants of the thin caller stub synced from the org template
# (petry-projects/.github standards/workflows/pr-review.yml): a review-job
# guard that skips Dependabot-triggered runs (#465); a top-level `concurrency:`
# block is neither required nor forbidden (#533). Dependabot events read the separate Dependabot secret store, so
# `secrets: inherit` forwards empty PAT/OAuth secrets and the reusable's
# auth-scope check fails; skipping the job makes those runs neutral instead of
# failed.
# No bats dependency: the guard is driven as a subprocess against temporary
# fixture workflows.
# Run: bash scripts/test-pr-review-workflow.test.sh
Expand Down Expand Up @@ -43,10 +44,10 @@ fi
trap 'rm -rf "$TMP"' EXIT

# ── Fixture builder ─────────────────────────────────────────────────────────
# Emits a minimal pr-review caller with a valid head-SHA concurrency lane
# (cancel-in-progress: false). The review job's `if:` guard is parameterised so
# each case isolates exactly the Dependabot-skip invariant (Check 5). Passing an
# empty string omits the guard entirely.
# Emits a minimal pr-review caller with no stub-level concurrency block (the
# template's rule). The review job's `if:` guard is parameterised so each case
# isolates exactly the Dependabot-skip invariant. Passing an empty string omits
# the guard entirely.
write_workflow() {
local file="$1"
local job_if="${2-}"
Expand All @@ -56,12 +57,6 @@ on:
pull_request:
types: [opened, synchronize]
permissions: {}
concurrency:
group: >-
pr-review-${{
github.event.pull_request.head.sha ||
github.run_id }}
cancel-in-progress: false
jobs:
review:
YAML
Expand Down Expand Up @@ -129,17 +124,23 @@ else
pass "an if: that runs only on dependabot[bot] (== not !=) is rejected"
fi

# ── Case 6: the concurrency invariant is still enforced (regression, #374) ──
# Drop cancel-in-progress to confirm the pre-existing checks were not weakened
# by the Dependabot guard addition.
nocancel="${TMP}/no-cancel.yml"
write_workflow "$nocancel" "\${{ github.actor != 'dependabot[bot]' }}"
# Flip cancel-in-progress: false -> true.
sed 's/cancel-in-progress: false/cancel-in-progress: true/' "$nocancel" > "${nocancel}.tmp" && mv "${nocancel}.tmp" "$nocancel"
if run_guard "$nocancel"; then
fail "cancel-in-progress: true should be REJECTED (concurrency invariant)"
# ── Case 6: a stub with a concurrency block is accepted (#533) ─────────
# A top-level concurrency block is tolerated (reported, not failed) until the
# standards sync removes it; main's stub has one.
withconc="${TMP}/with-concurrency.yml"
write_workflow "$withconc" "\${{ github.actor != 'dependabot[bot]' }}"
cat >> "$withconc" <<'YAML'
concurrency:
group: >-
pr-review-${{
github.event.pull_request.head.sha ||
github.run_id }}
cancel-in-progress: false
YAML
if run_guard "$withconc"; then
pass "a stub with a top-level concurrency block is accepted"
else
pass "cancel-in-progress: true is rejected (concurrency invariant intact)"
fail "a stub with a top-level concurrency block should be ACCEPTED"
fi

# ── Case 7: a missing workflow file fails cleanly ──────────────────────────
Expand All @@ -162,6 +163,89 @@ else
pass "an always-true if: guard is rejected (exact-match check rejects always-true expressions)"
fi

# ── Case 9: the org template itself is accepted (#533) ─────────────────────
# Vendored copy of petry-projects/.github standards/workflows/pr-review.yml
# (fetched 2026-10-11), inlined so the test needs no network access or token.
# Every standards sync replaces .github/workflows/pr-review.yml with this file,
# so the guard must accept it. Refresh this copy when the template changes.
template="${TMP}/org-template.yml"
cat > "$template" <<'YAML'
# ─────────────────────────────────────────────────────────────────────────────
# SOURCE OF TRUTH: petry-projects/.github/standards/workflows/pr-review.yml
# Standard: petry-projects/.github/standards/ci-standards.md
# Reusable: petry-projects/.github-private/.github/workflows/pr-review.yml
# (the pr-review ENGINE — grandfathered name, not *-reusable.yml)
#
# AGENTS — READ BEFORE EDITING:
# • This file is a THIN CALLER STUB. All review logic lives in the engine
# above, including concurrency (per PR + head SHA, never cancelling an
# in-flight review), the check_suite-without-a-PR skip, and reading
# `client_payload.pr_url` from repository_dispatch.
# • You MUST NOT change: the `@pr-review/v1-stable` channel in the `uses:`
# line or the matching `agent_ref` — both are re-pinned in lockstep to this
# repo's canary-ring tier by the standards sweep. Nor the workflow `name:`,
# the job id `review` (check names depend on both), the trigger event types,
# the Dependabot skip guard, or the job-level `permissions:` block. Do not
# add a stub-level `concurrency:` block.
# • The standards sweep only REPLACES an existing caller with this file; it
# never adds it to a repo that has none, and never touches
# petry-projects/.github-private (whose pr-review.yml is the engine).
# • If you need different behaviour, open a PR against the engine in
# petry-projects/.github-private. It reaches every repo when the channel
# tag is promoted.
# ─────────────────────────────────────────────────────────────────────────────

name: PR Review Agent

on:
check_suite:
types: [completed]
pull_request_review:
types: [submitted, dismissed]
pull_request:
types: [ready_for_review, reopened, synchronize]
workflow_dispatch:
inputs:
pr_url:
description: "Optional: review a single PR URL instead of enumerating"
required: false
type: string
dry_run:
description: "If true, never submit reviews or comments"
required: false
default: "false"
type: string
force_review:
description: "If true, bypass idempotency and re-review at the same head SHA"
required: false
default: "false"
type: string
repository_dispatch:
types: [pr-review-mention]

permissions: {}

jobs:
review:
if: ${{ github.actor != 'dependabot[bot]' }}
permissions:
contents: read
pull-requests: write
checks: read
uses: petry-projects/.github-private/.github/workflows/pr-review.yml@pr-review/v1-stable # NOSONAR(githubactions:S7637) first-party channel ref
with:
agent_ref: pr-review/v1-stable
pr_url: ${{ inputs.pr_url || '' }}
dry_run: ${{ inputs.dry_run || '' }}
force_review: ${{ inputs.force_review || '' }}
secrets: inherit # NOSONAR(githubactions:S7635) first-party trusted reusable
YAML
if run_guard "$template"; then
pass "the org template standards/workflows/pr-review.yml is accepted"
else
fail "the org template standards/workflows/pr-review.yml should be ACCEPTED"
fi

echo ""
if [[ "$fails" -eq 0 ]]; then
echo "All tests passed."
Expand Down
Loading