diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 3334bb539..b1195b705 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -142,6 +142,11 @@ jobs: # what caused the donpetry-bot regression (#1316); the manifest is now the # sole source of truth. See scripts/lib/resolve-persona-identity.sh. BOT_USER: 'don-petry' + # Repo-named test command for the pre-push regression guard + # (scripts/lib/test-regression-guard.sh, #2013). Mapped from a repository + # variable — NOT a workflow_call input — so no channel-pin sequencing is + # needed. Empty => the guard auto-discovers (npm/make/pytest/bats). + DEV_LEAD_TEST_CMD: ${{ vars.DEV_LEAD_TEST_CMD }} # Empty by default so dev-lead-intent.sh derives the trusted set from the # reviewer-source registry (scripts/lib/reviewer-sources.tsv), the single # source of truth — no hardcoded list to drift from the gate/scorecard @@ -447,6 +452,12 @@ jobs: fi ;; esac + # bats is not preinstalled on ubuntu-latest; the test-regression guard's + # `bats --recursive tests` fallback needs it (#2013). Same install as lint.yml. + - name: Install bats + if: env.INTENT_TYPE == 'fix-reviews' || env.INTENT_TYPE == 'fix-bot-comment' || env.INTENT_TYPE == 'review-changes' || env.INTENT_TYPE == 'human-pr' + run: sudo apt-get install -y bats + # ── fix-ci ──────────────────────────────────────────────────────────── - name: Run fix-ci if: env.INTENT_TYPE == 'fix-ci' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e6cdb445a..a23e18dad 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -216,6 +216,9 @@ jobs: tests/dev-lead/unit/test_comment_disposition_verify.bats \ tests/dev-lead/unit/test_conflict_integrity.bats \ tests/dev-lead/unit/test_addressed_claim_verify.bats \ + tests/dev-lead/unit/test_claim_landing.bats \ + tests/dev-lead/unit/test_test_tamper_guard.bats \ + tests/dev-lead/unit/test_test_regression_guard.bats \ tests/dev-lead/unit/test_fewshot_assemble.bats \ tests/dev-lead/unit/test_net_diff_guard.bats \ tests/dev-lead/unit/test_verify_approval_review.bats \ diff --git a/docs/architecture/adr/0011-write-mode-personas-on-a-shared-branch.md b/docs/architecture/adr/0011-write-mode-personas-on-a-shared-branch.md new file mode 100644 index 000000000..4b26e67f6 --- /dev/null +++ b/docs/architecture/adr/0011-write-mode-personas-on-a-shared-branch.md @@ -0,0 +1,94 @@ +# 0011. Write-mode personas on a shared branch: verify what landed, retract what did not + +## Status + +proposed + +## Context + +ADR-0005 makes write access an opt-in gate: dev-lead's write surfaces are armed +by the `dev-lead` label. It does not say how a write-mode persona should behave +on a branch that **other actors are pushing to at the same time**, such as a +maintainer, a Claude session, or the auto-rebase bot. + +petry-projects/.github#1220 showed what goes wrong +([#2013](https://github.com/petry-projects/.github-private/issues/2013)). While +a human session was driving the same branch: + +- dev-lead posted "Fixed in …" replies for changes that never reached the + branch. Its claim named the PR's **first** commit: the head the model read + with `git rev-parse HEAD` *before* committing anything, because the prompt + told it to cite that SHA and also told it not to commit. +- CodeRabbit marked a thread "Confirmed as addressed" on the strength of a + reply describing `15a919e`, which *was* pushed. A wrong fix cleared a review + gate: it added a new test and broke an existing one without editing it, and + the persona never ran the suite against the pre-pass baseline. +- A bot suggestion ("update the older test so the suite can pass") was applied + literally. That rewrote an existing test and inverted deliberate behavior. + +The no-clobber push (#1311/#1607) already prevented dev-lead from overwriting +another actor's commit. Everything *around* the push was still the model +describing itself. The #1692 claim check ran only when the harness resolved a +thread, and the pre-pass SHA passed it trivially. Nothing ever took a posted +claim back. That breaks ADR-0004's "Fail Loud, Never Fake". + +## Decision + +We will make every write-mode persona sharing a branch follow five rules, each +enforced by harness checks with bats tests (pure classifiers where applicable, +per ADR-0004), not by prompt text alone: + +1. **Incorporate or stand down, never force.** Before pushing, the persona + reconciles with the *true* remote head. It rebases a foreign commit in, or + stops and escalates when that cannot be done cleanly. A foreign commit is + never discarded (`push_no_clobber`, `scripts/lib/git-push-guard.sh`). +2. **"Pushed" is a verdict, not an assumption.** After the push, the harness + compares the pre-pass head, the pushed SHA and the re-fetched remote head + (`cl_push_landed_verdict`, `scripts/lib/claim-landing.sh`). Any result other + than `landed` counts as a failed push. +3. **A claim must name a commit this pass produced and that landed.** The claim + SHA must be on the reference head and **not** contained in the pre-pass + head (`acv_claim_in_pass`, `scripts/lib/addressed-claim-verify.sh`). The + harness checks this at thread-resolution time against the PR head. It + checks it again after every push outcome against the **remote** head. Every + claim reply posted during the pass that fails is **retracted**: both markers + are stripped and a retraction notice is added, so the reply can never + authorize resolution and other bots cannot read it as a fix. The outcomes + covered are a rejected or unverified push, a guard abort, an engine + failure, no commit, and a stale SHA. +4. **An existing test is not the persona's to rewrite.** A fix pass that + changes or deletes an existing test line, or adds a skip, without an + explicit cited `Test-Change-Justification:` trailer is refused before the + push and escalated to a human (`scripts/lib/test-tamper-guard.sh`). A bot + suggestion that contradicts an existing test goes to a human. +5. **A red suite blocks the push.** Before the push, the harness runs the + repo's test command on the pass's result. A test that passed on the + pre-pass head and fails now refuses the push and escalates like the tamper + guard (`scripts/lib/test-regression-guard.sh`). A failure that already + existed on the pre-pass head does not block. When no test command can be + determined, the run summary says the suite was not run; it never implies + green. + +The boundary a check can assert: **no dev-lead claim reply survives a pass +unless the claim's commit was produced by that pass and is on the remote +head.** + +## Consequences + +- A rejected, aborted or failed pass leaves retracted replies, not false + "Fixed" ones. The thread stays open and the next pass re-verifies. +- The fail-closed direction costs some true claims. When the push guard + rebases dev-lead's commits onto a foreign commit, the SHAs the model cited + no longer exist on the remote, so those claims are retracted even though the + fix landed. Re-mapping rebased SHAs is a possible follow-up. A stuck thread + is a nuisance; a wrongly cleared gate is a defect. +- The model must commit locally before it replies, so its claims can cite a + real commit. The prompts now say so. A pass that leaves its changes + uncommitted gets them committed by the harness, and its claims (citing the + pre-pass head) are retracted. +- Legitimate test changes need one extra line, the justification trailer. In + exchange, a bot-driven pass can no longer silently invert a tested behavior. +- Still open: whether a persona should *back off entirely* from a branch with + recent non-persona pushes, or require an opt-in label to push to a branch it + did not create. This ADR covers correctness of what is claimed. It does not + cover whether to participate at all. diff --git a/prompts/dev-lead/fix-bot-comment.md b/prompts/dev-lead/fix-bot-comment.md index 6e4897614..ab40ae9ff 100644 --- a/prompts/dev-lead/fix-bot-comment.md +++ b/prompts/dev-lead/fix-bot-comment.md @@ -81,11 +81,11 @@ gh api graphql -f query=' ))))' ``` -For each thread you fixed, first **reply with the specific change** — name the file(s)/function(s) you touched and how the change addresses the finding (one or two concrete sentences; never just "done"). End the reply with **two** HTML comments: the addressed-marker `` **and** a machine-readable claim `` (#1692). The harness now verifies the claim against the pushed diff before resolving — a marker without a verifiable claim leaves the thread **unresolved**. Stamp both **only** on a genuine addressed reply, never on a skip note. Pass the body as a GraphQL variable so quotes and newlines are safe: +For each thread you fixed, first **commit the fix locally** (`git add -A && git commit -m "fix(bot): "`, never push; if an earlier commit this pass already captured this fix the tree is clean — skip the `git commit` and cite that commit's SHA, since `git commit` on a clean tree fails), then **reply with the specific change** — name the file(s)/function(s) you touched and how the change addresses the finding (one or two concrete sentences; never just "done"). End the reply with **two** HTML comments: the addressed-marker `` **and** a machine-readable claim `` (#1692). The harness now verifies the claim against the pushed diff before resolving — a marker without a verifiable claim leaves the thread **unresolved**. Stamp both **only** on a genuine addressed reply, never on a skip note. Pass the body as a GraphQL variable so quotes and newlines are safe: ```bash # Replace THREAD_NODE_ID with the id value from the query above. -# Get the full 40-char head SHA the fix rides on with: git rev-parse HEAD +# Commit the fix FIRST, then read the SHA of that commit with: git rev-parse HEAD gh api graphql \ -f query='mutation($tid: ID!, $body: String!) { addPullRequestReviewThreadReply(input: {pullRequestReviewThreadId: $tid, body: $body}) { comment { id } } }' \ -f tid="THREAD_NODE_ID" \ @@ -95,7 +95,11 @@ gh api graphql \ " ``` -The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` (`git rev-parse HEAD`) and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. +The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. + +**Commit before you claim (#2013).** `sha` must be a commit **this pass produced**: run `git rev-parse HEAD` *after* you commit the fix. Citing the head you started from never verifies; it is the stale-claim defect from petry-projects/.github#1220. After the push, the harness checks the remote head. If the push was rejected or not incorporated, a guard refused it, or the pass failed, **every claim reply you posted this pass is retracted**. If you make no commit for a thread, post no addressed-marker or claim on it. + +**Never rewrite an existing test to make a bot suggestion pass (#2013).** Run the **full** test suite. A previously-passing test that turns red is a reason to question the change, not the test. A bot suggestion that contradicts an existing test is not applied, and that includes "update the older test so the suite can pass". Reply explaining the conflict, **without** the addressed-marker, and leave it for a human. Changing or deleting an existing test line, or adding a `skip`, requires a cited `Test-Change-Justification: ` commit trailer. Without one, the harness's test-tamper guard refuses to push and escalates. Adding a new test never needs it. Because `git commit -m` takes one message line, pass the trailer as a second `-m` paragraph: `git commit -m "fix(bot): " -m "Test-Change-Justification: , a commit SHA, or the review URL>"`. The justification must be at least 20 characters and contain such a reference. **Do not resolve the thread yourself.** You must not call the `resolveReviewThread` (or `unresolveReviewThread`) GraphQL mutation under any circumstance — resolution is done **only** by the harness (`dev-lead-fix-reviews.sh`), whose deterministic guards are the authoritative merge gate. The harness resolves every bot thread you addressed with an our-account addressed-marker reply, plus any thread from this bot marked `isOutdated: true`. Your reply and its marker are your only lever on resolution. @@ -203,7 +207,8 @@ Some bot comments are pure **operational notices**, not code findings: a rate-li - For every thread you fix, post a reply naming the specific change, ending with the addressed-marker — never reply-less - **Never resolve a thread yourself**: do not call the `resolveReviewThread` (or `unresolveReviewThread`) mutation. Resolution is the harness's job — it resolves the addressed and outdated threads from `${ACTOR}`; your only lever is the addressed-marker on your reply - Stay within the scope of the pull request's changed files where possible -- Do not commit or push — the CI workflow handles git operations after you finish +- Commit your fixes locally so your claims can cite them, but **never push**. The CI workflow pushes after you finish, verifies the push landed, and retracts any claim that did not land (#2013) +- Never edit an existing test to match your change. A conflicting bot suggestion goes to a human. An existing-test change needs a `Test-Change-Justification:` trailer (#2013) ## Output Format diff --git a/prompts/dev-lead/fix-reviews.md b/prompts/dev-lead/fix-reviews.md index 4c46b726f..aa88c3ddf 100644 --- a/prompts/dev-lead/fix-reviews.md +++ b/prompts/dev-lead/fix-reviews.md @@ -55,8 +55,9 @@ For each open review thread: 1. Read the relevant file(s) using Read/Grep/Glob tools 2. Understand the reviewer's concern 3. Apply the appropriate fix using Edit/Write tools -4. **Reply to the thread with the specific fix** — see below -5. **Do not resolve the thread yourself** — the harness resolves it (see "Resolution is the harness's responsibility" below). A marker-less human thread is never resolved, including one with `isOutdated: true` (outdated status never overrides marker ownership). +4. **Commit the fix locally** — `git add -A && git commit -m "fix(reviews): "` — **before** you reply, so the claim can name a commit this pass actually produced (see "Commit before you claim" below). Never push. +5. **Reply to the thread with the specific fix** — see below +6. **Do not resolve the thread yourself** — the harness resolves it (see "Resolution is the harness's responsibility" below). A marker-less human thread is never resolved, including one with `isOutdated: true` (outdated status never overrides marker ownership). #### Replying to a thread @@ -64,7 +65,7 @@ For every thread you fix, post a reply to that thread that states **specifically ```bash # Replace THREAD_NODE_ID with the id value from the thread JSON. -# Get the full 40-char head SHA the fix rides on with: git rev-parse HEAD +# Commit the fix FIRST, then read the SHA of that commit with: git rev-parse HEAD gh api graphql \ -f query='mutation($tid: ID!, $body: String!) { addPullRequestReviewThreadReply(input: {pullRequestReviewThreadId: $tid, body: $body}) { comment { id } } }' \ -f tid="THREAD_NODE_ID" \ @@ -79,11 +80,20 @@ gh api graphql \ | Field | Type | Rule | |---|---|---| | `v` | integer | Schema version — `1` today. A payload with any other `v` is unverifiable → the thread stays open. | -| `sha` | string | The **full 40-character** commit SHA your fix rides on (`git rev-parse HEAD`). Abbreviated SHAs are rejected. | +| `sha` | string | The **full 40-character** SHA of the commit **you made in this pass** that carries the fix (`git rev-parse HEAD` **after** committing it). Abbreviated SHAs are rejected. | | `files` | array of strings | The repo-relative POSIX path(s) your fix touches, **exactly as they appear in the diff** (no leading `./` or `/`, no quoting). Must be non-empty. It is a **JSON array** — never comma- or newline-separated. | Emit **exactly one** claim comment per reply — the harness treats zero or more-than-one as unverifiable and leaves the thread open. The harness then checks that the named commit is reachable from the PR head, its diff (or the cumulative `^..HEAD` range) is non-empty, and it touches at least one file in `files`. If any check fails, the thread stays unresolved — so name the real SHA and the real files. +#### Commit before you claim — a claim is checked, then kept or retracted (#2013) + +The harness checks every claim against what actually **landed**, not against what you meant to do: + +- The cited commit must be one **this pass produced**: it must be on the PR head and **not** already on the branch when the pass started. Citing the head you started from (`git rev-parse HEAD` before you committed anything) is the stale-claim defect from petry-projects/.github#1220. It never verifies. +- After pushing, the harness checks that the **remote** head contains the pushed commit. If the push was rejected, the remote moved and your commit could not be incorporated, a guard refused the push, or the pass failed, then **every claim reply you posted this pass is retracted**: the markers are stripped and a retraction notice is added. An unverified "Fixed" must never stand, because other review bots treat it as addressed. + +So: edit, **commit locally**, then reply citing that commit's SHA. If you end up making no commit for a thread, post no addressed-marker or claim on it. + For a thread that is `isOutdated: true` with no code change, a reply is optional — a one-line note that the referenced code no longer exists is helpful but not required. #### Resolution is the harness's responsibility — never call `resolveReviewThread` @@ -154,7 +164,7 @@ For **every other** comment (bot or human alike — a bot conflict report or tri | Disposition | Use when | Required evidence (harness-verified) | |---|---|---| -| `fixed` | you changed code to address the finding | `sha=` the **full 40-char** commit (`git rev-parse HEAD`) your fix rides on; the harness checks it is on the PR head with a non-empty diff | +| `fixed` | you changed code to address the finding | `sha=` the **full 40-char** SHA of the commit **you made in this pass** (`git rev-parse HEAD` *after* committing the fix locally); the harness checks it is on the PR head, was produced by this pass, and has a non-empty diff | | `out-of-scope` | the finding is real but belongs elsewhere | `ref=#` a tracking issue that **exists**; open one first if needed | | `invalid` | the finding is wrong / a false positive | a reply body with concrete reasoning (non-empty beyond the marker) | | `answered` | the comment asked a question you answer in the reply | a reply body that actually answers it | @@ -173,15 +183,20 @@ For **every other** comment (bot or human alike — a bot conflict report or tri After addressing all threads, run the test suite to ensure no regressions were introduced: 1. Identify the test command this repo uses (check AGENTS.md, `package.json`, `Makefile`, etc.) -2. Run the full test suite — all tests must pass -3. If a thread fix required adding new behavior, add or update tests to cover it +2. Run the **full** test suite, not only the tests near your change. All tests must pass. +3. If a thread fix required adding new behavior, **add** a test to cover it 4. **Do not suppress or delete tests to force a pass — fix the code instead** +5. **A previously-passing test that turns red is a signal about your change, not about the test.** Question the change first. An existing test often encodes a deliberate behavior. On petry-projects/.github#1220 a "success precedence" test that encoded deliberate recovery semantics was rewritten to "failure precedence" to match a bot's suggestion, which inverted that behavior. +6. **A bot suggestion that contradicts an existing test is not applied.** That includes a literal "update the old test so the suite passes". Reply on the thread explaining the conflict and naming the test, **without** the addressed-marker, so the thread stays open for a human. Do not rewrite the test. +7. **Changing or deleting an existing test line, or adding a `skip`, needs an explicit, cited justification** in a commit trailer: + `Test-Change-Justification: ` + The harness's test-tamper guard refuses to push a pass that changes an existing test without this trailer, and escalates it to a human. Adding a **new** test never needs it. ### Phase 3 — Rubber Duck Review Read every changed line as if you are the reviewer seeing the response: -1. Run `git diff HEAD` (or equivalent) to see all changes made this session +1. Run `git diff "$(git merge-base HEAD @{u})"` (or diff against the pre-pass head) to see all changes made this session — step 4 commits fixes locally, so a plain `git diff HEAD` no longer shows them 2. Ask: does each change directly and completely address its thread? 3. Ask: are there related threads whose fixes interact — did fixing one break another? 4. Ask: would the reviewer be satisfied, or is there still an issue? @@ -197,7 +212,8 @@ Read every changed line as if you are the reviewer seeing the response: - Do not make changes beyond what the review threads request, except that fixing Tier-1 blockers (failure/timed_out/cancelled/action_required/stale/startup_failure CI checks and CHANGES_REQUESTED reviews) is always in-scope - Never revert or undo the PR's own committed changes to satisfy a neutral `COMMENTED`/overview review — that produces a net-zero diff that silently cancels the fix (#1340) - If a review thread is ambiguous, apply the most conservative interpretation -- Do not commit or push — the CI workflow handles git operations after you finish +- Commit your fixes locally so your claims can cite them, but **never push**. The CI workflow pushes after you finish, verifies the push landed, and retracts any claim that did not land (#2013) +- Never edit an existing test to match your change. A bot suggestion that contradicts an existing test goes to a human (reply without the marker). An existing-test change needs a `Test-Change-Justification:` trailer (#2013) ## Output Format diff --git a/prompts/dev-lead/review-changes.md b/prompts/dev-lead/review-changes.md index c2102e7cb..ef50d1d1a 100644 --- a/prompts/dev-lead/review-changes.md +++ b/prompts/dev-lead/review-changes.md @@ -67,9 +67,10 @@ For each open review thread: 1. Read each thread carefully to understand the reviewer's intent 2. Use Read/Grep/Glob tools to examine the referenced code and surrounding context 3. Apply the requested changes using Edit/Write tools -4. If a thread requests new behavior that has no existing test coverage, write a test for it **before** making the implementation change -5. **Reply to the thread with the specific fix** — see below -6. **Do not resolve the thread yourself** — the harness resolves it after your reply (see below) +4. If a thread requests new behavior that has no existing test coverage, **add** a test for it **before** making the implementation change +5. **Commit the fix locally** — `git add -A && git commit -m "fix(reviews): "` — **before** you reply, so the claim can name a commit this pass actually produced (see "Commit before you claim" below). Never push. +6. **Reply to the thread with the specific fix** — see below +7. **Do not resolve the thread yourself** — the harness resolves it after your reply (see below) #### Replying to a thread @@ -77,7 +78,7 @@ For every thread you fix, post a reply that states **specifically what you chang ```bash # Replace THREAD_NODE_ID with the id value from the thread JSON. -# Get the full 40-char head SHA the fix rides on with: git rev-parse HEAD +# Commit the fix FIRST, then read the SHA of that commit with: git rev-parse HEAD gh api graphql \ -f query='mutation($tid: ID!, $body: String!) { addPullRequestReviewThreadReply(input: {pullRequestReviewThreadId: $tid, body: $body}) { comment { id } } }' \ -f tid="THREAD_NODE_ID" \ @@ -87,7 +88,9 @@ gh api graphql \ " ``` -The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` (`git rev-parse HEAD`) and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. +The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` of the commit **you made in this pass** (`git rev-parse HEAD` *after* committing the fix) and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. + +**Commit before you claim (#2013).** `sha` must be a commit **this pass produced**: run `git rev-parse HEAD` *after* you commit the fix. Citing the head you started from never verifies; it is the stale-claim defect from petry-projects/.github#1220. After the push, the harness checks the remote head. If the push was rejected or not incorporated, a guard refused it, or the pass failed, **every claim reply you posted this pass is retracted**. If you make no commit for a thread, post no addressed-marker or claim on it. #### Resolution is the harness's responsibility — never call `resolveReviewThread` @@ -100,15 +103,16 @@ The harness never resolves a marker-less human (maintainer) thread — it leaves After addressing all threads: 1. Identify the test command this repo uses (check AGENTS.md, `package.json`, `Makefile`, etc.) -2. Run the full test suite — every test must pass, not just the changed areas +2. Run the **full** test suite — every test must pass, not just the changed areas 3. Run any available lint/format checks 4. **Do not suppress or delete tests to force a pass — fix the code instead** +5. **Never rewrite an existing test to make a bot suggestion pass (#2013).** A previously-passing test that turns red is a reason to question the change, not the test. A suggestion that contradicts an existing test is not applied, including "update the older test so the suite can pass". Reply explaining the conflict, **without** the addressed-marker, and leave it for a human. Changing or deleting an existing test line, or adding a `skip`, requires a cited `Test-Change-Justification: ` commit trailer; without one the harness's test-tamper guard refuses to push and escalates. Adding a new test never needs it. The harness also re-runs the suite before pushing and refuses a pass that breaks a test that passed on the pre-pass head. ### Phase 3 — Rubber Duck Review Read all your changes from the reviewer's perspective: -1. Run `git diff HEAD` (or equivalent) to see every line changed this session +1. Diff against the pre-pass head (for example `git diff "$(git merge-base HEAD @{u})"`) to see every line changed this session — step 5 commits fixes locally, so a plain `git diff HEAD` no longer shows them 2. For each thread, ask: does this change fully satisfy what the reviewer requested? 3. Ask: if multiple threads conflict, was priority applied correctly (security > correctness > style)? 4. Ask: would this response prompt further review comments, or is it clean? @@ -125,7 +129,8 @@ Read all your changes from the reviewer's perspective: - For a thread you are intentionally skipping, post a skip note **without** the addressed-marker and explain why in your output - If multiple threads conflict, prioritize: security > correctness > style - Maintain the existing code style and patterns -- Do not commit or push — the CI workflow handles git operations after you finish +- Commit your fixes locally so your claims can cite them, but **never push**. The CI workflow pushes after you finish, verifies the push landed, and retracts any claim that did not land (#2013) +- Never edit an existing test to match your change; an existing-test change needs a `Test-Change-Justification:` trailer (#2013) ## Output Format diff --git a/scripts/dev-lead-fix-reviews.sh b/scripts/dev-lead-fix-reviews.sh index bdf459884..df82945a6 100755 --- a/scripts/dev-lead-fix-reviews.sh +++ b/scripts/dev-lead-fix-reviews.sh @@ -14,6 +14,13 @@ source "$(dirname "$0")/lib/conflict-integrity.sh" source "$(dirname "$0")/lib/review-change-evidence.sh" source "$(dirname "$0")/lib/resolution-integrity.sh" source "$(dirname "$0")/lib/addressed-claim-verify.sh" +# Claim landing (#2013): "did the push land?" as a pure verdict, and retraction of +# this pass's claim replies whose commit did not land on the remote head. +source "$(dirname "$0")/lib/claim-landing.sh" +# Test-tamper guard (#2013): a fix pass may not silently rewrite an existing test. +source "$(dirname "$0")/lib/test-tamper-guard.sh" +# Test-regression guard (#2013): a pass may not push with the suite newly red. +source "$(dirname "$0")/lib/test-regression-guard.sh" # PR issue-comment disposition verifier (#1813): the issue-comment sibling of # addressed-claim-verify.sh. Turns a dev-lead comment-disposition reply into a # machine-checkable claim the harness verifies before minimizing the original @@ -107,6 +114,9 @@ if [ "${DEV_LEAD_DRY_RUN:-false}" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then # gate on a no-commit pass. RESOLUTION_BASE_SHA is set once here and never # reassigned, so the gate measures only whether THIS pass advanced the head. RESOLUTION_BASE_SHA="$(git rev-parse HEAD 2>/dev/null || true)" + # The pass boundary for the claim-retraction sweep (#2013): only claim replies + # created at/after this instant are this pass's to retract. + PASS_START_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" setup_git_identity # Backfill the five required description sections into a pre-existing PR whose # body still lacks them (#1805). Idempotent + marker-keyed, so open PRs heal on @@ -803,13 +813,22 @@ resolve_addressed_bot_threads() { # Gather the git facts for the claimed commit (the only impure step). Fail # closed: an unresolvable commit reads as on_head=false with empty diffs. - local facts on_head own_files cumulative_files commit_date - facts=$(acv_gather_commit_facts "$claim_sha") + local facts on_head in_base in_pass own_files cumulative_files commit_date + facts=$(acv_gather_commit_facts "$claim_sha" "${RESOLUTION_BASE_SHA:-}") on_head=$(printf '%s' "$facts" | jq -r '.on_head // false' 2>/dev/null || echo "false") if [ "$on_head" != "true" ]; then echo "::notice::skipping thread ${id} — claimed commit ${claim_sha} is not on PR #${PR_NUMBER}'s head branch; leaving unresolved (#1692)" continue fi + # #2013: the claimed commit must be one THIS pass produced — not the PR's first + # commit or any other pre-pass ancestor, which passes the on-head + intersection + # checks trivially (the petry-projects/.github#1220 `571a3b8` case). Parity with + # the issue-comment `fixed` check in resolve_dispositioned_comments (#1813). + in_base=$(printf '%s' "$facts" | jq -r 'if .in_base == false then "false" else "true" end' 2>/dev/null || echo "true") + if ! in_pass=$(acv_claim_in_pass "${RESOLUTION_BASE_SHA:-}" "$on_head" "$in_base"); then + echo "::notice::skipping thread ${id} — claimed commit ${claim_sha} was not produced by this pass (${in_pass}; base=${RESOLUTION_BASE_SHA:-}); leaving unresolved (#2013)" + continue + fi own_files=$(printf '%s' "$facts" | jq -r '.own_files[]? // empty' 2>/dev/null || echo "") cumulative_files=$(printf '%s' "$facts" | jq -r '.cumulative_files[]? // empty' 2>/dev/null || echo "") commit_date=$(printf '%s' "$facts" | jq -r '.commit_date // ""' 2>/dev/null || echo "") @@ -2192,17 +2211,250 @@ commit_and_push() { fi ;; esac + # Test-tamper guard (#2013): a bot-driven fix pass must not silently rewrite an + # existing test to make its own change pass (the petry-projects/.github#1220 + # `13927fc` shape). Refuse the push and escalate, like the no-op guard (rc 3). + case "$intent" in + fix-reviews|fix-bot-comment|review-changes|human-pr) + local ttg_out ttg_rc=0 ttg_files + ttg_out=$(ttg_scan_pass "${RESOLUTION_BASE_SHA:-}" HEAD) || ttg_rc=$? + ttg_files=$(printf '%s\n' "$ttg_out" | sed '1d' | paste -sd ',' - | sed 's/,/, /g') + if [ "$ttg_rc" -eq 1 ]; then + echo "::error::Test-tamper guard: the ${intent} pass changed or skipped existing test(s) [${ttg_files:-skip added}] with no Test-Change-Justification trailer — refusing to push (#2013)" + flag_test_tamper "$intent" "${ttg_files:-a skip was added to a test file}" + return 4 + elif [ "$ttg_rc" -ne 0 ]; then + # Fail closed: an unverifiable scan is not a clean scan. + echo "::error::Test-tamper guard: pre-pass base unknown — could not verify ${intent} for rewritten tests, refusing to push (#2013)" + flag_test_tamper "$intent" "(the tamper scan could not resolve the pre-pass head, so the pass could not be verified)" + return 4 + elif [ "$(printf '%s\n' "$ttg_out" | head -1)" = "justified" ]; then + echo "::notice::Test-tamper guard: existing test(s) changed with a cited Test-Change-Justification: ${ttg_files:-skip added} (#2013)" + fi + ;; + esac + # Test-regression guard (#2013): the `15a919e` shape adds a new test and breaks an + # existing one WITHOUT editing it, so the tamper guard is silent. Run the suite; + # refuse the push when a test that passed on the pre-pass head fails on the result. + case "$intent" in + fix-reviews|fix-bot-comment|review-changes|human-pr) + local trg_out trg_rc=0 trg_verdict trg_cmd trg_tests + trg_out=$(trg_scan_pass "${RESOLUTION_BASE_SHA:-}") || trg_rc=$? + IFS=$'\t' read -r trg_verdict trg_cmd <<<"$(printf '%s\n' "$trg_out" | head -1)" + trg_tests=$(printf '%s\n' "$trg_out" | sed '1d' | paste -sd ',' - | sed 's/,/, /g') + record_suite_summary "$(trg_summary_line "$trg_verdict" "$trg_cmd")" + if [ "$trg_rc" -ne 0 ]; then + echo "::error::Test-regression guard: the ${intent} pass broke test(s) that passed on the pre-pass head [${trg_tests}] — refusing to push (#2013)" + flag_test_regression "$intent" "${trg_tests:-(unnamed)}" + return 4 + fi + ;; + esac # No-clobber push (#1311): never discard a concurrent writer's unseen commit. # push_no_clobber fast-forwards normally and only ever force-with-leases a # rewritten branch, aborting if the remote moved beyond what we fetched. + # A rejected push — or one the remote head does not reflect (#2013) — must not + # leave this pass's "Fixed" replies standing: retract them before failing. + # (commit_and_push returns 3 for the no-op guard and 4 for the test-tamper and + # test-regression guards.) push_no_clobber || { echo "::error::git push failed — check remote access and branch permissions" >&2 + retract_unlanded_claims "$intent" failed || true + exit 1 + } + verify_push_landed || { + retract_unlanded_claims "$intent" failed || true exit 1 } fi return 0 } +# verify_push_landed — "the push landed" is a verdict on SHAs, not an assumption +# (#2013). Compares the immutable pre-pass head (RESOLUTION_BASE_SHA), the pushed +# HEAD, and the TRUE remote head via cl_push_landed_verdict; returns non-zero +# unless the remote contains the pushed commit. Without a PR checkout (no +# pre-pass head) or an upstream (no push target to read back) there is nothing to +# compare, so it warns and passes — the push itself already succeeded. +verify_push_landed() { + local start="${RESOLUTION_BASE_SHA:-}" pushed remote="" remote_rc=0 on_remote=false verdict + if [ -z "$start" ]; then + echo "::warning::push-landed check skipped — no pre-pass head recorded for this run (#2013)" + return 0 + fi + pushed="$(git rev-parse HEAD 2>/dev/null || true)" + remote=$(cl_remote_head) || remote_rc=$? + if [ "$remote_rc" -eq 1 ]; then + echo "::warning::push-landed check skipped — the branch has no upstream to read back (#2013)" + return 0 + fi + if [ "$remote_rc" -ne 0 ]; then + # Fail closed: an unreadable remote head is indeterminate, not "landed". Abort + # so the caller retracts this pass's claim replies (#2013). + echo "::error::push not verified (unknown): could not read the remote head back after the push (pre-pass head ${start}) — treating the push as failed (#2013)" >&2 + return 1 + fi + if [ -n "$remote" ] && [ -n "$pushed" ] && git merge-base --is-ancestor "$pushed" "$remote" 2>/dev/null; then + on_remote=true + fi + if ! verdict=$(cl_push_landed_verdict "$start" "$pushed" "$remote" "$on_remote"); then + echo "::error::push not verified (${verdict}): the remote head ${remote:-} does not contain the pushed commit ${pushed:-} (pre-pass head ${start}) — treating the push as failed (#2013)" >&2 + return 1 + fi + echo "::notice::push verified: ${pushed} is on the remote head ${remote} (#2013)" + return 0 +} + +# retract_unlanded_claims — the claim-retraction sweep (#2013). +# The model posts its addressed/claim replies from its own shell BEFORE the +# harness commits and pushes, so a failed push, a guard abort, an engine failure, +# or a claim naming a pre-pass commit left a false "Fixed" on the thread — and +# another bot could treat it as addressed. For every claim reply OUR account +# posted since PASS_START_ISO, verify its commit is one this pass produced AND is +# on the REMOTE head (acv_claim_in_pass); otherwise rewrite it with +# cl_retract_body, which strips both markers so it can never resolve a thread. +# +# The reference head is the remote head. When there is no upstream (rc 1) on an +# `ok` outcome, the local HEAD is what was pushed; in every other case — including +# a failed read-back (rc 2), which is indeterminate — nothing verifiably landed, so +# the pre-pass base is used and every claim fails. +# Idempotent: a retracted reply carries no marker and is never selected again. +retract_unlanded_claims() { + local intent="$1" outcome="${2:-ok}" + if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then + echo "[dry-run] would retract unlanded claim replies posted by this ${intent} pass on PR #${PR_NUMBER:-}" + return 0 + fi + [ -z "${PR_NUMBER:-}" ] && return 0 + if [ -z "${PASS_START_ISO:-}" ]; then + echo "::notice::retract_unlanded_claims: no pass start recorded — skipping (#2013)" + return 0 + fi + + local bot_user="${BOT_USER:-donpetry-bot}" base="${RESOLUTION_BASE_SHA:-}" + local ref="" ref_rc=0 + ref=$(cl_remote_head) || ref_rc=$? + if [ -z "$ref" ]; then + if [ "$ref_rc" -eq 1 ] && [ "$outcome" = "ok" ]; then + ref="$(git rev-parse HEAD 2>/dev/null || true)" + else + ref="$base" + fi + fi + + local comments + if ! comments=$(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/comments?per_page=100" 2>/dev/null | jq -s 'add // []' 2>/dev/null); then + echo "::warning::retract_unlanded_claims: could not list review comments on PR #${PR_NUMBER} — this pass's claim replies were NOT verified (#2013)" + return 1 + fi + + local rows retracted=0 failed=0 id sha facts on_ref in_base reason body new_body + if ! rows=$(cl_select_pass_claims "$comments" "$bot_user" "$PASS_START_ISO"); then + echo "::warning::retract_unlanded_claims: could not parse the review comments on PR #${PR_NUMBER} — this pass's claim replies were NOT verified (#2013)" + return 1 + fi + while IFS=$'\t' read -r id sha; do + [ -z "$id" ] && continue + if [ -z "$sha" ]; then + reason="unverifiable" + else + facts=$(acv_gather_commit_facts "$sha" "$base" "${ref:-HEAD}") + on_ref=$(printf '%s' "$facts" | jq -r '.on_head // false' 2>/dev/null || echo "false") + in_base=$(printf '%s' "$facts" | jq -r 'if .in_base == false then "false" else "true" end' 2>/dev/null || echo "true") + [ -z "$ref" ] && on_ref=false + if reason=$(acv_claim_in_pass "$base" "$on_ref" "$in_base"); then + continue + fi + fi + body=$(printf '%s' "$comments" | jq -r --arg id "$id" 'first(.[] | select((.id | tostring) == $id)) | .body // ""' 2>/dev/null || echo "") + new_body=$(cl_retract_body "$body" "$reason") + if gh api -X PATCH "repos/${REPO}/pulls/comments/${id}" -f body="$new_body" >/dev/null 2>&1; then + retracted=$((retracted + 1)) + echo "::warning::retracted claim reply ${id} on PR #${PR_NUMBER} (${reason}; cited ${sha:-}, remote head ${ref:-}) (#2013)" + else + echo "::error::could not retract claim reply ${id} on PR #${PR_NUMBER} (${reason}) — a false 'Fixed' reply may remain (#2013)" + failed=1 + fi + done <<< "$rows" + echo "::notice::retract_unlanded_claims: retracted ${retracted} claim reply(ies) from this ${intent} pass on PR #${PR_NUMBER} (outcome=${outcome})" + # A listing or PATCH failure leaves a possibly-false claim standing: report it so + # callers do not treat the pass as clean (resolution gate closes). + [ "$failed" -eq 0 ] +} + +# record_suite_summary — put the test-suite verdict in the run summary and the +# log, so a pass that did not run the suite never reads as green (#2013). +record_suite_summary() { + local line="$1" + case "$line" in + *"NOT RUN"*|*"NOT verified"*) echo "::warning::${line}" ;; + *) echo "::notice::${line}" ;; + esac + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s\n' "$line" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + fi +} + +# flag_test_regression — the test-regression guard refused the push +# (#2013). Same escalation as flag_test_tamper: one deduped comment, the +# needs-human-review label, auto-merge disabled and its EXIT-trap restore suppressed. +flag_test_regression() { + local intent="$1" tests="$2" + _AM_NEEDS_RESTORE=0 + if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then + echo "[dry-run] test-regression guard: would flag PR #${PR_NUMBER} (${intent}), add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}, disable auto-merge" + return 0 + fi + local marker="" + if gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ + | jq -r '.[].body // ""' 2>/dev/null | grep -qF "$marker"; then + echo "::notice::PR #${PR_NUMBER} already flagged for a test regression for intent=${intent} — not reposting" + else + gh pr comment "$PR_NUMBER" --repo "$REPO" --body "${marker} +## Test suite newly red — human attention needed + +The \`${intent}\` pass left the test suite failing on test(s) that passed on the pre-pass head: ${tests}. A fix that breaks a passing test is wrong or incomplete, so dev-lead **did not push** this pass (#2013). Auto-merge has been disabled." \ + || echo "::warning::could not post test-regression flag comment on PR #${PR_NUMBER}" + fi + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}" 2>/dev/null \ + || echo "::warning::could not add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review} label on PR #${PR_NUMBER}" + gh pr merge "$PR_NUMBER" --repo "$REPO" --disable-auto 2>/dev/null \ + || echo "::notice::auto-merge was not enabled on PR #${PR_NUMBER} (nothing to disable)" + return 0 +} + +# flag_test_tamper — the test-tamper guard refused the push +# (#2013). Mirrors flag_noop_pr: one deduped human-attention comment, the +# needs-human-review label, auto-merge disabled and its EXIT-trap restore +# suppressed. A bot suggestion that contradicts an existing test is a question for +# a human, never a reason to rewrite the test. +flag_test_tamper() { + local intent="$1" files="$2" + _AM_NEEDS_RESTORE=0 + if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then + echo "[dry-run] test-tamper guard: would flag PR #${PR_NUMBER} (${intent}), add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}, disable auto-merge" + return 0 + fi + local marker="" + if gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ + | jq -r '.[].body // ""' 2>/dev/null | grep -qF "$marker"; then + echo "::notice::PR #${PR_NUMBER} already flagged for test tampering for intent=${intent} — not reposting" + else + gh pr comment "$PR_NUMBER" --repo "$REPO" --body "${marker} +## Existing test changed without justification — human attention needed + +The \`${intent}\` pass changed, deleted, or skipped **existing** test(s) to go with its fix: ${files}. A previously-passing test that disagrees with a change is a reason to question the change, not to edit the test, so dev-lead **did not push** this pass (#2013). + +If the test really is wrong, a human should make that call, or the fix should carry a \`Test-Change-Justification:\` commit trailer that cites the reason (for example, the review comment that asked for it). Auto-merge has been disabled." \ + || echo "::warning::could not post test-tamper flag comment on PR #${PR_NUMBER}" + fi + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}" 2>/dev/null \ + || echo "::warning::could not add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review} label on PR #${PR_NUMBER}" + gh pr merge "$PR_NUMBER" --repo "$REPO" --disable-auto 2>/dev/null \ + || echo "::notice::auto-merge was not enabled on PR #${PR_NUMBER} (nothing to disable)" + return 0 +} + # count_not_applied_markers — number of not-applied terminal markers # currently on the PR for this intent. They are cleared on the next `applied`, # so the count is the run of consecutive non-converging passes (#1567 AC #4). @@ -2423,6 +2675,9 @@ case "$INTENT_TYPE" in if [ "$rc" -eq 0 ]; then cp_rc=0 commit_and_push "fix-reviews" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "fix-reviews" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" || { [ "$cp_rc" -ne 0 ] || cp_rc=1; } if [ "$cp_rc" -eq 0 ]; then notify_coderabbit_resolve finalize_review_application "fix-reviews" @@ -2431,6 +2686,8 @@ case "$INTENT_TYPE" in # for a human, auto-merge disabled. Post no applied/no-changes/retry # marker and do not re-enable auto-merge or resolve threads. echo "::warning::fix-reviews produced a net-zero diff — flagged for human, not pushed (#1340)" + elif [ "$cp_rc" -eq 4 ]; then + echo "::warning::fix-reviews was refused by a test guard (tamper/regression) — flagged for human, not pushed (#2013)" else notify_coderabbit_resolve if has_hard_blockers; then @@ -2448,8 +2705,14 @@ case "$INTENT_TYPE" in # a tracking issue; invalid/answered/informational need a non-empty evidence # reply), so an answered/invalid disposition requires no head advance. Runs on # every successful pass, including a net-zero one where the model only replied. - resolve_dispositioned_comments "fix-reviews" - if [ "$cp_rc" -ne 3 ]; then + # A tamper-guard abort (rc 4) pushed nothing: a `fixed` disposition would cite a + # local commit that never landed, so treat it like a failed pass (#2013). + if [ "$cp_rc" -eq 4 ]; then + resolve_dispositioned_comments "fix-reviews" failed + else + resolve_dispositioned_comments "fix-reviews" + fi + if [ "$cp_rc" -ne 3 ] && [ "$cp_rc" -ne 4 ]; then # Resolution gate (#1617): auto-resolve threads only when this pass advanced # the PR head. A no-commit pass resolves zero threads (#1609/#1024). if resolution_gate_open "$cp_rc"; then @@ -2478,6 +2741,8 @@ case "$INTENT_TYPE" in # recovered in-process — but the next pass self-heals via the idempotent # posting + duplicate recovery above.) resolve_dispositioned_comments "fix-reviews" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "fix-reviews" failed || true fi exit "$rc" ;; @@ -2497,6 +2762,9 @@ case "$INTENT_TYPE" in # then leaves no marker, and the retry re-dispatches instead of stalling. _fbc_terminal="" commit_and_push "fix-bot-comment" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "fix-bot-comment" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" || { [ "$cp_rc" -ne 0 ] || cp_rc=1; } if [ "$cp_rc" -eq 0 ]; then notify_coderabbit_resolve _fbc_terminal="applied" @@ -2505,6 +2773,8 @@ case "$INTENT_TYPE" in # for a human, auto-merge disabled. Post no terminal marker and do not # re-enable auto-merge or resolve threads. echo "::warning::fix-bot-comment produced a net-zero diff — flagged for human, not pushed (#1340)" + elif [ "$cp_rc" -eq 4 ]; then + echo "::warning::fix-bot-comment was refused by a test guard (tamper/regression) — flagged for human, not pushed (#2013)" else notify_coderabbit_resolve if has_hard_blockers; then @@ -2518,7 +2788,13 @@ case "$INTENT_TYPE" in # Outside the review-thread resolution gate for the same reason as fix-reviews: # each disposition is independently verified, so a non-`fixed` disposition # needs no head advance. Runs on every successful pass, net-zero included. - resolve_dispositioned_comments "fix-bot-comment" + # A tamper-guard abort (rc 4) pushed nothing: a `fixed` disposition would cite a + # local commit that never landed, so treat it like a failed pass (#2013). + if [ "$cp_rc" -eq 4 ]; then + resolve_dispositioned_comments "fix-bot-comment" failed + else + resolve_dispositioned_comments "fix-bot-comment" + fi if [ "${RDC_STATE_UNKNOWN:-0}" = "1" ]; then echo "::warning::fix-bot-comment: a comment's current state could not be confirmed — withholding the terminal marker so the bot-comment retry can re-dispatch (#2017)" _fbc_terminal="" @@ -2533,7 +2809,7 @@ case "$INTENT_TYPE" in applied) post_reviews_terminal "fix-bot-comment" "applied" "Changes committed and pushed." ;; no-changes) post_no_changes "fix-bot-comment" ;; esac - if [ "$cp_rc" -ne 3 ]; then + if [ "$cp_rc" -ne 3 ] && [ "$cp_rc" -ne 4 ]; then # Resolution gate (#1617): auto-resolve threads only when this pass advanced # the PR head. A no-commit pass resolves zero threads (#1609/#1024). if resolution_gate_open "$cp_rc"; then @@ -2547,11 +2823,12 @@ case "$INTENT_TYPE" in fi try_enable_auto_merge fi - try_enable_auto_merge else # Don't orphan dispositions on a failed/timed-out pass (#1992) — see the # fix-reviews failure branch above for why running the resolver here is safe. resolve_dispositioned_comments "fix-bot-comment" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "fix-bot-comment" failed || true fi exit "$rc" ;; @@ -2610,6 +2887,9 @@ case "$INTENT_TYPE" in if [ "$rc" -eq 0 ]; then cp_rc=0 commit_and_push "review-changes" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "review-changes" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" || { [ "$cp_rc" -ne 0 ] || cp_rc=1; } # No-op guard (#1786): a review-changes pass whose net diff to base is empty # was already flagged by flag_noop_pr (needs-human label + auto-merge # disabled + suppressed EXIT-trap restore). Stop here — never resolve threads @@ -2618,6 +2898,14 @@ case "$INTENT_TYPE" in echo "::notice::review-changes: no-op guard aborted the push for PR #${PR_NUMBER} — flagged for human, not merged (#1786)" exit "$rc" fi + # Test guards (#2013): the push was refused and the PR flagged for a human. + # Nothing landed, so a `fixed` disposition would cite a local-only commit; no + # thread resolves and auto-merge stays off. + if [ "$cp_rc" -eq 4 ]; then + echo "::warning::review-changes was refused by a test guard (tamper/regression) — flagged for human, not pushed (#2013)" + resolve_dispositioned_comments "review-changes" failed + exit "$rc" + fi if [ "$cp_rc" -eq 0 ]; then notify_coderabbit_resolve finalize_review_application "review-changes" @@ -2654,6 +2942,8 @@ case "$INTENT_TYPE" in # Don't orphan dispositions on a failed/timed-out pass (#1992) — see the # fix-reviews failure branch above for why running the resolver here is safe. resolve_dispositioned_comments "review-changes" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "review-changes" failed || true fi exit "$rc" ;; diff --git a/scripts/lib/addressed-claim-verify.sh b/scripts/lib/addressed-claim-verify.sh index 756ffdec6..a0e35fec0 100644 --- a/scripts/lib/addressed-claim-verify.sh +++ b/scripts/lib/addressed-claim-verify.sh @@ -395,11 +395,46 @@ _acv_is_iso8601() { jq -e -n --arg s "$s" '($s | fromdateiso8601)' >/dev/null 2>&1 } -# acv_gather_commit_facts +# acv_claim_in_pass +# The in-pass verdict (#2013). A claim verifies only when its commit was produced +# by THIS pass: reachable from the reference head ( — the PR head at +# resolution time, or the REMOTE head when deciding whether a claim landed) and +# NOT already contained in the pre-pass base (). Without it a reply citing +# the PR's FIRST commit — the head the model read with `git rev-parse HEAD` before +# committing anything — passed the on-head + intersection checks trivially (the +# petry-projects/.github#1220 `571a3b8` case). This is the thread-path parity of the +# issue-comment `fixed` check in resolve_dispositioned_comments (#1813). +# / are the literal strings "true"/"false" from +# acv_gather_commit_facts; anything else fails closed. Echoes one token — +# in-pass | unknown-base | not-on-ref | predates-pass — and returns 0 only for +# in-pass. Pure — no gh/git/network. +acv_claim_in_pass() { + local base="${1:-}" on_ref="${2:-}" in_base="${3:-}" + if [[ -z "$base" ]]; then + echo "unknown-base" + return 1 + fi + if [[ "$on_ref" != "true" ]]; then + echo "not-on-ref" + return 1 + fi + if [[ "$in_base" != "false" ]]; then + echo "predates-pass" + return 1 + fi + echo "in-pass" + return 0 +} + +# acv_gather_commit_facts [base_sha] [ref] # The SINGLE impure gatherer: all git access lives here so the verifier above stays # pure. Echoes a JSON object: -# {"on_head":bool,"own_files":[...],"cumulative_files":[...],"commit_date":"iso"} -# - on_head: is reachable from HEAD (i.e. on the PR head branch). +# {"on_head":bool,"in_base":bool,"own_files":[...],"cumulative_files":[...],"commit_date":"iso"} +# - on_head: is reachable from (default HEAD — the PR head branch; the +# retraction sweep passes the REMOTE head so a local-only commit is not "on"). +# - in_base: is contained in (the pre-pass head, #2013). Fails +# closed to true when is empty or does not resolve, so an unknowable +# base can never make a stale commit look like this pass's work. # - own_files: files in 's own diff. # - cumulative_files: files in `^..HEAD` (falls back to `..HEAD` when # is a root commit with no parent). @@ -408,13 +443,20 @@ _acv_is_iso8601() { # never read an error as "verified". Runs in the current working directory (the PR # worktree at resolution time). acv_gather_commit_facts() { - local sha="${1:-}" - local on_head=false own_files='[]' cumulative_files='[]' commit_date="" + local sha="${1:-}" base="${2:-}" ref="${3:-HEAD}" + local on_head=false in_base=true own_files='[]' cumulative_files='[]' commit_date="" if [[ -n "$sha" ]] && git cat-file -e "${sha}^{commit}" 2>/dev/null; then - if git merge-base --is-ancestor "$sha" HEAD 2>/dev/null; then + if git merge-base --is-ancestor "$sha" "$ref" 2>/dev/null; then on_head=true fi + if [[ -n "$base" ]] && git cat-file -e "${base}^{commit}" 2>/dev/null; then + # Exit 1 means "not an ancestor"; any other status is an ancestry read error + # and leaves in_base=true (fail closed). + local mb_rc=0 + git merge-base --is-ancestor "$sha" "$base" 2>/dev/null || mb_rc=$? + [[ "$mb_rc" -eq 1 ]] && in_base=false + fi # Emit a Z-terminated UTC ISO-8601 instant (e.g. 2026-09-07T21:40:05Z), the # SAME shape GitHub's createdAt uses. `%cI` would emit a `+00:00` offset that # jq's fromdateiso8601 rejects and that does not compare lexicographically @@ -432,9 +474,10 @@ acv_gather_commit_facts() { jq -c -n \ --argjson on_head "$on_head" \ + --argjson in_base "$in_base" \ --argjson own "$own_files" \ --argjson cumulative "$cumulative_files" \ --arg date "$commit_date" \ - '{on_head: $on_head, own_files: $own, cumulative_files: $cumulative, commit_date: $date}' \ - 2>/dev/null || printf '{"on_head":false,"own_files":[],"cumulative_files":[],"commit_date":""}' + '{on_head: $on_head, in_base: $in_base, own_files: $own, cumulative_files: $cumulative, commit_date: $date}' \ + 2>/dev/null || printf '{"on_head":false,"in_base":true,"own_files":[],"cumulative_files":[],"commit_date":""}' } diff --git a/scripts/lib/claim-landing.sh b/scripts/lib/claim-landing.sh new file mode 100644 index 000000000..6ec6234ff --- /dev/null +++ b/scripts/lib/claim-landing.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# claim-landing.sh — "did the push land, and which claims must be retracted?" as +# pure verdicts (#2013). +# +# WHAT IS WRONG WITHOUT THIS +# The fix-reviews / fix-bot-comment model posts its "Fixed in …" reply (with the +# addressed-marker + #1692 claim) from its OWN shell, BEFORE the harness commits +# and pushes. On petry-projects/.github#1220 that left false "Fixed" replies +# standing when the push never landed, and when the claim named the PR's FIRST +# commit (a stale pre-pass head). Separately, CodeRabbit marked a thread +# "✅ Confirmed as addressed" on a reply describing `15a919e`, which WAS pushed: +# a wrong fix (a new test added, an existing one broken without being edited) +# cleared a review gate. Landing checks cannot catch that one; the +# test-regression guard does. +# The #1692 claim check ran only at thread-RESOLUTION time; nothing ever took a +# posted claim back. +# +# THE CONTRACT +# - cl_push_landed_verdict decides, from SHAs alone, whether the pass's pushed +# commit is on the remote head. commit_and_push treats anything but `landed` as +# a failed push. +# - cl_select_pass_claims picks OUR claim/marker replies created during this pass. +# - each selected claim is checked with acv_claim_in_pass against the REMOTE head +# (addressed-claim-verify.sh); any that fails is rewritten with cl_retract_body, +# which strips both markers so the reply can never authorize resolution again. +# +# PURITY / TESTABILITY (ADR-0004) +# Every cl_* function except cl_remote_head is PURE: no network, no git, no gh. +# cl_remote_head is the single impure reader. Sourced under `set -euo pipefail`, +# so helpers only `return`, never `exit`, and fail closed on every ambiguity. + +set -euo pipefail + +# acv_parse_claim is the single claim parser — reuse it, never a second one. +if ! declare -F acv_parse_claim >/dev/null 2>&1; then + # shellcheck source=addressed-claim-verify.sh + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/addressed-claim-verify.sh" +fi + +# The retraction marker. Matches the shared `` agent-marker +# discriminator, so a retracted reply is still recognised as ours, but it is NOT the +# addressed-marker, so it can never authorize resolution. +readonly _CL_RETRACTED_PREFIX='`. +# is restricted to [a-z-] (anything else becomes `unknown`) so it can never break +# out of the HTML comment. Pure. +cl_retract_body() { + local body="${1:-}" reason="${2:-unknown}" + [[ "$reason" =~ ^[a-z][a-z-]*$ ]] || reason="unknown" + + local stripped + stripped=$(jq -Rsr --arg marker "$_DEV_LEAD_ADDRESSED_MARKER" ' + gsub($marker; "") + | gsub(""; "") + | sub("\\s+$"; "") + ' <<<"$body" 2>/dev/null) || stripped="" + local quoted + quoted=$(printf '%s\n' "$stripped" | sed 's/^/> /') + + printf '%s\n\n%s\n\n%s\n\n%s%s -->\n' \ + "**⚠️ Retracted by the dev-lead harness (\`${reason}\`).** This reply's claim could not be verified as a fix produced and pushed by this pass (its commit was not produced by this pass, or the push did not land), so it no longer counts as addressed. The thread stays open; the next dev-lead pass re-verifies against the pushed diff." \ + "Original reply, kept for the record (unverified — it does not count as a fix):" \ + "$quoted" \ + "$_CL_RETRACTED_PREFIX" "$reason" +} + +# cl_remote_head +# The single impure reader: fetch the current branch's upstream and echo the TRUE +# remote head SHA (not a stale remote-tracking ref). Returns 1 (echoing nothing) +# when the branch has no upstream, 2 when the fetch or lookup fails. +cl_remote_head() { + local up remote branch + up=$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true) + [[ -z "$up" ]] && return 1 + remote="${up%%/*}" + branch="${up#*/}" + git fetch --quiet "$remote" "$branch" 2>/dev/null || return 2 + local sha + sha=$(git rev-parse --verify --quiet "refs/remotes/${remote}/${branch}" 2>/dev/null \ + || git rev-parse --verify --quiet FETCH_HEAD 2>/dev/null || true) + [[ -z "$sha" ]] && return 2 + echo "$sha" +} diff --git a/scripts/lib/test-regression-guard.sh b/scripts/lib/test-regression-guard.sh new file mode 100644 index 000000000..58c63e232 --- /dev/null +++ b/scripts/lib/test-regression-guard.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# test-regression-guard.sh — a fix pass may not push with the suite newly red (#2013). +# +# WHAT IS WRONG WITHOUT THIS +# On petry-projects/.github#1220 the `15a919e` fix added a NEW test and broke an +# EXISTING one without editing it. The test-tamper guard is silent about that (no +# existing test line changed) and the claim names a real in-pass commit, so every +# other check passed. Only running the suite against the pre-pass baseline sees it. +# +# THE RULE +# Run the repo's test command on the pass's result. Then: +# green the suite passes; +# preexisting it fails, but every failing test also fails on the pre-pass head; +# regression a test fails that did not fail on the pre-pass head (a previously +# passing test broke, or a new test fails). The harness refuses to +# push and escalates to a human; +# unattributed it fails on both heads and no failing test names could be read, +# so the failure cannot be pinned on the pass (reported, not blocked); +# unbaselined it fails and the pre-pass head could not be run (not blocked, said +# loudly); +# timeout the suite did not finish (not blocked, said loudly); +# not-run no test command could be determined — the suite was NOT run, and +# the run summary says so. Never report or imply green. +# +# HOW THE COMMAND IS FOUND (no workflow_call input — channel-pin sequencing) +# 1. $DEV_LEAD_TEST_CMD, 2. `npm test` (package.json with a test script), +# 3. `make test` (Makefile with a test target), 4. `pytest` (pytest config or tests +# dir with test_*.py), 5. `bats --recursive tests` when bats and *.bats are present. +# +# PURITY / TESTABILITY (ADR-0004) +# trg_extract_failures, trg_classify and trg_summary_line are PURE. trg_discover_cmd +# only reads the working tree; trg_scan_pass is the single impure gatherer (runs the +# suite, and checks out the pre-pass head to baseline a red suite). Sourced under +# `set -euo pipefail`; helpers only `return`, never `exit`. + +set -euo pipefail + +# trg_extract_failures — failing test names from suite output on stdin, sorted and +# unique. Reads TAP (`not ok 3 name`, as bats/prove print) and pytest short summaries +# (`FAILED path::name - reason`). A `# TODO`/`# skip` TAP line is not a failure. Pure. +trg_extract_failures() { + local line name + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" =~ ^not\ ok\ [0-9]+[[:space:]]+(.*)$ ]]; then + name="${BASH_REMATCH[1]}" + # bats appends the measured duration ("in 123ms"); strip it so the same test + # failing on both heads compares equal despite differing timings. + name="${name%% in [0-9]*ms}" + [[ "$name" =~ \#[[:space:]]*(TODO|skip|SKIP) ]] && continue + printf '%s\n' "$name" + elif [[ "$line" =~ ^FAILED\ ([^[:space:]]+) ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" + elif [[ "$line" =~ ^FAIL\ +([^[:space:]]+) ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" # jest/vitest: FAIL path/to/file.test.js + elif [[ "$line" =~ ^[[:space:]]*---\ FAIL:\ ([^[:space:]]+) ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" # go test: --- FAIL: TestName + fi + done | sort -u +} + +# trg_classify +# Echoes the verdict (above) on the first line, then the offending test names for a +# regression, one per line. Returns 1 only for `regression`. A non-numeric head rc +# fails closed (regression). Pure. +trg_classify() { + local base_ran="${1:-false}" base_rc="${2:-}" base_fail="${3:-}" head_rc="${4:-}" head_fail="${5:-}" + if ! [[ "$head_rc" =~ ^[0-9]+$ ]]; then + echo "regression" + echo "(suite result unreadable)" + return 1 + fi + if (( head_rc == 0 )); then + echo "green" + return 0 + fi + if (( head_rc == 124 )); then + echo "timeout" + return 0 + fi + if [[ "$base_ran" != "true" ]] || ! [[ "$base_rc" =~ ^[0-9]+$ ]] || (( base_rc == 124 )); then + echo "unbaselined" + return 0 + fi + local new="" t + if [[ -n "$(printf '%s' "$head_fail" | tr -d '[:space:]')" ]]; then + while IFS= read -r t; do + [[ -z "$t" ]] && continue + printf '%s\n' "$base_fail" | grep -qxF -- "$t" || new+="${t}"$'\n' + done <<<"$head_fail" + if [[ -n "$new" ]]; then + echo "regression" + printf '%s' "$new" + return 1 + fi + echo "preexisting" + return 0 + fi + # Red on the head, no names to compare. + if (( base_rc == 0 )); then + echo "regression" + echo "(suite exited ${head_rc}; it passed on the pre-pass head)" + return 1 + fi + echo "unattributed" + return 0 +} + +# trg_summary_line [cmd] — the one line for the run summary. Never implies +# green unless the suite really passed. Pure. +trg_summary_line() { + local verdict="${1:-}" cmd="${2:-}" + case "$verdict" in + green) echo "Test suite: PASSED on the pass's result (\`${cmd}\`)." ;; + preexisting) echo "Test suite: red, but every failing test already failed on the pre-pass head (\`${cmd}\`) — not caused by this pass." ;; + unattributed) echo "Test suite: red on both the pre-pass head and the result (\`${cmd}\`); no test names could be read, so the failure could not be attributed. NOT verified green." ;; + unbaselined) echo "Test suite: red (\`${cmd}\`) and the pre-pass head could not be run for comparison. NOT verified green." ;; + timeout) echo "Test suite: did not finish within the time limit (\`${cmd}\`). NOT verified green." ;; + regression) echo "Test suite: REGRESSION — a test that passed on the pre-pass head fails on the result (\`${cmd}\`). Push refused." ;; + not-run|*) echo "Test suite: NOT RUN — no test command could be determined (set DEV_LEAD_TEST_CMD). The pass is NOT verified green." ;; + esac +} + +# trg_discover_cmd [dir] — echoes the test command for (default .), or returns 1 +# when none can be determined. +trg_discover_cmd() { + local dir="${1:-.}" + if [[ -n "${DEV_LEAD_TEST_CMD:-}" ]]; then + echo "$DEV_LEAD_TEST_CMD" + return 0 + fi + if [[ -f "$dir/package.json" ]] && grep -q '"test"[[:space:]]*:' "$dir/package.json" 2>/dev/null; then + echo "npm test" + return 0 + fi + if [[ -f "$dir/Makefile" ]] && grep -qE '^test[[:space:]]*:' "$dir/Makefile" 2>/dev/null; then + echo "make test" + return 0 + fi + if [[ -f "$dir/pytest.ini" || -f "$dir/tox.ini" || -f "$dir/pyproject.toml" || -f "$dir/setup.cfg" ]] \ + && command -v pytest >/dev/null 2>&1 \ + && [[ -n "$(find "$dir" -path "$dir/.git" -prune -o -name 'test_*.py' -print -quit 2>/dev/null)" ]]; then + echo "pytest -q" + return 0 + fi + if [[ -d "$dir/tests" ]] && command -v bats >/dev/null 2>&1 \ + && [[ -n "$(find "$dir/tests" -name '*.bats' -print -quit 2>/dev/null)" ]]; then + echo "bats --recursive tests" + return 0 + fi + return 1 +} + +# _trg_stage [base_sha] — fill /tree with a credential-free copy of +# the current working tree (everything but .git, so installed deps come along). With +# , the tracked files are replaced by that commit's. A fresh `git init` gives +# git-using tests a repo; it carries none of the real checkout's git config, so the +# saved checkout credential (http.*.extraheader) does not exist in the copy. +_trg_stage() { + local scratch="$1" base="${2:-}" f + mkdir -p "$scratch/tree" "$scratch/home" "$scratch/tmp" + tar --exclude=.git -cf - . | tar -xf - -C "$scratch/tree" + if [[ -n "$base" ]]; then + # Drop head-tracked files, then lay the base commit's files over the copy. + while IFS= read -r -d '' f; do + rm -rf -- "${scratch:?}/tree/$f" + done < <(git ls-files -z 2>/dev/null) + git archive "$base" | tar -xf - -C "$scratch/tree" + fi + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null git -C "$scratch/tree" init -q 2>/dev/null || true +} + +# _trg_run [base_sha] — run the suite with a time limit in a scratch copy of the +# tree (never the real checkout, so a killed run cannot leave it changed); combined +# output on stdout, the exit status as the return code. +_trg_run() { + local cmd="$1" base="${2:-}" limit="${DEV_LEAD_TEST_TIMEOUT:-1500}" scratch rc=0 + scratch=$(mktemp -d "${TMPDIR:-/tmp}/trg.XXXXXX") || return 1 + _trg_stage "$scratch" "$base" + # PR-controlled code runs here: an ALLOWLIST environment (env -i), not a denylist, so + # no token — whatever its name — reaches the test process. + local -a runner=(env -i "PATH=${PATH}" "HOME=${scratch}/home" "TMPDIR=${scratch}/tmp" + "LANG=${LANG:-C.UTF-8}" "CI=${CI:-true}" "TERM=${TERM:-dumb}" + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null GIT_CONFIG_NOSYSTEM=1) + if command -v timeout >/dev/null 2>&1; then + runner+=(timeout "$limit") + fi + ( cd "$scratch/tree" && "${runner[@]}" bash -c "$cmd" 2>&1 ) || rc=$? + rm -rf -- "$scratch" + return "$rc" +} + +# trg_scan_pass +# The single impure gatherer. Runs the suite on the CURRENT checkout (the pass's +# result). Only when it is red, runs it again against in a scratch copy +# (the real checkout is never touched). Echoes `\t` on the first line, then the +# offending tests. Returns 1 only for `regression`. +trg_scan_pass() { + local base="${1:-}" cmd out head_rc=0 head_fail base_out base_rc="" base_fail="" base_ran=false + if ! cmd=$(trg_discover_cmd .); then + printf 'not-run\t\n' + return 0 + fi + out=$(_trg_run "$cmd") || head_rc=$? + head_fail=$(printf '%s\n' "$out" | trg_extract_failures) + if (( head_rc != 0 && head_rc != 124 )) && [[ -n "$base" ]] && git cat-file -e "${base}^{commit}" 2>/dev/null; then + base_rc=0 + base_out=$(_trg_run "$cmd" "$base") || base_rc=$? + base_fail=$(printf '%s\n' "$base_out" | trg_extract_failures) + base_ran=true + fi + local verdict_out rc=0 + verdict_out=$(trg_classify "$base_ran" "$base_rc" "$base_fail" "$head_rc" "$head_fail") || rc=$? + printf '%s\t%s\n' "$(printf '%s\n' "$verdict_out" | head -1)" "$cmd" + printf '%s\n' "$verdict_out" | sed '1d' + return "$rc" +} diff --git a/scripts/lib/test-tamper-guard.sh b/scripts/lib/test-tamper-guard.sh new file mode 100644 index 000000000..d81c04570 --- /dev/null +++ b/scripts/lib/test-tamper-guard.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# test-tamper-guard.sh — a fix pass must not silently rewrite an existing test to +# make its own change pass (#2013). +# +# WHAT IS WRONG WITHOUT THIS +# On petry-projects/.github#1220 a cubic finding said, literally, "update the +# older test to assert failure precedence so the Bats suite can pass". dev-lead +# applied it: it rewrote an existing test (#1023 "success precedence …") into +# "failure precedence", inverting a deliberate behavior, and the rewritten test +# did not even match the branch. "Do not modify tests to force a pass" lived only +# in the prompt, so nothing in the harness could stop it. +# +# THE RULE +# A fix pass (base = the pre-pass head) is: +# clean it changes no existing line of a test file and adds no skip — +# ADDING a new test is always fine; +# justified it does change/delete an existing test line or add a skip, AND a +# commit in the pass carries an explicit, cited +# `Test-Change-Justification: ` trailer; +# tampered it does so with no justification. The harness refuses to push and +# escalates to a human. +# "Changed" means the test file has REMOVED lines in `git diff --numstat` (an +# edit is a removal + an addition; a deletion is all removals), so a pure +# addition never trips it. +# +# PURITY / TESTABILITY (ADR-0004) +# ttg_is_test_path, ttg_touched_existing_tests, ttg_count_added_skips, +# ttg_has_justification and ttg_classify are PURE. ttg_scan_pass is the single +# impure gatherer (all git calls). Sourced under `set -euo pipefail`; helpers only +# `return`, never `exit`. + +set -euo pipefail + +# Minimum length of the justification text — rules out a bare "so it passes". +readonly _TTG_MIN_JUSTIFICATION_CHARS=20 + +# Added lines that silence a test, across the frameworks the org uses. +readonly _TTG_SKIP_RE='^\+[[:space:]]*(skip([[:space:]]|$|\()|@pytest\.mark\.(skip|xfail)|pytestmark[[:space:]]*=[[:space:]]*(\[[[:space:]]*)?pytest\.mark\.(skip|xfail)|pytest\.(skip|xfail|importorskip)\(|@unittest\.skip|@Disabled|@Ignore|#\[ignore\]|t\.Skip(Now|f)?\(|(it|test|describe)\.(skip|todo)\(|x(it|describe|test)\()' + +# ttg_is_test_path +# 0 when is a test file: under a tests?/ / __tests__/ / spec/ directory, a +# .bats file, or a test-named file (test_*.py, *_test.*, *.test.*, *.spec.*, +# *_spec.rb). Pure. +ttg_is_test_path() { + local p="${1:-}" + [[ -z "$p" ]] && return 1 + local base="${p##*/}" + case "/$p" in + */tests/*|*/test/*|*/__tests__/*|*/spec/*) return 0 ;; + esac + case "$base" in + *.bats|test_*.py|*_test.*|*.test.*|*.spec.*|*_spec.rb) return 0 ;; + esac + return 1 +} + +# ttg_touched_existing_tests +# is `git diff --numstat` output (`added\tdeleted\tpath`). Echoes, one +# per line, every test path with removed lines (changed or deleted existing +# lines). A binary row (`-\t-`) on a test path is reported too (fail closed). +# Pure. +ttg_touched_existing_tests() { + local numstat="${1:-}" + local _added deleted path + while IFS=$'\t' read -r _added deleted path; do + [[ -z "${path:-}" ]] && continue + ttg_is_test_path "$path" || continue + if [[ "$deleted" == "-" ]] || { [[ "$deleted" =~ ^[0-9]+$ ]] && (( deleted > 0 )); }; then + printf '%s\n' "$path" + fi + done <<<"$numstat" +} + +# ttg_count_added_skips +# Number of ADDED lines in test files (per the `+++ b/` headers) that skip +# or disable a test. Pure. +ttg_count_added_skips() { + local diff="${1:-}" + local line cur="" n=0 + while IFS= read -r line; do + if [[ "$line" == "+++ "* ]]; then + cur="${line#+++ }" + cur="${cur#b/}" + continue + fi + [[ -z "$cur" ]] && continue + ttg_is_test_path "$cur" || continue + [[ "$line" =~ $_TTG_SKIP_RE ]] && n=$((n + 1)) + done <<<"$diff" + echo "$n" +} + +# ttg_has_justification +# 0 when some line is `Test-Change-Justification: ` with at least +# _TTG_MIN_JUSTIFICATION_CHARS of non-blank text AND a recognizable reference +# (an issue/PR `#123`, a 7-40 hex commit SHA, or a URL) to the review, issue, or +# commit that establishes the old test was wrong. Pure. +ttg_has_justification() { + local msgs="${1:-}" line text + while IFS= read -r line; do + [[ "$line" =~ ^[[:space:]]*Test-Change-Justification:[[:space:]]*(.*)$ ]] || continue + text="${BASH_REMATCH[1]}" + text="${text%"${text##*[![:space:]]}"}" + (( ${#text} >= _TTG_MIN_JUSTIFICATION_CHARS )) || continue + [[ "$text" =~ (^|[^[:alnum:]])#[0-9]+ || "$text" =~ (^|[^[:alnum:]])[0-9a-fA-F]{7,40}($|[^[:alnum:]]) || "$text" =~ https?:// ]] && return 0 + done <<<"$msgs" + return 1 +} + +# ttg_classify +# Echoes clean | justified | tampered; returns 1 only for tampered. A +# non-numeric skip count fails closed (tampered). Pure. +ttg_classify() { + local touched="${1:-}" skips="${2:-}" justified="${3:-false}" + local has_touch=0 + [[ -n "$(printf '%s' "$touched" | tr -d '[:space:]')" ]] && has_touch=1 + if ! [[ "$skips" =~ ^[0-9]+$ ]]; then + echo "tampered" + return 1 + fi + if (( has_touch == 0 && skips == 0 )); then + echo "clean" + return 0 + fi + if [[ "$justified" == "true" ]]; then + echo "justified" + return 0 + fi + echo "tampered" + return 1 +} + +# ttg_scan_pass [head] +# The single impure gatherer. Compares against the WORKING TREE (so +# committed and not-yet-committed edits are both seen), reads the justification +# from the commit messages in .., and echoes the verdict on the +# first line followed by the offending test paths (one per line). Returns 0 for +# clean/justified, 1 for tampered, 2 (echoing `unknown`) when is empty +# or does not resolve. +ttg_scan_pass() { + local base="${1:-}" head="${2:-HEAD}" + if [[ -z "$base" ]] || ! git cat-file -e "${base}^{commit}" 2>/dev/null; then + echo "unknown" + return 2 + fi + local numstat diff msgs touched skips justified=false verdict rc=0 + numstat=$(git diff --no-renames --numstat "$base" 2>/dev/null) || { echo "unknown"; return 2; } + diff=$(git diff --unified=0 "$base" 2>/dev/null) || { echo "unknown"; return 2; } + msgs=$(git log --format=%B "${base}..${head}" 2>/dev/null || true) + touched=$(ttg_touched_existing_tests "$numstat") + skips=$(ttg_count_added_skips "$diff") + ttg_has_justification "$msgs" && justified=true + verdict=$(ttg_classify "$touched" "$skips" "$justified") || rc=$? + echo "$verdict" + [[ -n "$touched" ]] && printf '%s\n' "$touched" + return "$rc" +} diff --git a/tests/dev-lead/unit/test_addressed_claim_verify.bats b/tests/dev-lead/unit/test_addressed_claim_verify.bats index 403b6de08..3a6c7a886 100644 --- a/tests/dev-lead/unit/test_addressed_claim_verify.bats +++ b/tests/dev-lead/unit/test_addressed_claim_verify.bats @@ -409,3 +409,117 @@ setup() { [[ "$(echo "$output" | jq -r .on_head)" == "false" ]] [[ "$(echo "$output" | jq -c .own_files)" == "[]" ]] } + +# --------------------------------------------------------------------------- +# acv_claim_in_pass — the claim SHA must be a commit THIS pass produced (#2013) +# --------------------------------------------------------------------------- +# The thread-resolution gate used to accept any commit on the head branch whose +# ^..HEAD range touched the claimed file, so a reply citing the PR's FIRST +# commit (the pre-pass head the model read with `git rev-parse HEAD` before it +# committed anything) passed trivially — the petry-projects/.github#1220 `571a3b8` +# case. A claim now verifies only when its commit is on the reference head AND +# is not already contained in the pre-pass base. + +@test "acv_claim_in_pass: on ref and not in the pre-pass base -> in-pass rc0" { + run acv_claim_in_pass "$SHA40" true false + [[ "$status" -eq 0 ]] + [[ "$output" == "in-pass" ]] +} + +@test "acv_claim_in_pass: commit already in the pre-pass base (stale SHA) -> predates-pass rc1" { + run acv_claim_in_pass "$SHA40" true true + [[ "$status" -eq 1 ]] + [[ "$output" == "predates-pass" ]] +} + +@test "acv_claim_in_pass: commit not on the reference head -> not-on-ref rc1" { + run acv_claim_in_pass "$SHA40" false false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-on-ref" ]] +} + +@test "acv_claim_in_pass: unknown pre-pass base -> unknown-base rc1 (fail closed)" { + run acv_claim_in_pass "" true false + [[ "$status" -eq 1 ]] + [[ "$output" == "unknown-base" ]] +} + +@test "acv_claim_in_pass: any non-'true'/'false' fact fails closed" { + run acv_claim_in_pass "$SHA40" "" false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-on-ref" ]] + run acv_claim_in_pass "$SHA40" true "" + [[ "$status" -eq 1 ]] + [[ "$output" == "predates-pass" ]] +} + +@test "acv_gather_commit_facts: with a base, PR commit A cited after this pass pushed B -> in_base true (predates-pass)" { + # The QA fixture from #2013: the PR's own commit A touches F; this pass pushes B + # (also touching F). A claim {sha:A, files:[F]} passes the old on-head + + # intersection checks but must NOT verify as in-pass. + local repo="$BATS_TEST_TMPDIR/repo3" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/F" + git -C "$repo" add F + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m A + local a + a="$(git -C "$repo" rev-parse HEAD)" + echo two >> "$repo/F" + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -am B + local b + b="$(git -C "$repo" rev-parse HEAD)" + + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$a' '$a'" + [[ "$status" -eq 0 ]] + [[ "$(echo "$output" | jq -r .on_head)" == "true" ]] + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] + local facts_a on_head in_base + facts_a="$output" + on_head="$(echo "$facts_a" | jq -r .on_head)"; in_base="$(echo "$facts_a" | jq -r .in_base)" + run bash -c "source '$LIB' && acv_claim_in_pass '$a' '$on_head' '$in_base'" + [[ "$output" == "predates-pass" ]] + + # B — the commit this pass actually produced — is in-pass. + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$b' '$a'" + on_head="$(echo "$output" | jq -r .on_head)"; in_base="$(echo "$output" | jq -r .in_base)" + run bash -c "source '$LIB' && acv_claim_in_pass '$a' '$on_head' '$in_base'" + [[ "$status" -eq 0 ]] + [[ "$output" == "in-pass" ]] +} + +@test "acv_gather_commit_facts: an unresolvable base fails closed (in_base true)" { + local repo="$BATS_TEST_TMPDIR/repo4" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/a.txt" + git -C "$repo" add a.txt + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m c0 + local h + h="$(git -C "$repo" rev-parse HEAD)" + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$h' 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef'" + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] + # No base given at all is also fail-closed. + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$h'" + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] +} + +@test "acv_gather_commit_facts: an explicit reference ref replaces HEAD for on_head" { + # The retraction sweep checks claims against the REMOTE head, not local HEAD: + # a commit that exists locally but never reached the remote is not on the ref. + local repo="$BATS_TEST_TMPDIR/repo5" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/a.txt" + git -C "$repo" add a.txt + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m c0 + local base + base="$(git -C "$repo" rev-parse HEAD)" + echo two >> "$repo/a.txt" + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -am local-only + local local_only + local_only="$(git -C "$repo" rev-parse HEAD)" + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$local_only' '$base' '$base'" + [[ "$(echo "$output" | jq -r .on_head)" == "false" ]] + [[ "$(echo "$output" | jq -r .in_base)" == "false" ]] +} diff --git a/tests/dev-lead/unit/test_claim_landing.bats b/tests/dev-lead/unit/test_claim_landing.bats new file mode 100644 index 000000000..c36513fb9 --- /dev/null +++ b/tests/dev-lead/unit/test_claim_landing.bats @@ -0,0 +1,289 @@ +#!/usr/bin/env bats +# Unit tests for scripts/lib/claim-landing.sh (#2013). +# +# On petry-projects/.github#1220 dev-lead posted "Fixed in …" replies for changes +# that never reached the branch: the model posts its addressed/claim reply from +# its own shell BEFORE the harness commits and pushes, so a rejected push, a +# no-op-guard abort, an engine failure, or a claim citing the pre-pass head left a +# false "Fixed" on the thread — and CodeRabbit marked a thread addressed on the +# strength of it. claim-landing.sh is the pure verdict layer that decides whether +# a push landed and which of this pass's claim replies must be retracted, plus +# one impure remote-head reader. The harness wiring is covered in +# test_fix_reviews.bats. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +LIB="$SCRIPT_DIR/scripts/lib/claim-landing.sh" +# addressed-claim-verify.sh (acv_*) is sourced transitively by claim-landing.sh. +GUARD_LIB="$SCRIPT_DIR/scripts/lib/git-push-guard.sh" + +A40="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +B40="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +C40="cccccccccccccccccccccccccccccccccccccccc" + +setup() { + # shellcheck source=scripts/lib/claim-landing.sh + source "$LIB" +} + +# --------------------------------------------------------------------------- +# Sourcing safety +# --------------------------------------------------------------------------- + +@test "claim-landing.sh is safe to source under set -euo pipefail" { + run bash -c "set -euo pipefail; source '$LIB'" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +# --------------------------------------------------------------------------- +# cl_push_landed_verdict — "push landed" is a pure verdict, not a model claim +# --------------------------------------------------------------------------- + +@test "cl_push_landed_verdict: remote head == pushed sha, advanced past start -> landed rc0" { + run cl_push_landed_verdict "$A40" "$B40" "$B40" true + [[ "$status" -eq 0 ]] + [[ "$output" == "landed" ]] +} + +@test "cl_push_landed_verdict: remote moved past our pushed commit (contains it) -> landed rc0" { + run cl_push_landed_verdict "$A40" "$B40" "$C40" true + [[ "$status" -eq 0 ]] + [[ "$output" == "landed" ]] +} + +@test "cl_push_landed_verdict: non-fast-forward / remote moved without our commit -> not-landed rc1" { + run cl_push_landed_verdict "$A40" "$B40" "$C40" false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-landed" ]] +} + +@test "cl_push_landed_verdict: pushed sha equals the starting head -> no-advance rc1" { + run cl_push_landed_verdict "$A40" "$A40" "$A40" true + [[ "$status" -eq 1 ]] + [[ "$output" == "no-advance" ]] +} + +@test "cl_push_landed_verdict: any unknown sha fails closed -> unknown rc1" { + run cl_push_landed_verdict "" "$B40" "$B40" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] + run cl_push_landed_verdict "$A40" "" "$B40" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] + run cl_push_landed_verdict "$A40" "$B40" "" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] +} + +# --------------------------------------------------------------------------- +# cl_select_pass_claims — which replies did THIS pass post? +# --------------------------------------------------------------------------- + +_comments() { + # REST pulls/{pr}/comments shape: id, user.login, body, created_at. + jq -c -n --arg a "$A40" --arg b "$B40" '[ + {id: 1, user: {login: "donpetry-bot"}, created_at: "2026-10-01T09:00:00Z", + body: ("Fixed earlier.\n\n")}, + {id: 2, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:05:00Z", + body: ("Fixed in f.\n\n")}, + {id: 3, user: {login: "coderabbitai[bot]"}, created_at: "2026-10-02T10:06:00Z", + body: ("\n")}, + {id: 4, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:07:00Z", + body: "Skipping — ambiguous; leaving for a maintainer."}, + {id: 5, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:08:00Z", + body: "Fixed (marker only, no claim).\n"} + ]' +} + +@test "cl_select_pass_claims: selects only OUR claim/marker replies created since the pass started" { + run cl_select_pass_claims "$(_comments)" "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$status" -eq 0 ]] + # id 1 predates the pass; 3 is another account; 4 carries no marker. + [[ "$output" == "$(printf '2\t%s\n5\t' "$B40")" ]] +} + +@test "cl_select_pass_claims: matches the [bot]-suffixed REST login against a stripped BOT_USER" { + local json + json=$(jq -c -n --arg b "$B40" '[{id: 9, user: {login: "donpetry-bot[bot]"}, created_at: "2026-10-02T10:05:00Z", + body: ("x \n")}]') + run cl_select_pass_claims "$json" "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$output" == "$(printf '9\t%s' "$B40")" ]] +} + +@test "cl_select_pass_claims: no pass start -> selects nothing (never retracts history)" { + run cl_select_pass_claims "$(_comments)" "donpetry-bot" "" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +@test "cl_select_pass_claims: a non-array payload (stubbed/failed API) -> selects nothing" { + run cl_select_pass_claims '{"head":{"sha":"x"}}' "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +# --------------------------------------------------------------------------- +# cl_retract_body — the retracted reply can no longer authorize resolution +# --------------------------------------------------------------------------- + +@test "cl_retract_body: strips the addressed + claim markers and states the retraction" { + local body + body="Fixed in scripts/canary-rollout.sh: a transport failure is not cached. + + +" + run cl_retract_body "$body" "predates-pass" + [[ "$status" -eq 0 ]] + [[ "$output" == *"Retracted"* ]] + [[ "$output" == *"predates-pass"* ]] + [[ "$output" == *""* ]] + # The original text survives (quoted) for the record… + [[ "$output" == *"> Fixed in scripts/canary-rollout.sh"* ]] + # …but neither marker does, so the thread gate can never resolve on it. + [[ "$output" != *"dev-lead:addressed"* ]] + [[ "$output" != *"dev-lead:claim"* ]] + run acv_parse_claim "$(cl_retract_body "$body" "predates-pass")" + [[ "$status" -eq 1 ]] + [[ "$output" == "no-claim" ]] +} + +@test "cl_retract_body: an unknown reason is normalised to a safe token" { + run cl_retract_body "Fixed. " 'bad -->reason