From 8f5737545180970905198a09ac0b81b1706e593c Mon Sep 17 00:00:00 2001 From: don-petry <{}+don-petry@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:11:04 +0000 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20implement=20issue=20#2013=20?= =?UTF-8?q?=E2=80=94=20dev-lead=20`fix-reviews`:=20posts=20"Fixed"=20repli?= =?UTF-8?q?es=20for=20changes=20that=20were=20never=20pushed,=20and=20edit?= =?UTF-8?q?s=20tests=20to=20match=20its=20own=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/lint.yml | 2 + ...-write-mode-personas-on-a-shared-branch.md | 84 +++++ prompts/dev-lead/fix-bot-comment.md | 13 +- prompts/dev-lead/fix-reviews.md | 32 +- scripts/dev-lead-fix-reviews.sh | 193 +++++++++++- scripts/lib/addressed-claim-verify.sh | 56 +++- scripts/lib/claim-landing.sh | 155 ++++++++++ scripts/lib/test-tamper-guard.sh | 154 ++++++++++ .../unit/test_addressed_claim_verify.bats | 109 +++++++ tests/dev-lead/unit/test_claim_landing.bats | 288 ++++++++++++++++++ tests/dev-lead/unit/test_fix_reviews.bats | 186 ++++++++++- .../dev-lead/unit/test_test_tamper_guard.bats | 199 ++++++++++++ 12 files changed, 1444 insertions(+), 27 deletions(-) create mode 100644 docs/architecture/adr/0010-write-mode-personas-on-a-shared-branch.md create mode 100644 scripts/lib/claim-landing.sh create mode 100644 scripts/lib/test-tamper-guard.sh create mode 100644 tests/dev-lead/unit/test_claim_landing.bats create mode 100644 tests/dev-lead/unit/test_test_tamper_guard.bats diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e6cdb445a..0f7783f0b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -216,6 +216,8 @@ jobs: tests/dev-lead/unit/test_comment_disposition_verify.bats \ tests/dev-lead/unit/test_conflict_integrity.bats \ tests/dev-lead/unit/test_addressed_claim_verify.bats \ + tests/dev-lead/unit/test_claim_landing.bats \ + tests/dev-lead/unit/test_test_tamper_guard.bats \ tests/dev-lead/unit/test_fewshot_assemble.bats \ tests/dev-lead/unit/test_net_diff_guard.bats \ tests/dev-lead/unit/test_verify_approval_review.bats \ diff --git a/docs/architecture/adr/0010-write-mode-personas-on-a-shared-branch.md b/docs/architecture/adr/0010-write-mode-personas-on-a-shared-branch.md new file mode 100644 index 000000000..405d51797 --- /dev/null +++ b/docs/architecture/adr/0010-write-mode-personas-on-a-shared-branch.md @@ -0,0 +1,84 @@ +# 0010. Write-mode personas on a shared branch: verify what landed, retract what did not + +## Status + +proposed + +## Context + +ADR-0005 makes write access an opt-in gate: dev-lead's write surfaces are armed +by the `dev-lead` label. It does not say how a write-mode persona should behave +on a branch that **other actors are pushing to at the same time**, such as a +maintainer, a Claude session, or the auto-rebase bot. + +petry-projects/.github#1220 showed what goes wrong +([#2013](https://github.com/petry-projects/.github-private/issues/2013)). While +a human session was driving the same branch: + +- dev-lead posted "Fixed in …" replies for changes that never reached the + branch. Its claim named the PR's **first** commit: the head the model read + with `git rev-parse HEAD` *before* committing anything, because the prompt + told it to cite that SHA and also told it not to commit. +- CodeRabbit marked a thread "Confirmed as addressed" on the strength of one of + those replies, so an unverified claim cleared a review gate. +- A bot suggestion ("update the older test so the suite can pass") was applied + literally. That rewrote an existing test and inverted deliberate behavior. + +The no-clobber push (#1311/#1607) already prevented dev-lead from overwriting +another actor's commit. Everything *around* the push was still the model +describing itself. The #1692 claim check ran only when the harness resolved a +thread, and the pre-pass SHA passed it trivially. Nothing ever took a posted +claim back. That breaks ADR-0004's "Fail Loud, Never Fake". + +## Decision + +A write-mode persona sharing a branch follows four rules, each enforced by a +pure classifier with bats tests (ADR-0004), not by prompt text alone: + +1. **Incorporate or stand down, never force.** Before pushing, the persona + reconciles with the *true* remote head. It rebases a foreign commit in, or + stops and escalates when that cannot be done cleanly. A foreign commit is + never discarded (`push_no_clobber`, `scripts/lib/git-push-guard.sh`). +2. **"Pushed" is a verdict, not an assumption.** After the push, the harness + compares the pre-pass head, the pushed SHA and the re-fetched remote head + (`cl_push_landed_verdict`, `scripts/lib/claim-landing.sh`). Any result other + than `landed` counts as a failed push. +3. **A claim must name a commit this pass produced and that landed.** The claim + SHA must be on the reference head and **not** contained in the pre-pass + head (`acv_claim_in_pass`, `scripts/lib/addressed-claim-verify.sh`). The + harness checks this at thread-resolution time against the PR head. It + checks it again after every push outcome against the **remote** head. Every + claim reply posted during the pass that fails is **retracted**: both markers + are stripped and a retraction notice is added, so the reply can never + authorize resolution and other bots cannot read it as a fix. The outcomes + covered are a rejected or unverified push, a guard abort, an engine + failure, no commit, and a stale SHA. +4. **An existing test is not the persona's to rewrite.** A fix pass that + changes or deletes an existing test line, or adds a skip, without an + explicit cited `Test-Change-Justification:` trailer is refused before the + push and escalated to a human (`scripts/lib/test-tamper-guard.sh`). A bot + suggestion that contradicts an existing test goes to a human. + +The boundary a check can assert: **no dev-lead claim reply survives a pass +unless the claim's commit was produced by that pass and is on the remote +head.** + +## Consequences + +- A rejected, aborted or failed pass leaves retracted replies, not false + "Fixed" ones. The thread stays open and the next pass re-verifies. +- The fail-closed direction costs some true claims. When the push guard + rebases dev-lead's commits onto a foreign commit, the SHAs the model cited + no longer exist on the remote, so those claims are retracted even though the + fix landed. Re-mapping rebased SHAs is a possible follow-up. A stuck thread + is a nuisance; a wrongly cleared gate is a defect. +- The model must commit locally before it replies, so its claims can cite a + real commit. The prompts now say so. A pass that leaves its changes + uncommitted gets them committed by the harness, and its claims (citing the + pre-pass head) are retracted. +- Legitimate test changes need one extra line, the justification trailer. In + exchange, a bot-driven pass can no longer silently invert a tested behavior. +- Still open: whether a persona should *back off entirely* from a branch with + recent non-persona pushes, or require an opt-in label to push to a branch it + did not create. This ADR covers correctness of what is claimed. It does not + cover whether to participate at all. diff --git a/prompts/dev-lead/fix-bot-comment.md b/prompts/dev-lead/fix-bot-comment.md index 6e4897614..1d682992e 100644 --- a/prompts/dev-lead/fix-bot-comment.md +++ b/prompts/dev-lead/fix-bot-comment.md @@ -81,11 +81,11 @@ gh api graphql -f query=' ))))' ``` -For each thread you fixed, first **reply with the specific change** — name the file(s)/function(s) you touched and how the change addresses the finding (one or two concrete sentences; never just "done"). End the reply with **two** HTML comments: the addressed-marker `` **and** a machine-readable claim `` (#1692). The harness now verifies the claim against the pushed diff before resolving — a marker without a verifiable claim leaves the thread **unresolved**. Stamp both **only** on a genuine addressed reply, never on a skip note. Pass the body as a GraphQL variable so quotes and newlines are safe: +For each thread you fixed, first **commit the fix locally** (`git add -A && git commit -m "fix(bot): "`, never push), then **reply with the specific change** — name the file(s)/function(s) you touched and how the change addresses the finding (one or two concrete sentences; never just "done"). End the reply with **two** HTML comments: the addressed-marker `` **and** a machine-readable claim `` (#1692). The harness now verifies the claim against the pushed diff before resolving — a marker without a verifiable claim leaves the thread **unresolved**. Stamp both **only** on a genuine addressed reply, never on a skip note. Pass the body as a GraphQL variable so quotes and newlines are safe: ```bash # Replace THREAD_NODE_ID with the id value from the query above. -# Get the full 40-char head SHA the fix rides on with: git rev-parse HEAD +# Commit the fix FIRST, then read the SHA of that commit with: git rev-parse HEAD gh api graphql \ -f query='mutation($tid: ID!, $body: String!) { addPullRequestReviewThreadReply(input: {pullRequestReviewThreadId: $tid, body: $body}) { comment { id } } }' \ -f tid="THREAD_NODE_ID" \ @@ -95,7 +95,11 @@ gh api graphql \ " ``` -The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` (`git rev-parse HEAD`) and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. +The claim payload is schema `v1` — one comment per reply, with a full 40-char `sha` and a non-empty JSON array of repo-relative `files` exactly as they appear in the diff. The normative schema and parser live in `scripts/lib/addressed-claim-verify.sh`. + +**Commit before you claim (#2013).** `sha` must be a commit **this pass produced**: run `git rev-parse HEAD` *after* you commit the fix. Citing the head you started from never verifies; it is the stale-claim defect from petry-projects/.github#1220. After the push, the harness checks the remote head. If the push was rejected or not incorporated, a guard refused it, or the pass failed, **every claim reply you posted this pass is retracted**. If you make no commit for a thread, post no addressed-marker or claim on it. + +**Never rewrite an existing test to make a bot suggestion pass (#2013).** Run the **full** test suite. A previously-passing test that turns red is a reason to question the change, not the test. A bot suggestion that contradicts an existing test is not applied, and that includes "update the older test so the suite can pass". Reply explaining the conflict, **without** the addressed-marker, and leave it for a human. Changing or deleting an existing test line, or adding a `skip`, requires a cited `Test-Change-Justification: ` commit trailer. Without one, the harness's test-tamper guard refuses to push and escalates. Adding a new test never needs it. **Do not resolve the thread yourself.** You must not call the `resolveReviewThread` (or `unresolveReviewThread`) GraphQL mutation under any circumstance — resolution is done **only** by the harness (`dev-lead-fix-reviews.sh`), whose deterministic guards are the authoritative merge gate. The harness resolves every bot thread you addressed with an our-account addressed-marker reply, plus any thread from this bot marked `isOutdated: true`. Your reply and its marker are your only lever on resolution. @@ -203,7 +207,8 @@ Some bot comments are pure **operational notices**, not code findings: a rate-li - For every thread you fix, post a reply naming the specific change, ending with the addressed-marker — never reply-less - **Never resolve a thread yourself**: do not call the `resolveReviewThread` (or `unresolveReviewThread`) mutation. Resolution is the harness's job — it resolves the addressed and outdated threads from `${ACTOR}`; your only lever is the addressed-marker on your reply - Stay within the scope of the pull request's changed files where possible -- Do not commit or push — the CI workflow handles git operations after you finish +- Commit your fixes locally so your claims can cite them, but **never push**. The CI workflow pushes after you finish, verifies the push landed, and retracts any claim that did not land (#2013) +- Never edit an existing test to match your change. A conflicting bot suggestion goes to a human. An existing-test change needs a `Test-Change-Justification:` trailer (#2013) ## Output Format diff --git a/prompts/dev-lead/fix-reviews.md b/prompts/dev-lead/fix-reviews.md index 4c46b726f..db6653236 100644 --- a/prompts/dev-lead/fix-reviews.md +++ b/prompts/dev-lead/fix-reviews.md @@ -55,8 +55,9 @@ For each open review thread: 1. Read the relevant file(s) using Read/Grep/Glob tools 2. Understand the reviewer's concern 3. Apply the appropriate fix using Edit/Write tools -4. **Reply to the thread with the specific fix** — see below -5. **Do not resolve the thread yourself** — the harness resolves it (see "Resolution is the harness's responsibility" below). A marker-less human thread is never resolved, including one with `isOutdated: true` (outdated status never overrides marker ownership). +4. **Commit the fix locally** — `git add -A && git commit -m "fix(reviews): "` — **before** you reply, so the claim can name a commit this pass actually produced (see "Commit before you claim" below). Never push. +5. **Reply to the thread with the specific fix** — see below +6. **Do not resolve the thread yourself** — the harness resolves it (see "Resolution is the harness's responsibility" below). A marker-less human thread is never resolved, including one with `isOutdated: true` (outdated status never overrides marker ownership). #### Replying to a thread @@ -64,7 +65,7 @@ For every thread you fix, post a reply to that thread that states **specifically ```bash # Replace THREAD_NODE_ID with the id value from the thread JSON. -# Get the full 40-char head SHA the fix rides on with: git rev-parse HEAD +# Commit the fix FIRST, then read the SHA of that commit with: git rev-parse HEAD gh api graphql \ -f query='mutation($tid: ID!, $body: String!) { addPullRequestReviewThreadReply(input: {pullRequestReviewThreadId: $tid, body: $body}) { comment { id } } }' \ -f tid="THREAD_NODE_ID" \ @@ -79,11 +80,20 @@ gh api graphql \ | Field | Type | Rule | |---|---|---| | `v` | integer | Schema version — `1` today. A payload with any other `v` is unverifiable → the thread stays open. | -| `sha` | string | The **full 40-character** commit SHA your fix rides on (`git rev-parse HEAD`). Abbreviated SHAs are rejected. | +| `sha` | string | The **full 40-character** SHA of the commit **you made in this pass** that carries the fix (`git rev-parse HEAD` **after** committing it). Abbreviated SHAs are rejected. | | `files` | array of strings | The repo-relative POSIX path(s) your fix touches, **exactly as they appear in the diff** (no leading `./` or `/`, no quoting). Must be non-empty. It is a **JSON array** — never comma- or newline-separated. | Emit **exactly one** claim comment per reply — the harness treats zero or more-than-one as unverifiable and leaves the thread open. The harness then checks that the named commit is reachable from the PR head, its diff (or the cumulative `^..HEAD` range) is non-empty, and it touches at least one file in `files`. If any check fails, the thread stays unresolved — so name the real SHA and the real files. +#### Commit before you claim — a claim is checked, then kept or retracted (#2013) + +The harness checks every claim against what actually **landed**, not against what you meant to do: + +- The cited commit must be one **this pass produced**: it must be on the PR head and **not** already on the branch when the pass started. Citing the head you started from (`git rev-parse HEAD` before you committed anything) is the stale-claim defect from petry-projects/.github#1220. It never verifies. +- After pushing, the harness checks that the **remote** head contains the pushed commit. If the push was rejected, the remote moved and your commit could not be incorporated, a guard refused the push, or the pass failed, then **every claim reply you posted this pass is retracted**: the markers are stripped and a retraction notice is added. An unverified "Fixed" must never stand, because other review bots treat it as addressed. + +So: edit, **commit locally**, then reply citing that commit's SHA. If you end up making no commit for a thread, post no addressed-marker or claim on it. + For a thread that is `isOutdated: true` with no code change, a reply is optional — a one-line note that the referenced code no longer exists is helpful but not required. #### Resolution is the harness's responsibility — never call `resolveReviewThread` @@ -154,7 +164,7 @@ For **every other** comment (bot or human alike — a bot conflict report or tri | Disposition | Use when | Required evidence (harness-verified) | |---|---|---| -| `fixed` | you changed code to address the finding | `sha=` the **full 40-char** commit (`git rev-parse HEAD`) your fix rides on; the harness checks it is on the PR head with a non-empty diff | +| `fixed` | you changed code to address the finding | `sha=` the **full 40-char** SHA of the commit **you made in this pass** (`git rev-parse HEAD` *after* committing the fix locally); the harness checks it is on the PR head, was produced by this pass, and has a non-empty diff | | `out-of-scope` | the finding is real but belongs elsewhere | `ref=#` a tracking issue that **exists**; open one first if needed | | `invalid` | the finding is wrong / a false positive | a reply body with concrete reasoning (non-empty beyond the marker) | | `answered` | the comment asked a question you answer in the reply | a reply body that actually answers it | @@ -173,9 +183,14 @@ For **every other** comment (bot or human alike — a bot conflict report or tri After addressing all threads, run the test suite to ensure no regressions were introduced: 1. Identify the test command this repo uses (check AGENTS.md, `package.json`, `Makefile`, etc.) -2. Run the full test suite — all tests must pass -3. If a thread fix required adding new behavior, add or update tests to cover it +2. Run the **full** test suite, not only the tests near your change. All tests must pass. +3. If a thread fix required adding new behavior, **add** a test to cover it 4. **Do not suppress or delete tests to force a pass — fix the code instead** +5. **A previously-passing test that turns red is a signal about your change, not about the test.** Question the change first. An existing test often encodes a deliberate behavior. On petry-projects/.github#1220 a "success precedence" test that encoded deliberate recovery semantics was rewritten to "failure precedence" to match a bot's suggestion, which inverted that behavior. +6. **A bot suggestion that contradicts an existing test is not applied.** That includes a literal "update the old test so the suite passes". Reply on the thread explaining the conflict and naming the test, **without** the addressed-marker, so the thread stays open for a human. Do not rewrite the test. +7. **Changing or deleting an existing test line, or adding a `skip`, needs an explicit, cited justification** in a commit trailer: + `Test-Change-Justification: ` + The harness's test-tamper guard refuses to push a pass that changes an existing test without this trailer, and escalates it to a human. Adding a **new** test never needs it. ### Phase 3 — Rubber Duck Review @@ -197,7 +212,8 @@ Read every changed line as if you are the reviewer seeing the response: - Do not make changes beyond what the review threads request, except that fixing Tier-1 blockers (failure/timed_out/cancelled/action_required/stale/startup_failure CI checks and CHANGES_REQUESTED reviews) is always in-scope - Never revert or undo the PR's own committed changes to satisfy a neutral `COMMENTED`/overview review — that produces a net-zero diff that silently cancels the fix (#1340) - If a review thread is ambiguous, apply the most conservative interpretation -- Do not commit or push — the CI workflow handles git operations after you finish +- Commit your fixes locally so your claims can cite them, but **never push**. The CI workflow pushes after you finish, verifies the push landed, and retracts any claim that did not land (#2013) +- Never edit an existing test to match your change. A bot suggestion that contradicts an existing test goes to a human (reply without the marker). An existing-test change needs a `Test-Change-Justification:` trailer (#2013) ## Output Format diff --git a/scripts/dev-lead-fix-reviews.sh b/scripts/dev-lead-fix-reviews.sh index bdf459884..770e9c2d2 100755 --- a/scripts/dev-lead-fix-reviews.sh +++ b/scripts/dev-lead-fix-reviews.sh @@ -14,6 +14,11 @@ source "$(dirname "$0")/lib/conflict-integrity.sh" source "$(dirname "$0")/lib/review-change-evidence.sh" source "$(dirname "$0")/lib/resolution-integrity.sh" source "$(dirname "$0")/lib/addressed-claim-verify.sh" +# Claim landing (#2013): "did the push land?" as a pure verdict, and retraction of +# this pass's claim replies whose commit did not land on the remote head. +source "$(dirname "$0")/lib/claim-landing.sh" +# Test-tamper guard (#2013): a fix pass may not silently rewrite an existing test. +source "$(dirname "$0")/lib/test-tamper-guard.sh" # PR issue-comment disposition verifier (#1813): the issue-comment sibling of # addressed-claim-verify.sh. Turns a dev-lead comment-disposition reply into a # machine-checkable claim the harness verifies before minimizing the original @@ -107,6 +112,9 @@ if [ "${DEV_LEAD_DRY_RUN:-false}" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then # gate on a no-commit pass. RESOLUTION_BASE_SHA is set once here and never # reassigned, so the gate measures only whether THIS pass advanced the head. RESOLUTION_BASE_SHA="$(git rev-parse HEAD 2>/dev/null || true)" + # The pass boundary for the claim-retraction sweep (#2013): only claim replies + # created at/after this instant are this pass's to retract. + PASS_START_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" setup_git_identity # Backfill the five required description sections into a pre-existing PR whose # body still lacks them (#1805). Idempotent + marker-keyed, so open PRs heal on @@ -803,13 +811,22 @@ resolve_addressed_bot_threads() { # Gather the git facts for the claimed commit (the only impure step). Fail # closed: an unresolvable commit reads as on_head=false with empty diffs. - local facts on_head own_files cumulative_files commit_date - facts=$(acv_gather_commit_facts "$claim_sha") + local facts on_head in_base in_pass own_files cumulative_files commit_date + facts=$(acv_gather_commit_facts "$claim_sha" "${RESOLUTION_BASE_SHA:-}") on_head=$(printf '%s' "$facts" | jq -r '.on_head // false' 2>/dev/null || echo "false") if [ "$on_head" != "true" ]; then echo "::notice::skipping thread ${id} — claimed commit ${claim_sha} is not on PR #${PR_NUMBER}'s head branch; leaving unresolved (#1692)" continue fi + # #2013: the claimed commit must be one THIS pass produced — not the PR's first + # commit or any other pre-pass ancestor, which passes the on-head + intersection + # checks trivially (the petry-projects/.github#1220 `571a3b8` case). Parity with + # the issue-comment `fixed` check in resolve_dispositioned_comments (#1813). + in_base=$(printf '%s' "$facts" | jq -r 'if .in_base == false then "false" else "true" end' 2>/dev/null || echo "true") + if ! in_pass=$(acv_claim_in_pass "${RESOLUTION_BASE_SHA:-}" "$on_head" "$in_base"); then + echo "::notice::skipping thread ${id} — claimed commit ${claim_sha} was not produced by this pass (${in_pass}; base=${RESOLUTION_BASE_SHA:-}); leaving unresolved (#2013)" + continue + fi own_files=$(printf '%s' "$facts" | jq -r '.own_files[]? // empty' 2>/dev/null || echo "") cumulative_files=$(printf '%s' "$facts" | jq -r '.cumulative_files[]? // empty' 2>/dev/null || echo "") commit_date=$(printf '%s' "$facts" | jq -r '.commit_date // ""' 2>/dev/null || echo "") @@ -2192,14 +2209,170 @@ commit_and_push() { fi ;; esac + # Test-tamper guard (#2013): a bot-driven fix pass must not silently rewrite an + # existing test to make its own change pass (the petry-projects/.github#1220 + # `13927fc` shape). Refuse the push and escalate, like the no-op guard (rc 3). + case "$intent" in + fix-reviews|fix-bot-comment) + local ttg_out ttg_rc=0 ttg_files + ttg_out=$(ttg_scan_pass "${RESOLUTION_BASE_SHA:-}" HEAD) || ttg_rc=$? + ttg_files=$(printf '%s\n' "$ttg_out" | sed '1d' | paste -sd ',' - | sed 's/,/, /g') + if [ "$ttg_rc" -eq 1 ]; then + echo "::error::Test-tamper guard: the ${intent} pass changed or skipped existing test(s) [${ttg_files:-skip added}] with no Test-Change-Justification trailer — refusing to push (#2013)" + flag_test_tamper "$intent" "${ttg_files:-a skip was added to a test file}" + return 3 + elif [ "$ttg_rc" -eq 2 ]; then + echo "::warning::Test-tamper guard: pre-pass base unknown — could not check ${intent} for rewritten tests (#2013)" + elif [ "$(printf '%s\n' "$ttg_out" | head -1)" = "justified" ]; then + echo "::notice::Test-tamper guard: existing test(s) changed with a cited Test-Change-Justification: ${ttg_files:-skip added} (#2013)" + fi + ;; + esac # No-clobber push (#1311): never discard a concurrent writer's unseen commit. # push_no_clobber fast-forwards normally and only ever force-with-leases a # rewritten branch, aborting if the remote moved beyond what we fetched. + # A rejected push — or one the remote head does not reflect (#2013) — must not + # leave this pass's "Fixed" replies standing: retract them before failing. push_no_clobber || { echo "::error::git push failed — check remote access and branch permissions" >&2 + retract_unlanded_claims "$intent" failed exit 1 } + verify_push_landed || { + retract_unlanded_claims "$intent" failed + exit 1 + } + fi + return 0 +} + +# verify_push_landed — "the push landed" is a verdict on SHAs, not an assumption +# (#2013). Compares the immutable pre-pass head (RESOLUTION_BASE_SHA), the pushed +# HEAD, and the TRUE remote head via cl_push_landed_verdict; returns non-zero +# unless the remote contains the pushed commit. Without a PR checkout (no +# pre-pass head) or an upstream (no push target to read back) there is nothing to +# compare, so it warns and passes — the push itself already succeeded. +verify_push_landed() { + local start="${RESOLUTION_BASE_SHA:-}" pushed remote="" remote_rc=0 on_remote=false verdict + if [ -z "$start" ]; then + echo "::warning::push-landed check skipped — no pre-pass head recorded for this run (#2013)" + return 0 + fi + pushed="$(git rev-parse HEAD 2>/dev/null || true)" + remote=$(cl_remote_head) || remote_rc=$? + if [ "$remote_rc" -eq 1 ]; then + echo "::warning::push-landed check skipped — the branch has no upstream to read back (#2013)" + return 0 + fi + if [ -n "$remote" ] && [ -n "$pushed" ] && git merge-base --is-ancestor "$pushed" "$remote" 2>/dev/null; then + on_remote=true + fi + if ! verdict=$(cl_push_landed_verdict "$start" "$pushed" "$remote" "$on_remote"); then + echo "::error::push not verified (${verdict}): the remote head ${remote:-} does not contain the pushed commit ${pushed:-} (pre-pass head ${start}) — treating the push as failed (#2013)" >&2 + return 1 + fi + echo "::notice::push verified: ${pushed} is on the remote head ${remote} (#2013)" + return 0 +} + +# retract_unlanded_claims — the claim-retraction sweep (#2013). +# The model posts its addressed/claim replies from its own shell BEFORE the +# harness commits and pushes, so a failed push, a guard abort, an engine failure, +# or a claim naming a pre-pass commit left a false "Fixed" on the thread — and +# another bot could treat it as addressed. For every claim reply OUR account +# posted since PASS_START_ISO, verify its commit is one this pass produced AND is +# on the REMOTE head (acv_claim_in_pass); otherwise rewrite it with +# cl_retract_body, which strips both markers so it can never resolve a thread. +# +# The reference head is the remote head. When it cannot be read: on an `ok` +# outcome with no upstream, the local HEAD is what was pushed; in every other case +# nothing verifiably landed, so the pre-pass base is used and every claim fails. +# Idempotent: a retracted reply carries no marker and is never selected again. +retract_unlanded_claims() { + local intent="$1" outcome="${2:-ok}" + if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then + echo "[dry-run] would retract unlanded claim replies posted by this ${intent} pass on PR #${PR_NUMBER:-}" + return 0 + fi + [ -z "${PR_NUMBER:-}" ] && return 0 + if [ -z "${PASS_START_ISO:-}" ]; then + echo "::notice::retract_unlanded_claims: no pass start recorded — skipping (#2013)" + return 0 + fi + + local bot_user="${BOT_USER:-donpetry-bot}" base="${RESOLUTION_BASE_SHA:-}" + local ref="" ref_rc=0 + ref=$(cl_remote_head) || ref_rc=$? + if [ -z "$ref" ]; then + if [ "$ref_rc" -eq 1 ] && [ "$outcome" = "ok" ]; then + ref="$(git rev-parse HEAD 2>/dev/null || true)" + else + ref="$base" + fi + fi + + local comments + if ! comments=$(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/comments?per_page=100" 2>/dev/null | jq -s 'add // []' 2>/dev/null); then + echo "::warning::retract_unlanded_claims: could not list review comments on PR #${PR_NUMBER} — this pass's claim replies were NOT verified (#2013)" + return 0 + fi + + local rows retracted=0 id sha facts on_ref in_base reason body new_body + rows=$(cl_select_pass_claims "$comments" "$bot_user" "$PASS_START_ISO") + while IFS=$'\t' read -r id sha; do + [ -z "$id" ] && continue + if [ -z "$sha" ]; then + reason="unverifiable" + else + facts=$(acv_gather_commit_facts "$sha" "$base" "${ref:-HEAD}") + on_ref=$(printf '%s' "$facts" | jq -r '.on_head // false' 2>/dev/null || echo "false") + in_base=$(printf '%s' "$facts" | jq -r 'if .in_base == false then "false" else "true" end' 2>/dev/null || echo "true") + [ -z "$ref" ] && on_ref=false + if reason=$(acv_claim_in_pass "$base" "$on_ref" "$in_base"); then + continue + fi + fi + body=$(printf '%s' "$comments" | jq -r --arg id "$id" 'first(.[] | select((.id | tostring) == $id)) | .body // ""' 2>/dev/null || echo "") + new_body=$(cl_retract_body "$body" "$reason") + if gh api -X PATCH "repos/${REPO}/pulls/comments/${id}" -f body="$new_body" >/dev/null 2>&1; then + retracted=$((retracted + 1)) + echo "::warning::retracted claim reply ${id} on PR #${PR_NUMBER} (${reason}; cited ${sha:-}, remote head ${ref:-}) (#2013)" + else + echo "::error::could not retract claim reply ${id} on PR #${PR_NUMBER} (${reason}) — a false 'Fixed' reply may remain (#2013)" + fi + done <<< "$rows" + echo "::notice::retract_unlanded_claims: retracted ${retracted} claim reply(ies) from this ${intent} pass on PR #${PR_NUMBER} (outcome=${outcome})" +} + +# flag_test_tamper — the test-tamper guard refused the push +# (#2013). Mirrors flag_noop_pr: one deduped human-attention comment, the +# needs-human-review label, auto-merge disabled and its EXIT-trap restore +# suppressed. A bot suggestion that contradicts an existing test is a question for +# a human, never a reason to rewrite the test. +flag_test_tamper() { + local intent="$1" files="$2" + _AM_NEEDS_RESTORE=0 + if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then + echo "[dry-run] test-tamper guard: would flag PR #${PR_NUMBER} (${intent}), add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}, disable auto-merge" + return 0 fi + local marker="" + if gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments?per_page=100" 2>/dev/null \ + | jq -r '.[].body // ""' 2>/dev/null | grep -qF "$marker"; then + echo "::notice::PR #${PR_NUMBER} already flagged for test tampering for intent=${intent} — not reposting" + else + gh pr comment "$PR_NUMBER" --repo "$REPO" --body "${marker} +## Existing test changed without justification — human attention needed + +The \`${intent}\` pass changed, deleted, or skipped **existing** test(s) to go with its fix: ${files}. A previously-passing test that disagrees with a change is a reason to question the change, not to edit the test, so dev-lead **did not push** this pass (#2013). + +If the test really is wrong, a human should make that call, or the fix should carry a \`Test-Change-Justification:\` commit trailer that cites the reason (for example, the review comment that asked for it). Auto-merge has been disabled." \ + || echo "::warning::could not post test-tamper flag comment on PR #${PR_NUMBER}" + fi + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review}" 2>/dev/null \ + || echo "::warning::could not add ${NEEDS_HUMAN_REVIEW_LABEL:-needs-human-review} label on PR #${PR_NUMBER}" + gh pr merge "$PR_NUMBER" --repo "$REPO" --disable-auto 2>/dev/null \ + || echo "::notice::auto-merge was not enabled on PR #${PR_NUMBER} (nothing to disable)" return 0 } @@ -2423,6 +2596,9 @@ case "$INTENT_TYPE" in if [ "$rc" -eq 0 ]; then cp_rc=0 commit_and_push "fix-reviews" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "fix-reviews" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" if [ "$cp_rc" -eq 0 ]; then notify_coderabbit_resolve finalize_review_application "fix-reviews" @@ -2478,6 +2654,8 @@ case "$INTENT_TYPE" in # recovered in-process — but the next pass self-heals via the idempotent # posting + duplicate recovery above.) resolve_dispositioned_comments "fix-reviews" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "fix-reviews" failed fi exit "$rc" ;; @@ -2497,6 +2675,9 @@ case "$INTENT_TYPE" in # then leaves no marker, and the retry re-dispatches instead of stalling. _fbc_terminal="" commit_and_push "fix-bot-comment" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "fix-bot-comment" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" if [ "$cp_rc" -eq 0 ]; then notify_coderabbit_resolve _fbc_terminal="applied" @@ -2547,11 +2728,12 @@ case "$INTENT_TYPE" in fi try_enable_auto_merge fi - try_enable_auto_merge else # Don't orphan dispositions on a failed/timed-out pass (#1992) — see the # fix-reviews failure branch above for why running the resolver here is safe. resolve_dispositioned_comments "fix-bot-comment" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "fix-bot-comment" failed fi exit "$rc" ;; @@ -2610,6 +2792,9 @@ case "$INTENT_TYPE" in if [ "$rc" -eq 0 ]; then cp_rc=0 commit_and_push "review-changes" || cp_rc=$? + # Retract every claim reply this pass posted whose commit did not land on the + # remote head — no commit, a guard abort, or a stale pre-pass SHA (#2013). + retract_unlanded_claims "review-changes" "$([ "$cp_rc" -eq 0 ] && echo ok || echo failed)" # No-op guard (#1786): a review-changes pass whose net diff to base is empty # was already flagged by flag_noop_pr (needs-human label + auto-merge # disabled + suppressed EXIT-trap restore). Stop here — never resolve threads @@ -2654,6 +2839,8 @@ case "$INTENT_TYPE" in # Don't orphan dispositions on a failed/timed-out pass (#1992) — see the # fix-reviews failure branch above for why running the resolver here is safe. resolve_dispositioned_comments "review-changes" failed + # Nothing was pushed on a failed pass: retract its claim replies (#2013). + retract_unlanded_claims "review-changes" failed fi exit "$rc" ;; diff --git a/scripts/lib/addressed-claim-verify.sh b/scripts/lib/addressed-claim-verify.sh index 756ffdec6..f83cc7401 100644 --- a/scripts/lib/addressed-claim-verify.sh +++ b/scripts/lib/addressed-claim-verify.sh @@ -395,11 +395,46 @@ _acv_is_iso8601() { jq -e -n --arg s "$s" '($s | fromdateiso8601)' >/dev/null 2>&1 } -# acv_gather_commit_facts +# acv_claim_in_pass +# The in-pass verdict (#2013). A claim verifies only when its commit was produced +# by THIS pass: reachable from the reference head ( — the PR head at +# resolution time, or the REMOTE head when deciding whether a claim landed) and +# NOT already contained in the pre-pass base (). Without it a reply citing +# the PR's FIRST commit — the head the model read with `git rev-parse HEAD` before +# committing anything — passed the on-head + intersection checks trivially (the +# petry-projects/.github#1220 `571a3b8` case). This is the thread-path parity of the +# issue-comment `fixed` check in resolve_dispositioned_comments (#1813). +# / are the literal strings "true"/"false" from +# acv_gather_commit_facts; anything else fails closed. Echoes one token — +# in-pass | unknown-base | not-on-ref | predates-pass — and returns 0 only for +# in-pass. Pure — no gh/git/network. +acv_claim_in_pass() { + local base="${1:-}" on_ref="${2:-}" in_base="${3:-}" + if [[ -z "$base" ]]; then + echo "unknown-base" + return 1 + fi + if [[ "$on_ref" != "true" ]]; then + echo "not-on-ref" + return 1 + fi + if [[ "$in_base" != "false" ]]; then + echo "predates-pass" + return 1 + fi + echo "in-pass" + return 0 +} + +# acv_gather_commit_facts [base_sha] [ref] # The SINGLE impure gatherer: all git access lives here so the verifier above stays # pure. Echoes a JSON object: -# {"on_head":bool,"own_files":[...],"cumulative_files":[...],"commit_date":"iso"} -# - on_head: is reachable from HEAD (i.e. on the PR head branch). +# {"on_head":bool,"in_base":bool,"own_files":[...],"cumulative_files":[...],"commit_date":"iso"} +# - on_head: is reachable from (default HEAD — the PR head branch; the +# retraction sweep passes the REMOTE head so a local-only commit is not "on"). +# - in_base: is contained in (the pre-pass head, #2013). Fails +# closed to true when is empty or does not resolve, so an unknowable +# base can never make a stale commit look like this pass's work. # - own_files: files in 's own diff. # - cumulative_files: files in `^..HEAD` (falls back to `..HEAD` when # is a root commit with no parent). @@ -408,13 +443,17 @@ _acv_is_iso8601() { # never read an error as "verified". Runs in the current working directory (the PR # worktree at resolution time). acv_gather_commit_facts() { - local sha="${1:-}" - local on_head=false own_files='[]' cumulative_files='[]' commit_date="" + local sha="${1:-}" base="${2:-}" ref="${3:-HEAD}" + local on_head=false in_base=true own_files='[]' cumulative_files='[]' commit_date="" if [[ -n "$sha" ]] && git cat-file -e "${sha}^{commit}" 2>/dev/null; then - if git merge-base --is-ancestor "$sha" HEAD 2>/dev/null; then + if git merge-base --is-ancestor "$sha" "$ref" 2>/dev/null; then on_head=true fi + if [[ -n "$base" ]] && git cat-file -e "${base}^{commit}" 2>/dev/null \ + && ! git merge-base --is-ancestor "$sha" "$base" 2>/dev/null; then + in_base=false + fi # Emit a Z-terminated UTC ISO-8601 instant (e.g. 2026-09-07T21:40:05Z), the # SAME shape GitHub's createdAt uses. `%cI` would emit a `+00:00` offset that # jq's fromdateiso8601 rejects and that does not compare lexicographically @@ -432,9 +471,10 @@ acv_gather_commit_facts() { jq -c -n \ --argjson on_head "$on_head" \ + --argjson in_base "$in_base" \ --argjson own "$own_files" \ --argjson cumulative "$cumulative_files" \ --arg date "$commit_date" \ - '{on_head: $on_head, own_files: $own, cumulative_files: $cumulative, commit_date: $date}' \ - 2>/dev/null || printf '{"on_head":false,"own_files":[],"cumulative_files":[],"commit_date":""}' + '{on_head: $on_head, in_base: $in_base, own_files: $own, cumulative_files: $cumulative, commit_date: $date}' \ + 2>/dev/null || printf '{"on_head":false,"in_base":true,"own_files":[],"cumulative_files":[],"commit_date":""}' } diff --git a/scripts/lib/claim-landing.sh b/scripts/lib/claim-landing.sh new file mode 100644 index 000000000..073095f74 --- /dev/null +++ b/scripts/lib/claim-landing.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# claim-landing.sh — "did the push land, and which claims must be retracted?" as +# pure verdicts (#2013). +# +# WHAT IS WRONG WITHOUT THIS +# The fix-reviews / fix-bot-comment model posts its "Fixed in …" reply (with the +# addressed-marker + #1692 claim) from its OWN shell, BEFORE the harness commits +# and pushes. On petry-projects/.github#1220 that left false "Fixed" replies +# standing when the push never landed, and when the claim named the PR's FIRST +# commit (a stale pre-pass head). CodeRabbit then marked a thread "✅ Confirmed as +# addressed" on the strength of the reply, so an unverified claim cleared a gate. +# The #1692 claim check ran only at thread-RESOLUTION time; nothing ever took a +# posted claim back. +# +# THE CONTRACT +# - cl_push_landed_verdict decides, from SHAs alone, whether the pass's pushed +# commit is on the remote head. commit_and_push treats anything but `landed` as +# a failed push. +# - cl_select_pass_claims picks OUR claim/marker replies created during this pass. +# - each selected claim is checked with acv_claim_in_pass against the REMOTE head +# (addressed-claim-verify.sh); any that fails is rewritten with cl_retract_body, +# which strips both markers so the reply can never authorize resolution again. +# +# PURITY / TESTABILITY (ADR-0004) +# Every cl_* function except cl_remote_head is PURE: no network, no git, no gh. +# cl_remote_head is the single impure reader. Sourced under `set -euo pipefail`, +# so helpers only `return`, never `exit`, and fail closed on every ambiguity. + +set -euo pipefail + +# acv_parse_claim is the single claim parser — reuse it, never a second one. +if ! declare -F acv_parse_claim >/dev/null 2>&1; then + # shellcheck source=addressed-claim-verify.sh + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/addressed-claim-verify.sh" +fi + +# The retraction marker. Matches the shared `` agent-marker +# discriminator, so a retracted reply is still recognised as ours, but it is NOT the +# addressed-marker, so it can never authorize resolution. +readonly _CL_RETRACTED_PREFIX='`. +# is restricted to [a-z-] (anything else becomes `unknown`) so it can never break +# out of the HTML comment. Pure. +cl_retract_body() { + local body="${1:-}" reason="${2:-unknown}" + [[ "$reason" =~ ^[a-z][a-z-]*$ ]] || reason="unknown" + + local stripped + stripped=$(jq -Rsr --arg marker "$_DEV_LEAD_ADDRESSED_MARKER" ' + gsub($marker; "") + | gsub(""; "") + | sub("\\s+$"; "") + ' <<<"$body" 2>/dev/null) || stripped="" + local quoted + quoted=$(printf '%s\n' "$stripped" | sed 's/^/> /') + + printf '%s\n\n%s\n\n%s\n\n%s%s -->\n' \ + "**⚠️ Retracted by the dev-lead harness (\`${reason}\`).** The fix this reply describes is **not** on the PR head: its commit was not produced and pushed by this pass, or the push did not land. The thread stays open; the next dev-lead pass re-verifies against the pushed diff." \ + "Original reply, kept for the record (it does not describe the PR head):" \ + "$quoted" \ + "$_CL_RETRACTED_PREFIX" "$reason" +} + +# cl_remote_head +# The single impure reader: fetch the current branch's upstream and echo the TRUE +# remote head SHA (not a stale remote-tracking ref). Returns 1 (echoing nothing) +# when the branch has no upstream, 2 when the fetch or lookup fails. +cl_remote_head() { + local up remote branch + up=$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true) + [[ -z "$up" ]] && return 1 + remote="${up%%/*}" + branch="${up#*/}" + git fetch --quiet "$remote" "$branch" 2>/dev/null || return 2 + local sha + sha=$(git rev-parse --verify --quiet "refs/remotes/${remote}/${branch}" 2>/dev/null \ + || git rev-parse --verify --quiet FETCH_HEAD 2>/dev/null || true) + [[ -z "$sha" ]] && return 2 + echo "$sha" +} diff --git a/scripts/lib/test-tamper-guard.sh b/scripts/lib/test-tamper-guard.sh new file mode 100644 index 000000000..16c656116 --- /dev/null +++ b/scripts/lib/test-tamper-guard.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# test-tamper-guard.sh — a fix pass must not silently rewrite an existing test to +# make its own change pass (#2013). +# +# WHAT IS WRONG WITHOUT THIS +# On petry-projects/.github#1220 a cubic finding said, literally, "update the +# older test to assert failure precedence so the Bats suite can pass". dev-lead +# applied it: it rewrote an existing test (#1023 "success precedence …") into +# "failure precedence", inverting a deliberate behavior, and the rewritten test +# did not even match the branch. "Do not modify tests to force a pass" lived only +# in the prompt, so nothing in the harness could stop it. +# +# THE RULE +# A fix pass (base = the pre-pass head) is: +# clean it changes no existing line of a test file and adds no skip — +# ADDING a new test is always fine; +# justified it does change/delete an existing test line or add a skip, AND a +# commit in the pass carries an explicit, cited +# `Test-Change-Justification: ` trailer; +# tampered it does so with no justification. The harness refuses to push and +# escalates to a human. +# "Changed" means the test file has REMOVED lines in `git diff --numstat` (an +# edit is a removal + an addition; a deletion is all removals), so a pure +# addition never trips it. +# +# PURITY / TESTABILITY (ADR-0004) +# ttg_is_test_path, ttg_touched_existing_tests, ttg_count_added_skips, +# ttg_has_justification and ttg_classify are PURE. ttg_scan_pass is the single +# impure gatherer (all git calls). Sourced under `set -euo pipefail`; helpers only +# `return`, never `exit`. + +set -euo pipefail + +# Minimum length of the justification text — rules out a bare "so it passes". +readonly _TTG_MIN_JUSTIFICATION_CHARS=20 + +# Added lines that silence a test, across the frameworks the org uses. +readonly _TTG_SKIP_RE='^\+[[:space:]]*(skip([[:space:]]|$|\()|@pytest\.mark\.(skip|xfail)|pytest\.(skip|xfail)\(|@unittest\.skip|@Disabled|@Ignore|#\[ignore\]|t\.Skip(Now|f)?\(|(it|test|describe)\.(skip|todo)\(|x(it|describe|test)\()' + +# ttg_is_test_path +# 0 when is a test file: under a tests?/ / __tests__/ / spec/ directory, a +# .bats file, or a test-named file (test_*.py, *_test.*, *.test.*, *.spec.*, +# *_spec.rb). Pure. +ttg_is_test_path() { + local p="${1:-}" + [[ -z "$p" ]] && return 1 + local base="${p##*/}" + case "/$p" in + */tests/*|*/test/*|*/__tests__/*|*/spec/*) return 0 ;; + esac + case "$base" in + *.bats|test_*.py|*_test.*|*.test.*|*.spec.*|*_spec.rb) return 0 ;; + esac + return 1 +} + +# ttg_touched_existing_tests +# is `git diff --numstat` output (`added\tdeleted\tpath`). Echoes, one +# per line, every test path with removed lines (changed or deleted existing +# lines). A binary row (`-\t-`) on a test path is reported too (fail closed). +# Pure. +ttg_touched_existing_tests() { + local numstat="${1:-}" + local _added deleted path + while IFS=$'\t' read -r _added deleted path; do + [[ -z "${path:-}" ]] && continue + ttg_is_test_path "$path" || continue + if [[ "$deleted" == "-" ]] || { [[ "$deleted" =~ ^[0-9]+$ ]] && (( deleted > 0 )); }; then + printf '%s\n' "$path" + fi + done <<<"$numstat" +} + +# ttg_count_added_skips +# Number of ADDED lines in test files (per the `+++ b/` headers) that skip +# or disable a test. Pure. +ttg_count_added_skips() { + local diff="${1:-}" + local line cur="" n=0 + while IFS= read -r line; do + if [[ "$line" == "+++ "* ]]; then + cur="${line#+++ }" + cur="${cur#b/}" + continue + fi + [[ -z "$cur" ]] && continue + ttg_is_test_path "$cur" || continue + [[ "$line" =~ $_TTG_SKIP_RE ]] && n=$((n + 1)) + done <<<"$diff" + echo "$n" +} + +# ttg_has_justification +# 0 when some line is `Test-Change-Justification: ` with at least +# _TTG_MIN_JUSTIFICATION_CHARS of non-blank text. Pure. +ttg_has_justification() { + local msgs="${1:-}" line text + while IFS= read -r line; do + [[ "$line" =~ ^[[:space:]]*Test-Change-Justification:[[:space:]]*(.*)$ ]] || continue + text="${BASH_REMATCH[1]}" + text="${text%"${text##*[![:space:]]}"}" + (( ${#text} >= _TTG_MIN_JUSTIFICATION_CHARS )) && return 0 + done <<<"$msgs" + return 1 +} + +# ttg_classify +# Echoes clean | justified | tampered; returns 1 only for tampered. A +# non-numeric skip count fails closed (tampered). Pure. +ttg_classify() { + local touched="${1:-}" skips="${2:-}" justified="${3:-false}" + local has_touch=0 + [[ -n "$(printf '%s' "$touched" | tr -d '[:space:]')" ]] && has_touch=1 + if ! [[ "$skips" =~ ^[0-9]+$ ]]; then + echo "tampered" + return 1 + fi + if (( has_touch == 0 && skips == 0 )); then + echo "clean" + return 0 + fi + if [[ "$justified" == "true" ]]; then + echo "justified" + return 0 + fi + echo "tampered" + return 1 +} + +# ttg_scan_pass [head] +# The single impure gatherer. Compares against the WORKING TREE (so +# committed and not-yet-committed edits are both seen), reads the justification +# from the commit messages in .., and echoes the verdict on the +# first line followed by the offending test paths (one per line). Returns 0 for +# clean/justified, 1 for tampered, 2 (echoing `unknown`) when is empty +# or does not resolve. +ttg_scan_pass() { + local base="${1:-}" head="${2:-HEAD}" + if [[ -z "$base" ]] || ! git cat-file -e "${base}^{commit}" 2>/dev/null; then + echo "unknown" + return 2 + fi + local numstat diff msgs touched skips justified=false verdict rc=0 + numstat=$(git diff --numstat "$base" 2>/dev/null) || { echo "unknown"; return 2; } + diff=$(git diff --unified=0 "$base" 2>/dev/null) || { echo "unknown"; return 2; } + msgs=$(git log --format=%B "${base}..${head}" 2>/dev/null || true) + touched=$(ttg_touched_existing_tests "$numstat") + skips=$(ttg_count_added_skips "$diff") + ttg_has_justification "$msgs" && justified=true + verdict=$(ttg_classify "$touched" "$skips" "$justified") || rc=$? + echo "$verdict" + [[ -n "$touched" ]] && printf '%s\n' "$touched" + return "$rc" +} diff --git a/tests/dev-lead/unit/test_addressed_claim_verify.bats b/tests/dev-lead/unit/test_addressed_claim_verify.bats index 403b6de08..a49850f59 100644 --- a/tests/dev-lead/unit/test_addressed_claim_verify.bats +++ b/tests/dev-lead/unit/test_addressed_claim_verify.bats @@ -409,3 +409,112 @@ setup() { [[ "$(echo "$output" | jq -r .on_head)" == "false" ]] [[ "$(echo "$output" | jq -c .own_files)" == "[]" ]] } + +# --------------------------------------------------------------------------- +# acv_claim_in_pass — the claim SHA must be a commit THIS pass produced (#2013) +# --------------------------------------------------------------------------- +# The thread-resolution gate used to accept any commit on the head branch whose +# ^..HEAD range touched the claimed file, so a reply citing the PR's FIRST +# commit (the pre-pass head the model read with `git rev-parse HEAD` before it +# committed anything) passed trivially — the petry-projects/.github#1220 `571a3b8` +# case. A claim now verifies only when its commit is on the reference head AND +# is not already contained in the pre-pass base. + +@test "acv_claim_in_pass: on ref and not in the pre-pass base -> in-pass rc0" { + run acv_claim_in_pass "$SHA40" true false + [[ "$status" -eq 0 ]] + [[ "$output" == "in-pass" ]] +} + +@test "acv_claim_in_pass: commit already in the pre-pass base (stale SHA) -> predates-pass rc1" { + run acv_claim_in_pass "$SHA40" true true + [[ "$status" -eq 1 ]] + [[ "$output" == "predates-pass" ]] +} + +@test "acv_claim_in_pass: commit not on the reference head -> not-on-ref rc1" { + run acv_claim_in_pass "$SHA40" false false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-on-ref" ]] +} + +@test "acv_claim_in_pass: unknown pre-pass base -> unknown-base rc1 (fail closed)" { + run acv_claim_in_pass "" true false + [[ "$status" -eq 1 ]] + [[ "$output" == "unknown-base" ]] +} + +@test "acv_claim_in_pass: any non-'true'/'false' fact fails closed" { + run acv_claim_in_pass "$SHA40" "" false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-on-ref" ]] + run acv_claim_in_pass "$SHA40" true "" + [[ "$status" -eq 1 ]] + [[ "$output" == "predates-pass" ]] +} + +@test "acv_gather_commit_facts: with a base, PR commit A cited after this pass pushed B -> in_base true (predates-pass)" { + # The QA fixture from #2013: the PR's own commit A touches F; this pass pushes B + # (also touching F). A claim {sha:A, files:[F]} passes the old on-head + + # intersection checks but must NOT verify as in-pass. + local repo="$BATS_TEST_TMPDIR/repo3" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/F" + git -C "$repo" add F + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m A + local a + a="$(git -C "$repo" rev-parse HEAD)" + echo two >> "$repo/F" + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -am B + local b + b="$(git -C "$repo" rev-parse HEAD)" + + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$a' '$a'" + [[ "$status" -eq 0 ]] + [[ "$(echo "$output" | jq -r .on_head)" == "true" ]] + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] + run bash -c "source '$LIB' && acv_claim_in_pass '$a' true true" + [[ "$output" == "predates-pass" ]] + + # B — the commit this pass actually produced — is in-pass. + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$b' '$a'" + [[ "$(echo "$output" | jq -r .in_base)" == "false" ]] + [[ "$(echo "$output" | jq -r .on_head)" == "true" ]] +} + +@test "acv_gather_commit_facts: an unresolvable base fails closed (in_base true)" { + local repo="$BATS_TEST_TMPDIR/repo4" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/a.txt" + git -C "$repo" add a.txt + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m c0 + local h + h="$(git -C "$repo" rev-parse HEAD)" + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$h' 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef'" + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] + # No base given at all is also fail-closed. + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$h'" + [[ "$(echo "$output" | jq -r .in_base)" == "true" ]] +} + +@test "acv_gather_commit_facts: an explicit reference ref replaces HEAD for on_head" { + # The retraction sweep checks claims against the REMOTE head, not local HEAD: + # a commit that exists locally but never reached the remote is not on the ref. + local repo="$BATS_TEST_TMPDIR/repo5" + mkdir -p "$repo" + git -C "$repo" init -q + echo one > "$repo/a.txt" + git -C "$repo" add a.txt + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -m c0 + local base + base="$(git -C "$repo" rev-parse HEAD)" + echo two >> "$repo/a.txt" + git -C "$repo" -c user.email=t@t -c user.name=T commit -q -am local-only + local local_only + local_only="$(git -C "$repo" rev-parse HEAD)" + run bash -c "cd '$repo' && source '$LIB' && acv_gather_commit_facts '$local_only' '$base' '$base'" + [[ "$(echo "$output" | jq -r .on_head)" == "false" ]] + [[ "$(echo "$output" | jq -r .in_base)" == "false" ]] +} diff --git a/tests/dev-lead/unit/test_claim_landing.bats b/tests/dev-lead/unit/test_claim_landing.bats new file mode 100644 index 000000000..fe3d0aff4 --- /dev/null +++ b/tests/dev-lead/unit/test_claim_landing.bats @@ -0,0 +1,288 @@ +#!/usr/bin/env bats +# Unit tests for scripts/lib/claim-landing.sh (#2013). +# +# On petry-projects/.github#1220 dev-lead posted "Fixed in …" replies for changes +# that never reached the branch: the model posts its addressed/claim reply from +# its own shell BEFORE the harness commits and pushes, so a rejected push, a +# no-op-guard abort, an engine failure, or a claim citing the pre-pass head left a +# false "Fixed" on the thread — and CodeRabbit marked a thread addressed on the +# strength of it. claim-landing.sh is the pure verdict layer that decides whether +# a push landed and which of this pass's claim replies must be retracted, plus +# one impure remote-head reader. The harness wiring is covered in +# test_fix_reviews.bats. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +LIB="$SCRIPT_DIR/scripts/lib/claim-landing.sh" +# addressed-claim-verify.sh (acv_*) is sourced transitively by claim-landing.sh. +GUARD_LIB="$SCRIPT_DIR/scripts/lib/git-push-guard.sh" + +A40="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +B40="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +C40="cccccccccccccccccccccccccccccccccccccccc" + +setup() { + # shellcheck source=scripts/lib/claim-landing.sh + source "$LIB" +} + +# --------------------------------------------------------------------------- +# Sourcing safety +# --------------------------------------------------------------------------- + +@test "claim-landing.sh is safe to source under set -euo pipefail" { + run bash -c "set -euo pipefail; source '$LIB'" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +# --------------------------------------------------------------------------- +# cl_push_landed_verdict — "push landed" is a pure verdict, not a model claim +# --------------------------------------------------------------------------- + +@test "cl_push_landed_verdict: remote head == pushed sha, advanced past start -> landed rc0" { + run cl_push_landed_verdict "$A40" "$B40" "$B40" true + [[ "$status" -eq 0 ]] + [[ "$output" == "landed" ]] +} + +@test "cl_push_landed_verdict: remote moved past our pushed commit (contains it) -> landed rc0" { + run cl_push_landed_verdict "$A40" "$B40" "$C40" true + [[ "$status" -eq 0 ]] + [[ "$output" == "landed" ]] +} + +@test "cl_push_landed_verdict: non-fast-forward / remote moved without our commit -> not-landed rc1" { + run cl_push_landed_verdict "$A40" "$B40" "$C40" false + [[ "$status" -eq 1 ]] + [[ "$output" == "not-landed" ]] +} + +@test "cl_push_landed_verdict: pushed sha equals the starting head -> no-advance rc1" { + run cl_push_landed_verdict "$A40" "$A40" "$A40" true + [[ "$status" -eq 1 ]] + [[ "$output" == "no-advance" ]] +} + +@test "cl_push_landed_verdict: any unknown sha fails closed -> unknown rc1" { + run cl_push_landed_verdict "" "$B40" "$B40" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] + run cl_push_landed_verdict "$A40" "" "$B40" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] + run cl_push_landed_verdict "$A40" "$B40" "" true + [[ "$output" == "unknown" ]]; [[ "$status" -eq 1 ]] +} + +# --------------------------------------------------------------------------- +# cl_select_pass_claims — which replies did THIS pass post? +# --------------------------------------------------------------------------- + +_comments() { + # REST pulls/{pr}/comments shape: id, user.login, body, created_at. + jq -c -n --arg a "$A40" --arg b "$B40" '[ + {id: 1, user: {login: "donpetry-bot"}, created_at: "2026-10-01T09:00:00Z", + body: ("Fixed earlier.\n\n")}, + {id: 2, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:05:00Z", + body: ("Fixed in f.\n\n")}, + {id: 3, user: {login: "coderabbitai[bot]"}, created_at: "2026-10-02T10:06:00Z", + body: ("\n")}, + {id: 4, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:07:00Z", + body: "Skipping — ambiguous; leaving for a maintainer."}, + {id: 5, user: {login: "donpetry-bot"}, created_at: "2026-10-02T10:08:00Z", + body: "Fixed (marker only, no claim).\n"} + ]' +} + +@test "cl_select_pass_claims: selects only OUR claim/marker replies created since the pass started" { + run cl_select_pass_claims "$(_comments)" "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$status" -eq 0 ]] + # id 1 predates the pass; 3 is another account; 4 carries no marker. + [[ "$output" == "$(printf '2\t%s\n5\t' "$B40")" ]] +} + +@test "cl_select_pass_claims: matches the [bot]-suffixed REST login against a stripped BOT_USER" { + local json + json=$(jq -c -n --arg b "$B40" '[{id: 9, user: {login: "donpetry-bot[bot]"}, created_at: "2026-10-02T10:05:00Z", + body: ("x \n")}]') + run cl_select_pass_claims "$json" "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$output" == "$(printf '9\t%s' "$B40")" ]] +} + +@test "cl_select_pass_claims: no pass start -> selects nothing (never retracts history)" { + run cl_select_pass_claims "$(_comments)" "donpetry-bot" "" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +@test "cl_select_pass_claims: a non-array payload (stubbed/failed API) -> selects nothing" { + run cl_select_pass_claims '{"head":{"sha":"x"}}' "donpetry-bot" "2026-10-02T10:00:00Z" + [[ "$status" -eq 0 ]] + [[ -z "$output" ]] +} + +# --------------------------------------------------------------------------- +# cl_retract_body — the retracted reply can no longer authorize resolution +# --------------------------------------------------------------------------- + +@test "cl_retract_body: strips the addressed + claim markers and states the retraction" { + local body + body="Fixed in scripts/canary-rollout.sh: a transport failure is not cached. + + +" + run cl_retract_body "$body" "predates-pass" + [[ "$status" -eq 0 ]] + [[ "$output" == *"Retracted"* ]] + [[ "$output" == *"predates-pass"* ]] + [[ "$output" == *""* ]] + # The original text survives (quoted) for the record… + [[ "$output" == *"> Fixed in scripts/canary-rollout.sh"* ]] + # …but neither marker does, so the thread gate can never resolve on it. + [[ "$output" != *"dev-lead:addressed"* ]] + [[ "$output" != *"dev-lead:claim"* ]] + run acv_parse_claim "$(cl_retract_body "$body" "predates-pass")" + [[ "$status" -eq 1 ]] + [[ "$output" == "no-claim" ]] +} + +@test "cl_retract_body: an unknown reason is normalised to a safe token" { + run cl_retract_body "Fixed. " 'bad -->reason