From 70459e74dd337f999f80dac852508bfb15fe67a2 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:46:57 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20implement=20issue=20#1242=20?= =?UTF-8?q?=E2=80=94=20canary-rollout:=20move=20the=20pair/ring=20verdict?= =?UTF-8?q?=20(per-pair=20state,=20tag-gap=20hold)=20into=20the=20pure=20g?= =?UTF-8?q?ate=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/canary-rollout.sh | 33 +++++++----- scripts/lib/canary-rollout.sh | 24 +++++++++ tests/canary_rollout.bats | 96 +++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 12 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index b118d5e3a..465e483ab 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1313,16 +1313,24 @@ _frontier_state() { for i in "${!chan_array[@]}"; do ch="${chan_array[$i]}" if [ -n "$prev" ]; then - # Pending iff dst is not on src's commit. An UNRESOLVABLE src commit (empty) with a populated - # dst is still pending and _pair_state holds it BLOCKED (fail closed, as the single-frontier - # code did) — never silently skipped into a false COMPLETE. + # The pair verdict is the pure core's (pair_verdict, #1242); this loop only acts on it. A + # tag-lookup error is not yet distinguishable from an absent tag here (channel_commit, #1225), + # so both rings are passed as known (unknown=0). cand="${commits[$((i-1))]}" dstc="${commits[$i]}" - if [ "$dstc" != "$cand" ]; then - transition="${prev}->${ch}" - _pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv" - emitted=1 - fi + transition="${prev}->${ch}" + case "$(pair_verdict "${cand:--}" 0 "${dstc:--}" 0)" in + ON_CANDIDATE) ;; + PENDING|UNRESOLVABLE_SOURCE) + # An UNRESOLVABLE src commit (empty) with a populated dst is still pending and _pair_state + # holds it BLOCKED (fail closed, as the single-frontier code did). + _pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv" + emitted=1 ;; + *) + # HOLD_UNKNOWN (or any unrecognised verdict): fail closed — held BLOCKED, never evaluated. + echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 - - - 0 0 0 0" + emitted=1 ;; + esac fi prev="$ch" done @@ -1900,10 +1908,11 @@ _frontier_state_resilient() { if [ -n "$prev" ]; then cand="$prev_commit" dstc="$ch_commit" - # Same pending rule as _frontier_state: dst not on src's commit, including an unresolvable - # (empty) src commit, which stays tracked as BLOCKED rather than vanishing (fail closed). - if [ "$dstc" != "$cand" ]; then - transition="${prev}->${ch}" + transition="${prev}->${ch}" + # Same pure pair verdict as _frontier_state (#1242): a pending pair, including an + # unresolvable (empty) src commit, stays tracked as BLOCKED rather than vanishing (fail + # closed); anything other than ON_CANDIDATE is held. + if [ "$(pair_verdict "${cand:--}" 0 "${dstc:--}" 0)" != "ON_CANDIDATE" ]; then prior="$dstc" differs="$(_reusable_differs "$agent" "$cand" "$prior")" # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index 74d993720..91d6403d9 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -437,6 +437,30 @@ transition_key() { echo "" } +# pair_verdict +# Pure per-pair verdict for ONE adjacent ring transition src->dst (#1242). The orchestrator only +# gathers the facts: each ring's resolved commit ("-" or empty = no tag, i.e. ABSENT) and whether +# its tag lookup ERRORED (unknown=1, #1225). An empty unknown flag means known (0); any other +# non-0 value fails closed as unknown. Echoes exactly one of, in precedence order: +# HOLD_UNKNOWN — either ring's lookup errored: its commit is not known, so the pair is held +# BLOCKED and never evaluated or promoted, even with --override (#1225). An +# API outage must never read as "dst is on the candidate", nor a total +# outage as COMPLETE. +# ON_CANDIDATE — dst is already on src's commit (including both ABSENT, the legacy +# behaviour, #1118 AC6): nothing pending. +# UNRESOLVABLE_SOURCE — src has no tag but dst does: still pending and held BLOCKED (fail closed), +# never silently skipped into a false COMPLETE (#1118). +# PENDING — dst is not on src's commit: evaluate the gate for this pair. A ring only +# ever advances to the commit on the ring directly below it, so no tier is +# skipped (#1118). +pair_verdict() { + local src="${1:--}" su="${2:-0}" dst="${3:--}" du="${4:-0}" + if [ "$su" != "0" ] || [ "$du" != "0" ]; then echo "HOLD_UNKNOWN"; return 0; fi + if [ "$src" = "$dst" ]; then echo "ON_CANDIDATE"; return 0; fi + if [ "$src" = "-" ]; then echo "UNRESOLVABLE_SOURCE"; return 0; fi + echo "PENDING" +} + # ── set-diff core (drift detection, #1082) ──────────────────────────────────── # The registry (.agents{}) is the MANUAL source of truth for what the canary pipeline # manages; drift detection diffs it against the *-reusable.yml actually present on each diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 851c96c19..757ac3853 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -808,6 +808,102 @@ GHEOF [ "$(transition_key stable 'next,ring0,ring1,stable')" = "ring1->stable" ] } +# ── pair_verdict (pure per-pair verdict, #1242: #1118 pending rule + #1225 tag-gap hold) ── +# Each ring is one of: RESOLVED (a commit; same or different from the other side), ABSENT +# ("-", unknown=0 — legacy "no tag"), or ERRORED (unknown=1 — the lookup failed, so the +# commit is not known). Full {resolved, absent, errored} × {src, dst} matrix, gh-free. +@test "pair_verdict: resolved src, dst on the same commit → ON_CANDIDATE (nothing pending)" { + [ "$(pair_verdict aaa 0 aaa 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: resolved src, dst on a different commit → PENDING" { + [ "$(pair_verdict aaa 0 bbb 0)" = "PENDING" ] +} +@test "pair_verdict: resolved src, absent dst → PENDING (dst never received a release)" { + [ "$(pair_verdict aaa 0 - 0)" = "PENDING" ] +} +@test "pair_verdict: resolved src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict aaa 0 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: absent src, resolved dst → UNRESOLVABLE_SOURCE (BLOCKED, never a false COMPLETE)" { + [ "$(pair_verdict - 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ] +} +@test "pair_verdict: absent src, absent dst → ON_CANDIDATE (legacy: nothing to promote)" { + [ "$(pair_verdict - 0 - 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: absent src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 0 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, resolved dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 1 bbb 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, absent dst → HOLD_UNKNOWN (an outage is never read as 'on the candidate')" { + [ "$(pair_verdict - 1 - 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 1 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: precedence — an errored side beats a matching commit (HOLD_UNKNOWN over ON_CANDIDATE)" { + [ "$(pair_verdict aaa 1 aaa 0)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 0 aaa 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: precedence — an errored side beats an unresolvable source and a pending pair" { + [ "$(pair_verdict - 1 bbb 1)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 1 bbb 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: an empty commit is the absent sentinel '-' (same verdicts as '-')" { + [ "$(pair_verdict '' 0 '' 0)" = "ON_CANDIDATE" ] + [ "$(pair_verdict '' 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ] + [ "$(pair_verdict aaa 0 '' 0)" = "PENDING" ] + [ "$(pair_verdict '' 0 - 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: an omitted/empty unknown flag means known (0)" { + [ "$(pair_verdict aaa '' aaa '')" = "ON_CANDIDATE" ] + [ "$(pair_verdict aaa)" = "PENDING" ] +} +@test "pair_verdict: any unknown flag other than 0 fails CLOSED to HOLD_UNKNOWN" { + [ "$(pair_verdict aaa yes aaa 0)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 0 aaa 2)" = "HOLD_UNKNOWN" ] +} + +# _pv_rings ... — fold pair_verdict over an ordered ring list, echoing one +# " " per pair NOT on its candidate, or COMPLETE when none is (the shape +# _frontier_state emits). Test-only glue; the decision itself is pair_verdict's. +_pv_rings() { + local prev="" ring i=0 v any=0 + for ring in "$@"; do + if [ -n "$prev" ]; then + v="$(pair_verdict "${prev%:*}" "${prev##*:}" "${ring%:*}" "${ring##*:}")" + if [ "$v" != "ON_CANDIDATE" ]; then echo "$i $v"; any=1; fi + i=$((i + 1)) + fi + prev="$ring" + done + [ "$any" -eq 1 ] || echo "COMPLETE" +} +@test "pair_verdict rings: every ring on one commit → legacy COMPLETE" { + [ "$(_pv_rings aaa:0 aaa:0 aaa:0 aaa:0)" = "COMPLETE" ] +} +@test "pair_verdict rings: every tag ABSENT → legacy COMPLETE (#1118 AC6, unchanged)" { + [ "$(_pv_rings -:0 -:0 -:0 -:0)" = "COMPLETE" ] +} +@test "pair_verdict rings: TOTAL tag-lookup outage → every pair HOLD_UNKNOWN, never COMPLETE (#1225)" { + run _pv_rings -:1 -:1 -:1 -:1 + [ "$output" = $'0 HOLD_UNKNOWN\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] +} +@test "pair_verdict rings: one errored ring holds BOTH pairs touching it; the rest evaluate normally" { + run _pv_rings ccc:0 bbb:0 -:1 aaa:0 + [ "$output" = $'0 PENDING\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] +} +@test "pair_verdict rings: no tier skipping — each pair compares only adjacent rings" { + # stable (aaa) differs from next (ccc) but is on ring1's commit: only the lower pairs are pending. + run _pv_rings ccc:0 bbb:0 aaa:0 aaa:0 + [ "$output" = $'0 PENDING\n1 PENDING' ] +} +@test "pair_verdict rings: absent source over a populated ring → UNRESOLVABLE_SOURCE, not COMPLETE" { + run _pv_rings -:0 bbb:0 bbb:0 bbb:0 + [ "$output" = "0 UNRESOLVABLE_SOURCE" ] +} + # ── canary-rings.json SoT shape (rings + gate knobs) ────────────────────────── @test "canary-rings.json: pr-auto-review onboarded to canary; unmanaged block emptied" { run jq -e '.agents["pr-auto-review"].host == "petry-projects/.github"' "$RINGS" From 24c2733a185bb2ac5d075e4c0ac606f3b856eadd Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:54:18 +0000 Subject: [PATCH 2/4] fix(reviews): address review comments [skip ci-relay] --- scripts/canary-rollout.sh | 4 +++- tests/canary_rollout.bats | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 465e483ab..07c341755 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1914,7 +1914,9 @@ _frontier_state_resilient() { # closed); anything other than ON_CANDIDATE is held. if [ "$(pair_verdict "${cand:--}" 0 "${dstc:--}" 0)" != "ON_CANDIDATE" ]; then prior="$dstc" - differs="$(_reusable_differs "$agent" "$cand" "$prior")" + # An unresolvable (empty) src commit makes _reusable_differs return 0 by contract, which would + # classify as PRE_EXISTING and suppress regression triage — fail closed to "differs" instead. + if [ -z "$cand" ]; then differs=1; else differs="$(_reusable_differs "$agent" "$cand" "$prior")"; fi # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION # (fail closed — a changed reusable with UNREADABLE health is a suspected regression that # needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot be a diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 757ac3853..3cca160b6 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -888,19 +888,23 @@ _pv_rings() { } @test "pair_verdict rings: TOTAL tag-lookup outage → every pair HOLD_UNKNOWN, never COMPLETE (#1225)" { run _pv_rings -:1 -:1 -:1 -:1 + [ "$status" -eq 0 ] [ "$output" = $'0 HOLD_UNKNOWN\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] } @test "pair_verdict rings: one errored ring holds BOTH pairs touching it; the rest evaluate normally" { run _pv_rings ccc:0 bbb:0 -:1 aaa:0 + [ "$status" -eq 0 ] [ "$output" = $'0 PENDING\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] } @test "pair_verdict rings: no tier skipping — each pair compares only adjacent rings" { # stable (aaa) differs from next (ccc) but is on ring1's commit: only the lower pairs are pending. run _pv_rings ccc:0 bbb:0 aaa:0 aaa:0 + [ "$status" -eq 0 ] [ "$output" = $'0 PENDING\n1 PENDING' ] } @test "pair_verdict rings: absent source over a populated ring → UNRESOLVABLE_SOURCE, not COMPLETE" { run _pv_rings -:0 bbb:0 bbb:0 bbb:0 + [ "$status" -eq 0 ] [ "$output" = "0 UNRESOLVABLE_SOURCE" ] } From 592e08113e385d0e4ac06d78a8a7fdececc6be55 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:00:51 +0000 Subject: [PATCH 3/4] fix(reviews): address review comments [skip ci-relay] --- scripts/canary-rollout.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 07c341755..e2e00c471 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1914,14 +1914,19 @@ _frontier_state_resilient() { # closed); anything other than ON_CANDIDATE is held. if [ "$(pair_verdict "${cand:--}" 0 "${dstc:--}" 0)" != "ON_CANDIDATE" ]; then prior="$dstc" - # An unresolvable (empty) src commit makes _reusable_differs return 0 by contract, which would - # classify as PRE_EXISTING and suppress regression triage — fail closed to "differs" instead. - if [ -z "$cand" ]; then differs=1; else differs="$(_reusable_differs "$agent" "$cand" "$prior")"; fi - # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION - # (fail closed — a changed reusable with UNREADABLE health is a suspected regression that - # needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot be a - # candidate regression). Either verdict still tracks the pair as BLOCKED. - triage="$(classify_failure "$differs" unknown 0)" + if [ -z "$cand" ]; then + # Unresolvable (empty) src commit: whether the candidate changed the reusable is + # indeterminate (a tag gap), so do not claim REGRESSION nor PRE_EXISTING — leave the triage + # blank ("-", as the HOLD_UNKNOWN row does). The pair is still tracked BLOCKED (fail closed). + triage="-" + else + differs="$(_reusable_differs "$agent" "$cand" "$prior")" + # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION + # (fail closed — a changed reusable with UNREADABLE health is a suspected regression that + # needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot be a + # candidate regression). Either verdict still tracks the pair as BLOCKED. + triage="$(classify_failure "$differs" unknown 0)" + fi echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 $triage - - 0 0 0 0 1" emitted=1 fi From f6b8090c14020b97b8f3b3ee4b4c5819d2f6f12c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 22:22:04 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(canary-rollout):=20keep=20the=20refacto?= =?UTF-8?q?r=20pure=20=E2=80=94=20restore=20PRE=5FEXISTING=20triage=20for?= =?UTF-8?q?=20an=20empty=20src=20commit;=20[[=20in=20pair=5Fverdict=20(#12?= =?UTF-8?q?42)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _frontier_state_resilient: drop the empty-src '-' triage branch so the data-gap path is byte-identical to main (classify_failure on _reusable_differs); pin it with a test. - pair_verdict uses [[ ]] (Sonar S7688). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MjXPsBTaL6WV9AhCH2ajuv --- scripts/canary-rollout.sh | 19 ++++++------------- scripts/lib/canary-rollout.sh | 6 +++--- tests/canary_rollout.bats | 12 ++++++++++++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 33334f5ee..781230707 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -2409,19 +2409,12 @@ _frontier_state_resilient() { if [ "$(pair_verdict "${cand:--}" "$prev_unknown" "${dstc:--}" "$ch_unknown")" != "ON_CANDIDATE" ]; then transition="${prev}->${ch}" prior="$dstc" - if [ -z "$cand" ]; then - # Unresolvable (empty) src commit: whether the candidate changed the reusable is - # indeterminate (a tag gap), so do not claim REGRESSION nor PRE_EXISTING — leave the triage - # blank ("-", as the HOLD_UNKNOWN row does). The pair is still tracked BLOCKED (fail closed). - triage="-" - else - differs="$(_reusable_differs "$agent" "$cand" "$prior")" - # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION - # (fail closed — a changed reusable with UNREADABLE health is a suspected regression that - # needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot be a - # candidate regression). Either verdict still tracks the pair as BLOCKED. - triage="$(classify_failure "$differs" unknown 0)" - fi + differs="$(_reusable_differs "$agent" "$cand" "$prior")" + # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION + # (fail closed — a changed reusable with UNREADABLE health is a suspected regression that + # needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot be a + # candidate regression). Either verdict still tracks the pair as BLOCKED. + triage="$(classify_failure "$differs" unknown 0)" echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 $triage - - 0 0 0 0 1" emitted=1 fi diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index 91d6403d9..e9ade0ed3 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -455,9 +455,9 @@ transition_key() { # skipped (#1118). pair_verdict() { local src="${1:--}" su="${2:-0}" dst="${3:--}" du="${4:-0}" - if [ "$su" != "0" ] || [ "$du" != "0" ]; then echo "HOLD_UNKNOWN"; return 0; fi - if [ "$src" = "$dst" ]; then echo "ON_CANDIDATE"; return 0; fi - if [ "$src" = "-" ]; then echo "UNRESOLVABLE_SOURCE"; return 0; fi + if [[ "$su" != "0" || "$du" != "0" ]]; then echo "HOLD_UNKNOWN"; return 0; fi + if [[ "$src" = "$dst" ]]; then echo "ON_CANDIDATE"; return 0; fi + if [[ "$src" = "-" ]]; then echo "UNRESOLVABLE_SOURCE"; return 0; fi echo "PENDING" } diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 5262aa754..2e975d7ef 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -3371,6 +3371,18 @@ _sync_gap_stub() { [[ "$output" == "ccccccc ring0 next->ring0 BLOCKED 0 4 0 3 2 0 0 REGRESSION - - 0 0 0 0 0" ]] } +@test "orchestrator: _frontier_state_resilient keeps an unresolvable (empty) src commit tracked BLOCKED with triage PRE_EXISTING — unchanged by the pair_verdict refactor (#1242)" { + # A run-history fetch failure (data gap) makes the fallback walk the rings itself. ring0 has no tag but ring1 does: + # the pair is still pending (never vanishes), and with no resolvable candidate the reusable cannot + # differ, so classify_failure reads it as PRE_EXISTING — exactly as before the refactor. + run bash -c "source '$ORCH'; _frontier_state() { echo org/x >> \"\$_CANARY_FETCH_FAIL_FLAG\"; return 1; } + ordered_channels() { echo 'next,ring0,ring1'; } + _ring_commits() { printf 'next - 0\nring0 - 0\nring1 abc1234 0\n'; } + _frontier_state_resilient dev-lead 2>/dev/null" + [ "$status" -eq 0 ] + [[ "$output" == *" ring1 ring0->ring1 BLOCKED "*" PRE_EXISTING "* ]] +} + @test "orchestrator: _frontier_state_resilient fails CLOSED (non-zero) on a TOTAL inability to determine state (#820)" { # _frontier_state aborts to empty AND the candidate/frontier cannot be re-resolved from tags: # a total inability must stay a HARD ERROR (non-zero), never a green no-op.