diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index c9c1da6b6..781230707 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1725,28 +1725,30 @@ _frontier_state() { for i in "${!chan_array[@]}"; do ch="${chan_array[$i]}" if [ -n "$prev" ]; then + # The pair verdict is the pure core's (pair_verdict, #1242); this loop only acts on it. A ring + # whose tag lookup ERRORED (#1225) is unknown, not absent. cand="${commits[$((i-1))]}" dstc="${commits[$i]}" transition="${prev}->${ch}" - # A ring whose tag lookup ERRORED (#1225) is unknown, not absent: hold every pair touching it - # BLOCKED (fail closed) — comparing its empty commit would read an API outage as "on the - # candidate", and a total outage as COMPLETE. Never evaluated (or promoted) by the gate. - if [ "${unknown[$((i-1))]:-0}" = 1 ] || [ "${unknown[$i]:-0}" = 1 ]; then - echo "::warning::$agent $transition: a ring tag lookup errored — holding the pair BLOCKED (fail closed) until it resolves (#1225)." >&2 - if [ -n "${_CANARY_TAG_GAP_FLAG:-}" ]; then - printf '%s\n' "$transition" >> "$_CANARY_TAG_GAP_FLAG" 2>/dev/null || true - fi - echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 - - - 0 0 0 0" - emitted=1 - prev="$ch"; continue - fi - # Pending iff dst is not on src's commit. An UNRESOLVABLE src commit (empty) with a populated - # dst is still pending and _pair_state holds it BLOCKED (fail closed, as the single-frontier - # code did) — never silently skipped into a false COMPLETE. - if [ "$dstc" != "$cand" ]; then - _pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv" - emitted=1 - fi + case "$(pair_verdict "${cand:--}" "${unknown[$((i-1))]:-0}" "${dstc:--}" "${unknown[$i]:-0}")" in + ON_CANDIDATE) ;; + PENDING|UNRESOLVABLE_SOURCE) + # An UNRESOLVABLE src commit (empty) with a populated dst is still pending and _pair_state + # holds it BLOCKED (fail closed, as the single-frontier code did) — never silently skipped + # into a false COMPLETE. + _pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv" + emitted=1 ;; + *) + # HOLD_UNKNOWN (or any unrecognised verdict): hold the pair BLOCKED (fail closed) — comparing + # an errored ring's empty commit would read an API outage as "on the candidate", and a total + # outage as COMPLETE. Never evaluated (or promoted) by the gate (#1225). + echo "::warning::$agent $transition: a ring tag lookup errored — holding the pair BLOCKED (fail closed) until it resolves (#1225)." >&2 + if [ -n "${_CANARY_TAG_GAP_FLAG:-}" ]; then + printf '%s\n' "$transition" >> "$_CANARY_TAG_GAP_FLAG" 2>/dev/null || true + fi + echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 - - - 0 0 0 0" + emitted=1 ;; + esac fi prev="$ch" done @@ -2401,9 +2403,10 @@ _frontier_state_resilient() { prev="$ch"; prev_commit="$ch_commit"; prev_unknown="$ch_unknown" continue fi - # Same pending rule as _frontier_state: dst not on src's commit, including an unresolvable - # (empty) src commit, which stays tracked as BLOCKED rather than vanishing (fail closed). - if [ "$dstc" != "$cand" ]; then + # Same pure pair verdict as _frontier_state (#1242): a pending pair, including an unresolvable + # (empty) src commit, stays tracked as BLOCKED rather than vanishing (fail closed); anything + # other than ON_CANDIDATE is held. + if [ "$(pair_verdict "${cand:--}" "$prev_unknown" "${dstc:--}" "$ch_unknown")" != "ON_CANDIDATE" ]; then transition="${prev}->${ch}" prior="$dstc" differs="$(_reusable_differs "$agent" "$cand" "$prior")" diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index 74d993720..e9ade0ed3 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -437,6 +437,30 @@ transition_key() { echo "" } +# pair_verdict +# Pure per-pair verdict for ONE adjacent ring transition src->dst (#1242). The orchestrator only +# gathers the facts: each ring's resolved commit ("-" or empty = no tag, i.e. ABSENT) and whether +# its tag lookup ERRORED (unknown=1, #1225). An empty unknown flag means known (0); any other +# non-0 value fails closed as unknown. Echoes exactly one of, in precedence order: +# HOLD_UNKNOWN — either ring's lookup errored: its commit is not known, so the pair is held +# BLOCKED and never evaluated or promoted, even with --override (#1225). An +# API outage must never read as "dst is on the candidate", nor a total +# outage as COMPLETE. +# ON_CANDIDATE — dst is already on src's commit (including both ABSENT, the legacy +# behaviour, #1118 AC6): nothing pending. +# UNRESOLVABLE_SOURCE — src has no tag but dst does: still pending and held BLOCKED (fail closed), +# never silently skipped into a false COMPLETE (#1118). +# PENDING — dst is not on src's commit: evaluate the gate for this pair. A ring only +# ever advances to the commit on the ring directly below it, so no tier is +# skipped (#1118). +pair_verdict() { + local src="${1:--}" su="${2:-0}" dst="${3:--}" du="${4:-0}" + if [[ "$su" != "0" || "$du" != "0" ]]; then echo "HOLD_UNKNOWN"; return 0; fi + if [[ "$src" = "$dst" ]]; then echo "ON_CANDIDATE"; return 0; fi + if [[ "$src" = "-" ]]; then echo "UNRESOLVABLE_SOURCE"; return 0; fi + echo "PENDING" +} + # ── set-diff core (drift detection, #1082) ──────────────────────────────────── # The registry (.agents{}) is the MANUAL source of truth for what the canary pipeline # manages; drift detection diffs it against the *-reusable.yml actually present on each diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 7ad770057..2e975d7ef 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -829,6 +829,106 @@ GHEOF [ "$(transition_key stable 'next,ring0,ring1,stable')" = "ring1->stable" ] } +# ── pair_verdict (pure per-pair verdict, #1242: #1118 pending rule + #1225 tag-gap hold) ── +# Each ring is one of: RESOLVED (a commit; same or different from the other side), ABSENT +# ("-", unknown=0 — legacy "no tag"), or ERRORED (unknown=1 — the lookup failed, so the +# commit is not known). Full {resolved, absent, errored} × {src, dst} matrix, gh-free. +@test "pair_verdict: resolved src, dst on the same commit → ON_CANDIDATE (nothing pending)" { + [ "$(pair_verdict aaa 0 aaa 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: resolved src, dst on a different commit → PENDING" { + [ "$(pair_verdict aaa 0 bbb 0)" = "PENDING" ] +} +@test "pair_verdict: resolved src, absent dst → PENDING (dst never received a release)" { + [ "$(pair_verdict aaa 0 - 0)" = "PENDING" ] +} +@test "pair_verdict: resolved src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict aaa 0 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: absent src, resolved dst → UNRESOLVABLE_SOURCE (BLOCKED, never a false COMPLETE)" { + [ "$(pair_verdict - 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ] +} +@test "pair_verdict: absent src, absent dst → ON_CANDIDATE (legacy: nothing to promote)" { + [ "$(pair_verdict - 0 - 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: absent src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 0 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, resolved dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 1 bbb 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, absent dst → HOLD_UNKNOWN (an outage is never read as 'on the candidate')" { + [ "$(pair_verdict - 1 - 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: errored src, errored dst → HOLD_UNKNOWN" { + [ "$(pair_verdict - 1 - 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: precedence — an errored side beats a matching commit (HOLD_UNKNOWN over ON_CANDIDATE)" { + [ "$(pair_verdict aaa 1 aaa 0)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 0 aaa 1)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: precedence — an errored side beats an unresolvable source and a pending pair" { + [ "$(pair_verdict - 1 bbb 1)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 1 bbb 0)" = "HOLD_UNKNOWN" ] +} +@test "pair_verdict: an empty commit is the absent sentinel '-' (same verdicts as '-')" { + [ "$(pair_verdict '' 0 '' 0)" = "ON_CANDIDATE" ] + [ "$(pair_verdict '' 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ] + [ "$(pair_verdict aaa 0 '' 0)" = "PENDING" ] + [ "$(pair_verdict '' 0 - 0)" = "ON_CANDIDATE" ] +} +@test "pair_verdict: an omitted/empty unknown flag means known (0)" { + [ "$(pair_verdict aaa '' aaa '')" = "ON_CANDIDATE" ] + [ "$(pair_verdict aaa)" = "PENDING" ] +} +@test "pair_verdict: any unknown flag other than 0 fails CLOSED to HOLD_UNKNOWN" { + [ "$(pair_verdict aaa yes aaa 0)" = "HOLD_UNKNOWN" ] + [ "$(pair_verdict aaa 0 aaa 2)" = "HOLD_UNKNOWN" ] +} + +# _pv_rings ... — fold pair_verdict over an ordered ring list, echoing one +# " " per pair NOT on its candidate, or COMPLETE when none is (the shape +# _frontier_state emits). Test-only glue; the decision itself is pair_verdict's. +_pv_rings() { + local prev="" ring i=0 v any=0 + for ring in "$@"; do + if [ -n "$prev" ]; then + v="$(pair_verdict "${prev%:*}" "${prev##*:}" "${ring%:*}" "${ring##*:}")" + if [ "$v" != "ON_CANDIDATE" ]; then echo "$i $v"; any=1; fi + i=$((i + 1)) + fi + prev="$ring" + done + [ "$any" -eq 1 ] || echo "COMPLETE" +} +@test "pair_verdict rings: every ring on one commit → legacy COMPLETE" { + [ "$(_pv_rings aaa:0 aaa:0 aaa:0 aaa:0)" = "COMPLETE" ] +} +@test "pair_verdict rings: every tag ABSENT → legacy COMPLETE (#1118 AC6, unchanged)" { + [ "$(_pv_rings -:0 -:0 -:0 -:0)" = "COMPLETE" ] +} +@test "pair_verdict rings: TOTAL tag-lookup outage → every pair HOLD_UNKNOWN, never COMPLETE (#1225)" { + run _pv_rings -:1 -:1 -:1 -:1 + [ "$status" -eq 0 ] + [ "$output" = $'0 HOLD_UNKNOWN\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] +} +@test "pair_verdict rings: one errored ring holds BOTH pairs touching it; the rest evaluate normally" { + run _pv_rings ccc:0 bbb:0 -:1 aaa:0 + [ "$status" -eq 0 ] + [ "$output" = $'0 PENDING\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ] +} +@test "pair_verdict rings: no tier skipping — each pair compares only adjacent rings" { + # stable (aaa) differs from next (ccc) but is on ring1's commit: only the lower pairs are pending. + run _pv_rings ccc:0 bbb:0 aaa:0 aaa:0 + [ "$status" -eq 0 ] + [ "$output" = $'0 PENDING\n1 PENDING' ] +} +@test "pair_verdict rings: absent source over a populated ring → UNRESOLVABLE_SOURCE, not COMPLETE" { + run _pv_rings -:0 bbb:0 bbb:0 bbb:0 + [ "$status" -eq 0 ] + [ "$output" = "0 UNRESOLVABLE_SOURCE" ] +} + # ── canary-rings.json SoT shape (rings + gate knobs) ────────────────────────── @test "canary-rings.json: pr-auto-review onboarded to canary; unmanaged block emptied" { run jq -e '.agents["pr-auto-review"].host == "petry-projects/.github"' "$RINGS" @@ -3271,6 +3371,18 @@ _sync_gap_stub() { [[ "$output" == "ccccccc ring0 next->ring0 BLOCKED 0 4 0 3 2 0 0 REGRESSION - - 0 0 0 0 0" ]] } +@test "orchestrator: _frontier_state_resilient keeps an unresolvable (empty) src commit tracked BLOCKED with triage PRE_EXISTING — unchanged by the pair_verdict refactor (#1242)" { + # A run-history fetch failure (data gap) makes the fallback walk the rings itself. ring0 has no tag but ring1 does: + # the pair is still pending (never vanishes), and with no resolvable candidate the reusable cannot + # differ, so classify_failure reads it as PRE_EXISTING — exactly as before the refactor. + run bash -c "source '$ORCH'; _frontier_state() { echo org/x >> \"\$_CANARY_FETCH_FAIL_FLAG\"; return 1; } + ordered_channels() { echo 'next,ring0,ring1'; } + _ring_commits() { printf 'next - 0\nring0 - 0\nring1 abc1234 0\n'; } + _frontier_state_resilient dev-lead 2>/dev/null" + [ "$status" -eq 0 ] + [[ "$output" == *" ring1 ring0->ring1 BLOCKED "*" PRE_EXISTING "* ]] +} + @test "orchestrator: _frontier_state_resilient fails CLOSED (non-zero) on a TOTAL inability to determine state (#820)" { # _frontier_state aborts to empty AND the candidate/frontier cannot be re-resolved from tags: # a total inability must stay a HARD ERROR (non-zero), never a green no-op.