Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 25 additions & 22 deletions scripts/canary-rollout.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1725,28 +1725,30 @@ _frontier_state() {
for i in "${!chan_array[@]}"; do
ch="${chan_array[$i]}"
if [ -n "$prev" ]; then
# The pair verdict is the pure core's (pair_verdict, #1242); this loop only acts on it. A ring
# whose tag lookup ERRORED (#1225) is unknown, not absent.
cand="${commits[$((i-1))]}"
dstc="${commits[$i]}"
transition="${prev}->${ch}"
# A ring whose tag lookup ERRORED (#1225) is unknown, not absent: hold every pair touching it
# BLOCKED (fail closed) — comparing its empty commit would read an API outage as "on the
# candidate", and a total outage as COMPLETE. Never evaluated (or promoted) by the gate.
if [ "${unknown[$((i-1))]:-0}" = 1 ] || [ "${unknown[$i]:-0}" = 1 ]; then
echo "::warning::$agent $transition: a ring tag lookup errored — holding the pair BLOCKED (fail closed) until it resolves (#1225)." >&2
if [ -n "${_CANARY_TAG_GAP_FLAG:-}" ]; then
printf '%s\n' "$transition" >> "$_CANARY_TAG_GAP_FLAG" 2>/dev/null || true
fi
echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 - - - 0 0 0 0"
emitted=1
prev="$ch"; continue
fi
# Pending iff dst is not on src's commit. An UNRESOLVABLE src commit (empty) with a populated
# dst is still pending and _pair_state holds it BLOCKED (fail closed, as the single-frontier
# code did) — never silently skipped into a false COMPLETE.
if [ "$dstc" != "$cand" ]; then
_pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv"
emitted=1
fi
case "$(pair_verdict "${cand:--}" "${unknown[$((i-1))]:-0}" "${dstc:--}" "${unknown[$i]:-0}")" in
ON_CANDIDATE) ;;
PENDING|UNRESOLVABLE_SOURCE)
# An UNRESOLVABLE src commit (empty) with a populated dst is still pending and _pair_state
# holds it BLOCKED (fail closed, as the single-frontier code did) — never silently skipped
# into a false COMPLETE.
_pair_state "$agent" "$prev" "$ch" "$cand" "$transition" "$chans" "$commits_csv"
emitted=1 ;;
*)
# HOLD_UNKNOWN (or any unrecognised verdict): hold the pair BLOCKED (fail closed) — comparing
# an errored ring's empty commit would read an API outage as "on the candidate", and a total
# outage as COMPLETE. Never evaluated (or promoted) by the gate (#1225).
echo "::warning::$agent $transition: a ring tag lookup errored — holding the pair BLOCKED (fail closed) until it resolves (#1225)." >&2
if [ -n "${_CANARY_TAG_GAP_FLAG:-}" ]; then
printf '%s\n' "$transition" >> "$_CANARY_TAG_GAP_FLAG" 2>/dev/null || true
fi
echo "${cand:--} $ch $transition BLOCKED 0 0 0 0 0 0 0 - - - 0 0 0 0"
emitted=1 ;;
esac
fi
prev="$ch"
done
Expand Down Expand Up @@ -2401,9 +2403,10 @@ _frontier_state_resilient() {
prev="$ch"; prev_commit="$ch_commit"; prev_unknown="$ch_unknown"
continue
fi
# Same pending rule as _frontier_state: dst not on src's commit, including an unresolvable
# (empty) src commit, which stays tracked as BLOCKED rather than vanishing (fail closed).
if [ "$dstc" != "$cand" ]; then
# Same pure pair verdict as _frontier_state (#1242): a pending pair, including an unresolvable
# (empty) src commit, stays tracked as BLOCKED rather than vanishing (fail closed); anything
# other than ON_CANDIDATE is held.
if [ "$(pair_verdict "${cand:--}" "$prev_unknown" "${dstc:--}" "$ch_unknown")" != "ON_CANDIDATE" ]; then
transition="${prev}->${ch}"
prior="$dstc"
differs="$(_reusable_differs "$agent" "$cand" "$prior")"
Expand Down
24 changes: 24 additions & 0 deletions scripts/lib/canary-rollout.sh
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,30 @@ transition_key() {
echo ""
}

# pair_verdict <src_commit|-> <src_unknown> <dst_commit|-> <dst_unknown>
# Pure per-pair verdict for ONE adjacent ring transition src->dst (#1242). The orchestrator only
# gathers the facts: each ring's resolved commit ("-" or empty = no tag, i.e. ABSENT) and whether
# its tag lookup ERRORED (unknown=1, #1225). An empty unknown flag means known (0); any other
# non-0 value fails closed as unknown. Echoes exactly one of, in precedence order:
# HOLD_UNKNOWN — either ring's lookup errored: its commit is not known, so the pair is held
# BLOCKED and never evaluated or promoted, even with --override (#1225). An
# API outage must never read as "dst is on the candidate", nor a total
# outage as COMPLETE.
# ON_CANDIDATE — dst is already on src's commit (including both ABSENT, the legacy
# behaviour, #1118 AC6): nothing pending.
# UNRESOLVABLE_SOURCE — src has no tag but dst does: still pending and held BLOCKED (fail closed),
# never silently skipped into a false COMPLETE (#1118).
# PENDING — dst is not on src's commit: evaluate the gate for this pair. A ring only
# ever advances to the commit on the ring directly below it, so no tier is
# skipped (#1118).
pair_verdict() {
local src="${1:--}" su="${2:-0}" dst="${3:--}" du="${4:-0}"
if [[ "$su" != "0" || "$du" != "0" ]]; then echo "HOLD_UNKNOWN"; return 0; fi
if [[ "$src" = "$dst" ]]; then echo "ON_CANDIDATE"; return 0; fi
if [[ "$src" = "-" ]]; then echo "UNRESOLVABLE_SOURCE"; return 0; fi
echo "PENDING"
}

# ── set-diff core (drift detection, #1082) ────────────────────────────────────
# The registry (.agents{}) is the MANUAL source of truth for what the canary pipeline
# manages; drift detection diffs it against the *-reusable.yml actually present on each
Expand Down
112 changes: 112 additions & 0 deletions tests/canary_rollout.bats
Original file line number Diff line number Diff line change
Expand Up @@ -829,6 +829,106 @@ GHEOF
[ "$(transition_key stable 'next,ring0,ring1,stable')" = "ring1->stable" ]
}

# ── pair_verdict (pure per-pair verdict, #1242: #1118 pending rule + #1225 tag-gap hold) ──
# Each ring is one of: RESOLVED (a commit; same or different from the other side), ABSENT
# ("-", unknown=0 — legacy "no tag"), or ERRORED (unknown=1 — the lookup failed, so the
# commit is not known). Full {resolved, absent, errored} × {src, dst} matrix, gh-free.
@test "pair_verdict: resolved src, dst on the same commit → ON_CANDIDATE (nothing pending)" {
[ "$(pair_verdict aaa 0 aaa 0)" = "ON_CANDIDATE" ]
}
@test "pair_verdict: resolved src, dst on a different commit → PENDING" {
[ "$(pair_verdict aaa 0 bbb 0)" = "PENDING" ]
}
@test "pair_verdict: resolved src, absent dst → PENDING (dst never received a release)" {
[ "$(pair_verdict aaa 0 - 0)" = "PENDING" ]
}
@test "pair_verdict: resolved src, errored dst → HOLD_UNKNOWN" {
[ "$(pair_verdict aaa 0 - 1)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: absent src, resolved dst → UNRESOLVABLE_SOURCE (BLOCKED, never a false COMPLETE)" {
[ "$(pair_verdict - 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ]
}
@test "pair_verdict: absent src, absent dst → ON_CANDIDATE (legacy: nothing to promote)" {
[ "$(pair_verdict - 0 - 0)" = "ON_CANDIDATE" ]
}
@test "pair_verdict: absent src, errored dst → HOLD_UNKNOWN" {
[ "$(pair_verdict - 0 - 1)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: errored src, resolved dst → HOLD_UNKNOWN" {
[ "$(pair_verdict - 1 bbb 0)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: errored src, absent dst → HOLD_UNKNOWN (an outage is never read as 'on the candidate')" {
[ "$(pair_verdict - 1 - 0)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: errored src, errored dst → HOLD_UNKNOWN" {
[ "$(pair_verdict - 1 - 1)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: precedence — an errored side beats a matching commit (HOLD_UNKNOWN over ON_CANDIDATE)" {
[ "$(pair_verdict aaa 1 aaa 0)" = "HOLD_UNKNOWN" ]
[ "$(pair_verdict aaa 0 aaa 1)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: precedence — an errored side beats an unresolvable source and a pending pair" {
[ "$(pair_verdict - 1 bbb 1)" = "HOLD_UNKNOWN" ]
[ "$(pair_verdict aaa 1 bbb 0)" = "HOLD_UNKNOWN" ]
}
@test "pair_verdict: an empty commit is the absent sentinel '-' (same verdicts as '-')" {
[ "$(pair_verdict '' 0 '' 0)" = "ON_CANDIDATE" ]
[ "$(pair_verdict '' 0 bbb 0)" = "UNRESOLVABLE_SOURCE" ]
[ "$(pair_verdict aaa 0 '' 0)" = "PENDING" ]
[ "$(pair_verdict '' 0 - 0)" = "ON_CANDIDATE" ]
}
@test "pair_verdict: an omitted/empty unknown flag means known (0)" {
[ "$(pair_verdict aaa '' aaa '')" = "ON_CANDIDATE" ]
[ "$(pair_verdict aaa)" = "PENDING" ]
}
@test "pair_verdict: any unknown flag other than 0 fails CLOSED to HOLD_UNKNOWN" {
[ "$(pair_verdict aaa yes aaa 0)" = "HOLD_UNKNOWN" ]
[ "$(pair_verdict aaa 0 aaa 2)" = "HOLD_UNKNOWN" ]
}

# _pv_rings <commit:unknown>... — fold pair_verdict over an ordered ring list, echoing one
# "<i> <verdict>" per pair NOT on its candidate, or COMPLETE when none is (the shape
# _frontier_state emits). Test-only glue; the decision itself is pair_verdict's.
_pv_rings() {
local prev="" ring i=0 v any=0
for ring in "$@"; do
if [ -n "$prev" ]; then
v="$(pair_verdict "${prev%:*}" "${prev##*:}" "${ring%:*}" "${ring##*:}")"
if [ "$v" != "ON_CANDIDATE" ]; then echo "$i $v"; any=1; fi
i=$((i + 1))
fi
prev="$ring"
done
[ "$any" -eq 1 ] || echo "COMPLETE"
}
@test "pair_verdict rings: every ring on one commit → legacy COMPLETE" {
[ "$(_pv_rings aaa:0 aaa:0 aaa:0 aaa:0)" = "COMPLETE" ]
}
@test "pair_verdict rings: every tag ABSENT → legacy COMPLETE (#1118 AC6, unchanged)" {
[ "$(_pv_rings -:0 -:0 -:0 -:0)" = "COMPLETE" ]
}
@test "pair_verdict rings: TOTAL tag-lookup outage → every pair HOLD_UNKNOWN, never COMPLETE (#1225)" {
run _pv_rings -:1 -:1 -:1 -:1
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
[ "$status" -eq 0 ]
[ "$output" = $'0 HOLD_UNKNOWN\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ]
Comment thread
don-petry marked this conversation as resolved.
Resolved
}
@test "pair_verdict rings: one errored ring holds BOTH pairs touching it; the rest evaluate normally" {
run _pv_rings ccc:0 bbb:0 -:1 aaa:0
[ "$status" -eq 0 ]
[ "$output" = $'0 PENDING\n1 HOLD_UNKNOWN\n2 HOLD_UNKNOWN' ]
Comment thread
don-petry marked this conversation as resolved.
Resolved
}
@test "pair_verdict rings: no tier skipping — each pair compares only adjacent rings" {
# stable (aaa) differs from next (ccc) but is on ring1's commit: only the lower pairs are pending.
run _pv_rings ccc:0 bbb:0 aaa:0 aaa:0
[ "$status" -eq 0 ]
[ "$output" = $'0 PENDING\n1 PENDING' ]
Comment thread
don-petry marked this conversation as resolved.
Resolved
}
@test "pair_verdict rings: absent source over a populated ring → UNRESOLVABLE_SOURCE, not COMPLETE" {
run _pv_rings -:0 bbb:0 bbb:0 bbb:0
[ "$status" -eq 0 ]
[ "$output" = "0 UNRESOLVABLE_SOURCE" ]
Comment thread
don-petry marked this conversation as resolved.
Resolved
}

# ── canary-rings.json SoT shape (rings + gate knobs) ──────────────────────────
@test "canary-rings.json: pr-auto-review onboarded to canary; unmanaged block emptied" {
run jq -e '.agents["pr-auto-review"].host == "petry-projects/.github"' "$RINGS"
Expand Down Expand Up @@ -3271,6 +3371,18 @@ _sync_gap_stub() {
[[ "$output" == "ccccccc ring0 next->ring0 BLOCKED 0 4 0 3 2 0 0 REGRESSION - - 0 0 0 0 0" ]]
}

@test "orchestrator: _frontier_state_resilient keeps an unresolvable (empty) src commit tracked BLOCKED with triage PRE_EXISTING — unchanged by the pair_verdict refactor (#1242)" {
# A run-history fetch failure (data gap) makes the fallback walk the rings itself. ring0 has no tag but ring1 does:
# the pair is still pending (never vanishes), and with no resolvable candidate the reusable cannot
# differ, so classify_failure reads it as PRE_EXISTING — exactly as before the refactor.
run bash -c "source '$ORCH'; _frontier_state() { echo org/x >> \"\$_CANARY_FETCH_FAIL_FLAG\"; return 1; }
ordered_channels() { echo 'next,ring0,ring1'; }
_ring_commits() { printf 'next - 0\nring0 - 0\nring1 abc1234 0\n'; }
_frontier_state_resilient dev-lead 2>/dev/null"
[ "$status" -eq 0 ]
[[ "$output" == *" ring1 ring0->ring1 BLOCKED "*" PRE_EXISTING "* ]]
}

@test "orchestrator: _frontier_state_resilient fails CLOSED (non-zero) on a TOTAL inability to determine state (#820)" {
# _frontier_state aborts to empty AND the candidate/frontier cannot be re-resolved from tags:
# a total inability must stay a HARD ERROR (non-zero), never a green no-op.
Expand Down
Loading