Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions scripts/canary-rollout.sh
Original file line number Diff line number Diff line change
Expand Up @@ -983,8 +983,15 @@ _run_jobs_json() {
# cuts the list at its own `since`, and the "list reached CANARY_INGRESS_RUN_LIMIT" fail-closed guard
# is unchanged (a bounded list that hits the cap is judged exactly like an unbounded one).
_ingress_horizon() {
local since="$1" tiers="${CANARY_INGRESS_HORIZON_TIERS-8 15}" s now days t
_INGRESS_FROM=""
_horizon_from "$1" "${CANARY_INGRESS_HORIZON_TIERS-8 15}"
_INGRESS_FROM="$_HORIZON_FROM"
}

# _horizon_from <since_z> <tiers> — the tier logic behind _ingress_horizon, sets _HORIZON_FROM (also used for the
# per-role run lists, _agent_run_json, #1259 option B).
_horizon_from() {
local since="$1" tiers="$2" s now days t
_HORIZON_FROM=""
[ -n "$since" ] || return 0
s="$(_epoch "$since")"
case "$s" in ''|*[!0-9]*|0) return 0 ;; esac
Expand All @@ -996,7 +1003,7 @@ _ingress_horizon() {
for t in $tiers; do
case "$t" in ''|*[!0-9]*) continue ;; esac
if [ "$days" -le "$t" ]; then
_INGRESS_FROM="$(date -u -d "-${t} days" +%Y-%m-%d 2>/dev/null || date -u -v"-${t}d" +%Y-%m-%d 2>/dev/null || true)"
_HORIZON_FROM="$(date -u -d "-${t} days" +%Y-%m-%d 2>/dev/null || date -u -v"-${t}d" +%Y-%m-%d 2>/dev/null || true)"
return 0
fi
done
Expand Down Expand Up @@ -1195,7 +1202,14 @@ _agent_run_json() {
[ -n "${_TIMING_CALLS:-}" ] && { printf . >> "$_TIMING_CALLS" 2>/dev/null || true; } # call counter (CANARY_TIMING)
if [ -z "$repo" ] || [ "$repo" = '*' ]; then echo '[]'; return 0; fi
_reg_run_workflow "$agent"; wf="$_REG_VAL"
raw="$(_repo_wf_runs_cached "$repo" "$wf" 1)" || rc=$?
# Date-bound the per-role list (#1259, option B): every window the gate samples starts at most ~2 weeks back, so the
# newest 1000 runs of a busy workflow are mostly older than anything read. One fixed tier (CANARY_RUNS_HORIZON_TIERS,
# days, default "15"; EMPTY disables the bound) so every window shares ONE fetch per (repo, workflow). No fallback
# read is needed when the bounded list hits the 1000 cap: GitHub returns newest-first, so it is then exactly the
# newest-1000 the unbounded call returns (the existing truncation behavior, unchanged). A window older than the tier
# (a long soak) gets the unbounded list as before.
_horizon_from "$since" "${CANARY_RUNS_HORIZON_TIERS-15}"
raw="$(_repo_wf_runs_cached "$repo" "$wf" 1 1000 "$_HORIZON_FROM")" || rc=$?
if [ "$rc" -eq 3 ]; then
rc=0; _reg_ingress_wf; iwf="$_REG_VAL"
iraw="$(_ingress_runs_fetch "$repo" "$iwf" "$since")" || rc=$?
Expand Down
104 changes: 102 additions & 2 deletions tests/canary_rollout.bats
Original file line number Diff line number Diff line change
Expand Up @@ -6523,10 +6523,11 @@ GHEOF
[ "$status" -eq 0 ]
local from; from="$(date -u -d '-8 days' +%Y-%m-%d)"
grep -q -- "run list --repo org/busy --workflow Agent Ingress -L 5000 --created >=$from " "$GH_LOG"
# the per-role workflow lookup is NOT date-bounded
# the per-role workflow lookup is date-bounded too, to its own single 15-day tier (#1259 option B; it was unbounded
# before — this assertion is flipped deliberately)
run grep -- "--workflow Dev-Lead Agent" "$GH_LOG"
[ "$status" -eq 0 ]
[[ "$output" != *"--created"* ]]
[[ "$output" == *"--created >=$(date -u -d '-15 days' +%Y-%m-%d)"* ]]
: > "$GH_LOG"
run env CANARY_RINGS="$INGRESS_RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e
Expand Down Expand Up @@ -6969,3 +6970,102 @@ YML
[ "$status" -eq 0 ]
[ -z "$output" ]
}

# ── #1259 option B: the per-role (legacy) run list is date-bounded to one 15-day tier ─────────────────────
# The gate only samples windows that start at most ~2 weeks back, so the newest 1000 runs of a busy workflow were
# mostly dead weight (promote-all spent ~750 s of 925 s on `run list -L 1000`). The stub honors `--created` the
# way GitHub does (newest-first, createdAt date >= bound), so bounded and unbounded reads can be compared.
_legacy_runs_stub() {
export TMPDIR="$BATS_TEST_TMPDIR"
STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH"
export GH_LOG="$BATS_TEST_TMPDIR/gh-legacy.log"; : > "$GH_LOG"
export FIXTURE="$BATS_TEST_TMPDIR/legacy-runs.json"
local d2 d9 d14 d20
d2="$(date -u -d '-2 days' +%Y-%m-%dT10:00:00Z)"; d9="$(date -u -d '-9 days' +%Y-%m-%dT10:00:00Z)"
d14="$(date -u -d '-14 days' +%Y-%m-%dT10:00:00Z)"; d20="$(date -u -d '-20 days' +%Y-%m-%dT10:00:00Z)"
jq -n --arg a "$d2" --arg b "$d9" --arg c "$d14" --arg d "$d20" '[
{conclusion:"success",createdAt:$a,databaseId:1,workflowName:"Dev-Lead Agent"},
{conclusion:"failure",createdAt:$b,databaseId:2,workflowName:"Dev-Lead Agent"},
{conclusion:"success",createdAt:$c,databaseId:3,workflowName:"Dev-Lead Agent"},
{conclusion:"failure",createdAt:$d,databaseId:4,workflowName:"Dev-Lead Agent"}]' > "$FIXTURE"
cat > "$STUB_BIN/gh" <<'GHEOF'
#!/usr/bin/env bash
echo "$*" >> "$GH_LOG"
created=""; prev=""
for a in "$@"; do [ "$prev" = "--created" ] && created="${a#>=}"; prev="$a"; done
case "$1 $2" in
"run list") jq -c --arg c "$created" '[.[] | select($c == "" or (.createdAt[0:10] >= $c))]' "$FIXTURE" ;;
*) echo '{}' ;;
esac
GHEOF
chmod +x "$STUB_BIN/gh"
}
_ago() { date -u -d "-$1 days" +%Y-%m-%dT%H:%M:%SZ; }

@test "_agent_run_json: the per-role run list is fetched date-bounded to the 15-day tier for a recent window (#1259 option B)" {
_legacy_runs_stub
run env CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago 5)' | jq -r '[.[].databaseId] | join(\",\")'"
[ "$status" -eq 0 ]
[ "$output" = "1" ]
grep -q -- "--created >=$(date -u -d '-15 days' +%Y-%m-%d)" "$GH_LOG"
grep -q -- "-L 1000" "$GH_LOG"
}

@test "_agent_run_json: a window older than the tier, or an empty window, reads the unbounded list as before (#1259 option B)" {
_legacy_runs_stub
run env CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago 21)' | jq -r '[.[].databaseId] | join(\",\")'"
[ "$status" -eq 0 ]
[ "$output" = "1,2,3,4" ]
run grep -q -- "--created" "$GH_LOG"
[ "$status" -eq 1 ]
: > "$GH_LOG"
run env CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '' | jq -r '[.[].databaseId] | join(\",\")'"
[ "$status" -eq 0 ]
[ "$output" = "1,2,3,4" ]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low

Assert [ "$status" -eq 0 ] immediately after executing a command with run before making assertions on its output. This ensures that command failures or missing binaries are caught explicitly instead of causing confusing pattern-matching failures.

  [ "$status" -eq 0 ]
  [ "$output" = "1,2,3,4" ]
References
  1. In BATS tests, assert '[ "$status" -eq 0 ]' immediately after executing a command with 'run' before making assertions on its output.

@don-petry don-petry Oct 10, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Retracted by the dev-lead harness (not-on-ref). This reply's claim could not be verified as a fix produced and pushed by this pass (its commit was not produced by this pass, or the push did not land), so it no longer counts as addressed. The thread stays open; the next dev-lead pass re-verifies against the pushed diff.

Original reply, kept for the record (unverified — it does not count as a fix):

Done in tests/canary_rollout.bats: the #1259 option B tests now use run grep -q -- "--created" "$GH_LOG" with [ "$status" -eq 1 ] instead of ! grep -q, and assert [ "$status" -eq 0 ] right after each run before checking output.

run grep -q -- "--created" "$GH_LOG"
[ "$status" -eq 1 ]
}

@test "_agent_run_json: CANARY_RUNS_HORIZON_TIERS='' disables the per-role bound (#1259 option B)" {
_legacy_runs_stub
run env CANARY_RUNS_HORIZON_TIERS='' CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago 5)' | jq -r '[.[].databaseId] | join(\",\")'"
[ "$status" -eq 0 ]
[ "$output" = "1" ]
run grep -q -- "--created" "$GH_LOG"
[ "$status" -eq 1 ]
}

@test "_agent_run_json: bounded and unbounded reads give the same runs for every window within the tier (#1259 option B)" {
_legacy_runs_stub
local days out_b out_u
for days in 1 3 5 10 13 14; do
out_b="$(env CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago $days)'")"
out_u="$(env CANARY_RUNS_HORIZON_TIERS='' CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago $days)'")"
[ -n "$out_b" ]
[ "$out_b" = "$out_u" ] || { echo "window=$days bounded=$out_b unbounded=$out_u"; return 1; }
done
}

@test "_agent_run_json: windows of different ages share ONE per-role fetch, and a capped bounded list triggers no refetch (#1259 option B)" {
_legacy_runs_stub
local cache="$BATS_TEST_TMPDIR/rc-legacy" recent; mkdir -p "$cache"
run env _RUNS_CACHE_DIR="$cache" CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e
for d in 2 7 11 14; do _agent_run_json dev-lead org/legacy \"\$(date -u -d \"-\$d days\" +%Y-%m-%dT%H:%M:%SZ)\" >/dev/null; done"
[ "$status" -eq 0 ]
[ "$(grep -c '^run list ' "$GH_LOG")" -eq 1 ]
# A bounded list that fills the 1000 cap IS the newest-1000 (newest-first), so it is used as is: no second read.
: > "$GH_LOG"; cache="$BATS_TEST_TMPDIR/rc-legacy-cap"; mkdir -p "$cache"
# Dated relative to now: a fixed date would fall behind the stub's 15-day bound and filter every row out.
recent="$(date -u -d '-1 days' +%Y-%m-%dT10:00:00Z)"
jq -n -c --arg c "$recent" '[range(1000) as $i | {conclusion:"success",createdAt:$c,databaseId:$i,workflowName:"Dev-Lead Agent"}]' > "$FIXTURE"
run env _RUNS_CACHE_DIR="$cache" CANARY_RINGS="$RINGS" CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=1 bash -c "
source '$ORCH'; set +e; _agent_run_json dev-lead org/legacy '$(_ago 5)' | jq 'length'"
[ "$status" -eq 0 ]
[ "$output" = "1000" ]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low

Assert [ "$status" -eq 0 ] immediately after executing a command with run before making assertions on its output. This ensures that command failures or missing binaries are caught explicitly instead of causing confusing pattern-matching failures.

  [ "$status" -eq 0 ]
  [ "$output" = "1000" ]
References
  1. In BATS tests, assert '[ "$status" -eq 0 ]' immediately after executing a command with 'run' before making assertions on its output.

@don-petry don-petry Oct 10, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Retracted by the dev-lead harness (not-on-ref). This reply's claim could not be verified as a fix produced and pushed by this pass (its commit was not produced by this pass, or the push did not land), so it no longer counts as addressed. The thread stays open; the next dev-lead pass re-verifies against the pushed diff.

Original reply, kept for the record (unverified — it does not count as a fix):

Done in tests/canary_rollout.bats: the #1259 option B tests now use run grep -q -- "--created" "$GH_LOG" with [ "$status" -eq 1 ] instead of ! grep -q, and assert [ "$status" -eq 0 ] right after each run before checking output.

[ "$(grep -c '^run list ' "$GH_LOG")" -eq 1 ]
}
Loading