diff --git a/scripts/evals/notify-eval-health.sh b/scripts/evals/notify-eval-health.sh
index 0ce2f5d1b..bee0c3e70 100644
--- a/scripts/evals/notify-eval-health.sh
+++ b/scripts/evals/notify-eval-health.sh
@@ -9,14 +9,22 @@ set -euo pipefail
# step summary / artifact / annotation — effectively invisible. This notifier
# makes the outcome visible WITHOUT making it blocking:
#
-# outcome=regression | error -> open OR update a single `eval-health` tracking
-# issue (idempotent: the same issue is rewritten
-# in place, never a new one per day).
-# outcome=pass -> if a tracking issue is currently open, comment
-# + close it (recovery); otherwise do NOTHING
-# (a healthy run must create zero noise).
+# outcome=regression -> open OR update the single `eval-health` REGRESSION
+# tracker (idempotent: rewritten in place, never a new
+# one per day).
+# outcome=error -> open OR update a SEPARATE `eval-infra` INFRA-ERROR
+# tracker. An infra error (e.g. every model throttled,
+# #920, or bad usage / missing tooling) means the skill
+# was never scored — it must NOT open or update the
+# held-out regression tracker, or a transient outage
+# reads as a skill regression. It stays visible via its
+# own clearly-labeled issue.
+# outcome=pass -> a green run clears either condition: comment + close
+# whichever tracker(s) (regression and/or infra) are
+# currently open. With none open, do NOTHING (a healthy
+# run must create zero noise).
#
-# De-dup key: the single open issue carrying $EVAL_HEALTH_LABEL whose title
+# De-dup key: the single open issue carrying the matching label whose title
# matches the stable per-skill title below. Stable title => one issue, rewritten.
#
# This script only adds visibility; it never changes the scorer's pass/fail
@@ -28,7 +36,8 @@ set -euo pipefail
# SKILL skill whose holdout was scored (default: triage)
# REPORT_PATH eval-report.json from the scorer (default: ./eval-report.json)
# RUN_URL link back to the workflow run (optional but recommended)
-# EVAL_HEALTH_LABEL tracking-issue label (default: eval-health)
+# EVAL_HEALTH_LABEL regression tracking-issue label (default: eval-health)
+# EVAL_INFRA_LABEL infra-error tracking-issue label (default: eval-infra)
# DRY_RUN "1"/"true" => render + log intent, never mutate via gh
#
# Exit codes: 0 success (including the no-op healthy path); 1 bad usage / gh error.
@@ -39,40 +48,56 @@ SKILL="${SKILL:-triage}"
REPORT_PATH="${REPORT_PATH:-eval-report.json}"
RUN_URL="${RUN_URL:-}"
EVAL_HEALTH_LABEL="${EVAL_HEALTH_LABEL:-eval-health}"
+EVAL_INFRA_LABEL="${EVAL_INFRA_LABEL:-eval-infra}"
+
+command -v jq >/dev/null 2>&1 || { echo "::error::notify-eval-health: jq is required but not installed" >&2; exit 1; }
+command -v gh >/dev/null 2>&1 || { echo "::error::notify-eval-health: gh is required but not installed" >&2; exit 1; }
case "$OUTCOME" in
pass|regression|error) ;;
*) echo "::error::notify-eval-health: unknown OUTCOME '$OUTCOME' (expected pass|regression|error)" >&2; exit 1 ;;
esac
-# Stable, per-skill title so the lookup always lands on the same issue.
-TITLE="Skill eval health — \`${SKILL}\` (holdout)"
+# Stable, per-skill titles so each lookup always lands on the same issue. The
+# regression tracker and the infra-error tracker are SEPARATE issues with
+# distinct labels (#920): a throttle/outage must not touch the regression
+# tracker, so each is de-duped independently.
+TITLE_REGRESSION="Skill eval health — \`${SKILL}\` (holdout)"
+TITLE_INFRA="Skill eval infra error — \`${SKILL}\` (holdout)"
_is_dry_run() { [[ "${DRY_RUN:-0}" == "1" || "${DRY_RUN:-0}" == "true" ]]; }
-# Locate the single open tracking issue (scoped by label) whose title matches.
-# gh emits the candidate list as JSON; system jq does the title de-dup so the
-# expression is unambiguous (no reliance on gh's embedded jq arg passing).
+# find_existing
— locate the single open tracking issue (scoped
+# by label) whose title matches. gh emits the candidate list as JSON; system jq
+# does the title de-dup so the expression is unambiguous (no reliance on gh's
+# embedded jq arg passing).
find_existing() {
+ if [ "$#" -lt 2 ]; then
+ echo "::error::find_existing requires title and label" >&2
+ exit 1
+ fi
+ local title="$1" label="$2"
if _is_dry_run; then
printf '%s' "${DRY_RUN_EXISTING:-}"
return 0
fi
- gh issue list --repo "$REPO" --label "$EVAL_HEALTH_LABEL" --state open \
+ gh issue list --repo "$REPO" --label "$label" --state open \
--json number,title 2>/dev/null \
- | jq -r --arg t "$TITLE" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \
+ | jq -r --arg t "$title" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \
|| true
}
# Render the tracking-issue body for a regression/error outcome to stdout.
render_body() {
- echo "## Skill eval health — \`${SKILL}\` (holdout)"
- echo ""
if [ "$OUTCOME" = "error" ]; then
- echo "**Outcome:** \`error\` — the scorer hard-errored (bad usage / missing tooling)."
+ echo "## ${TITLE_INFRA}"
+ echo ""
+ echo "**Outcome:** \`error\` — the eval could not score the skill (infrastructure failure: e.g. every model in the fallback chain throttled, or bad usage / missing tooling)."
echo ""
- echo "_The eval run could not produce a score. This is report-only (Phase 1): it does not block any merge — but it does mean the held-out set is currently un-scored and needs a look._"
+ echo "_This is an **infra** signal, **not** a skill regression: the model never produced a scorable answer, so the held-out set is currently un-scored and needs a look. Report-only (Phase 1): it does not block any merge._"
else
+ echo "## ${TITLE_REGRESSION}"
+ echo ""
echo "**Outcome:** \`regression\` — at least one held-out case regressed."
echo ""
echo "_Report-only (Phase 1): a regression does not fail CI or block any merge._"
@@ -105,48 +130,80 @@ render_body() {
echo "Auto-managed by \`notify-eval-health.sh\` — this issue is updated in place each run and closed automatically on recovery. "
}
-existing="$(find_existing)"
+# _recover — comment a recovery note on the issue and close it.
+_recover() {
+ if [ "$#" -lt 1 ]; then
+ echo "::error::_recover requires an issue number" >&2
+ exit 1
+ fi
+ local num="$1" comment
+ comment="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; }
+ {
+ echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again."
+ echo ""
+ [ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})"
+ } >"$comment"
+ if _is_dry_run; then
+ echo "[dry-run] would comment on and close issue #${num}"
+ rm -f "$comment"
+ return 0
+ fi
+ gh issue comment "$num" --repo "$REPO" --body-file "$comment" >/dev/null
+ gh issue close "$num" --repo "$REPO" >/dev/null
+ rm -f "$comment"
+ echo "closed tracking issue #${num} (recovered)"
+}
+
+# _open_or_update — open the
+# tracker (de-duped by title+label) or rewrite it in place if already open.
+_open_or_update() {
+ if [ "$#" -lt 4 ]; then
+ echo "::error::_open_or_update requires title, label, description, and color" >&2
+ exit 1
+ fi
+ local title="$1" label="$2" desc="$3" color="$4" existing body url
+ existing="$(find_existing "$title" "$label")"
+ body="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; }
+ render_body >"$body"
+ if _is_dry_run; then
+ echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) issue '${title}' (${label})"
+ rm -f "$body"
+ return 0
+ fi
+ if [ -n "$existing" ]; then
+ gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null
+ echo "updated tracking issue #${existing} (${label})"
+ else
+ # Ensure the label exists (ignore failure if it already does).
+ gh label create "$label" --repo "$REPO" --description "$desc" --color "$color" 2>/dev/null || true
+ url="$(gh issue create --repo "$REPO" --title "$title" --label "$label" --body-file "$body")"
+ echo "created tracking issue ${url} (${label})"
+ fi
+ rm -f "$body"
+}
case "$OUTCOME" in
pass)
- if [ -z "$existing" ]; then
+ # A green run clears either condition — close whichever tracker(s) are open.
+ reg="$(find_existing "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL")"
+ inf="$(find_existing "$TITLE_INFRA" "$EVAL_INFRA_LABEL")"
+ if [ -z "$reg" ] && [ -z "$inf" ]; then
echo "eval health pass: no open tracking issue — nothing to do."
exit 0
fi
- comment="$(mktemp)"
- trap 'rm -f "$comment"' EXIT
- {
- echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again."
- echo ""
- [ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})"
- } >"$comment"
- if _is_dry_run; then
- echo "[dry-run] would comment on and close eval-health issue #${existing}"
- exit 0
- fi
- gh issue comment "$existing" --repo "$REPO" --body-file "$comment" >/dev/null
- gh issue close "$existing" --repo "$REPO" >/dev/null
- echo "closed eval-health issue #${existing} (recovered)"
+ if [ -n "$reg" ]; then _recover "$reg"; fi
+ if [ -n "$inf" ]; then _recover "$inf"; fi
;;
- regression|error)
- body="$(mktemp)"
- trap 'rm -f "$body"' EXIT
- render_body >"$body"
- if _is_dry_run; then
- echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) eval-health issue '${TITLE}'"
- exit 0
- fi
- if [ -n "$existing" ]; then
- gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null
- echo "updated eval-health issue #${existing}"
- else
- # Ensure the label exists (ignore failure if it already does).
- gh label create "$EVAL_HEALTH_LABEL" --repo "$REPO" \
- --description "Skill-eval holdout health tracking" --color d93f0b 2>/dev/null || true
- url="$(gh issue create --repo "$REPO" --title "$TITLE" \
- --label "$EVAL_HEALTH_LABEL" --body-file "$body")"
- echo "created eval-health issue ${url}"
- fi
+ regression)
+ _open_or_update "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL" \
+ "Skill-eval holdout health tracking" "d93f0b"
+ ;;
+
+ error)
+ # Distinct infra tracker (#920): an infra error never touches the regression
+ # tracker, so a transient throttle cannot read as a held-out regression.
+ _open_or_update "$TITLE_INFRA" "$EVAL_INFRA_LABEL" \
+ "Skill-eval holdout infra-error tracking" "fbca04"
;;
esac
diff --git a/scripts/evals/run-eval.sh b/scripts/evals/run-eval.sh
index 28681c3c0..cf277a102 100755
--- a/scripts/evals/run-eval.sh
+++ b/scripts/evals/run-eval.sh
@@ -51,8 +51,23 @@ set -euo pipefail
#
# Exit codes:
# 0 run completed, every case passed
-# 1 run completed, at least one case failed
-# 2 hard error (bad usage / missing prompt or cases file / missing tooling)
+# 1 run completed, at least one case failed on QUALITY (a genuine regression)
+# 2 hard error — bad usage / missing prompt or cases file / missing tooling,
+# OR an all-infra run where the skill was never actually scored (see below)
+#
+# Infra failure vs. quality regression (#920): each case records the engine's
+# exit status (engine_rc). A failing case whose engine call exited NON-ZERO is an
+# INFRA failure (e.g. every model in engine.sh's fallback chain throttled) — the
+# model never answered, so there is no answer to call right or wrong. A failing
+# case whose engine call exited ZERO is a QUALITY miss (the model answered, but
+# the output was wrong or unparseable). The aggregate exit code, when >=1 case
+# fails, is:
+# - every failing case is an infra failure (no quality miss anywhere) -> 2
+# (outcome=error). The held-out set is currently UN-scored; a transient
+# throttle must NOT open a false held-out regression tracker.
+# - otherwise (>=1 quality miss) -> 1 (regression), unchanged.
+# A MIXED run with some cases answered (pass) and the rest throttled therefore
+# exits 2: there is no parseable-but-wrong answer to call a regression.
#
# Cases are read-only here — the scorer never writes evals//holdout/cases.jsonl.
@@ -128,12 +143,17 @@ trap 'rm -f "$results" "$work_prompt" "$work_judge"' EXIT
# lines, exit 1) from tripping `set -e`.
extract_json() { grep -v $'^[[:space:]]*\x60\x60\x60' <<<"$1" || true; }
-# score_deterministic
+# score_deterministic
# Equality scorer (triage). Parses the model output; a non-JSON-object output
# yields a null actual decision (an automatic failure). Every result carries a
-# numeric `score` (1 pass / 0 fail) so the report shape matches the judge mode.
+# numeric `score` (1 pass / 0 fail) plus the engine exit status `engine_rc` (so
+# an all-infra run can be told apart from a quality regression, #920) so the
+# report shape matches the judge mode.
score_deterministic() {
- local expected="$1" raw="$2" cid="$3"
+ if [ "$#" -lt 3 ]; then
+ die "score_deterministic requires expected, raw, and cid arguments"
+ fi
+ local expected="$1" raw="$2" cid="$3" eng_rc="${4:-0}"
local exp_esc exp_risk actual pass score cleaned raw_excerpt=""
exp_esc="$(jq -r '.escalate' <<<"$expected")"
exp_risk="$(jq -r '.risk' <<<"$expected")"
@@ -169,9 +189,10 @@ score_deterministic() {
# raw_excerpt is attached ONLY when the output failed to parse (actual=null), so
# passing/mismatch cases keep the lean shape the report and tests expect.
jq -cn --arg id "$cid" --argjson pass "$pass" --argjson score "$score" \
+ --argjson engine_rc "$eng_rc" \
--argjson exp_esc "$exp_esc" --arg exp_risk "$exp_risk" \
--argjson actual "$actual" --arg raw_excerpt "$raw_excerpt" \
- '{id:$id, score:$score, pass:$pass, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual}
+ '{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual}
+ (if $raw_excerpt == "" then {} else {raw_excerpt: $raw_excerpt} end)'
}
@@ -182,7 +203,10 @@ score_deterministic() {
# judge engine (default run_triage), and parses its numeric `score`. Judge output
# that is not a JSON object with a numeric `score` scores 0 (an automatic failure).
score_llm_judge() {
- local expected="$1" candidate="$2" cid="$3"
+ if [ "$#" -lt 3 ]; then
+ die "score_llm_judge requires expected, candidate, and cid arguments"
+ fi
+ local expected="$1" candidate="$2" cid="$3" eng_rc="${4:-0}"
local judge_raw judge_obj score pass
{
@@ -216,9 +240,10 @@ score_llm_judge() {
fi
jq -cn --arg id "$cid" --argjson score "$score" --argjson pass "$pass" \
+ --argjson engine_rc "$eng_rc" \
--argjson expected "$expected" --argjson judge "$judge_obj" \
--arg candidate "$candidate" \
- '{id:$id, score:$score, pass:$pass, expected:$expected, judge:$judge, candidate:$candidate}'
+ '{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:$expected, judge:$judge, candidate:$candidate}'
}
while IFS= read -r line; do
@@ -236,15 +261,18 @@ while IFS= read -r line; do
} >"$work_prompt"
# Invoke the skill through the engine command (default run_triage). Capture
- # stdout (the model decision); engine diagnostics flow to our stderr. A
- # non-zero engine exit leaves $raw empty -> scored as a failure.
- raw=""
- raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || raw=""
+ # stdout (the model decision); engine diagnostics flow to our stderr. Preserve
+ # the engine's exit status (eng_rc): a NON-ZERO exit means the call failed for
+ # infra reasons (e.g. every model in the fallback chain throttled) — the only
+ # signal that distinguishes a throttle from a model that answered wrong (#920).
+ raw=""; eng_rc=0
+ raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || eng_rc=$?
- # Dispatch on the per-skill scorer mode — never on the skill name.
+ # Dispatch on the per-skill scorer mode — never on the skill name. eng_rc is
+ # threaded through so each case result records whether the engine actually ran.
case "$SCORER_MODE" in
- deterministic) score_deterministic "$expected" "$raw" "$cid" >>"$results" ;;
- llm-judge) score_llm_judge "$expected" "$raw" "$cid" >>"$results" ;;
+ deterministic) score_deterministic "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;;
+ llm-judge) score_llm_judge "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;;
esac
done <"$cases_file"
@@ -259,5 +287,17 @@ jq -s --arg skill "$skill" '
cases: .
}' "$results"
+# Decide the exit code. Classify failing cases as infra (engine exited non-zero —
+# throttle/outage, the model never answered) vs. quality (engine exited zero but
+# the answer was wrong/unparseable). If EVERY failing case is infra, the skill
+# was never actually scored: exit 2 -> outcome=error, so a transient throttle
+# cannot open a false held-out regression (#920). Any quality miss keeps exit 1.
failed="$(jq -s 'map(select(.pass | not)) | length' "$results")"
-[ "$failed" -eq 0 ]
+if [ "$failed" -eq 0 ]; then
+ exit 0
+fi
+quality_failed="$(jq -s 'map(select((.pass | not) and ((.engine_rc // 0) == 0))) | length' "$results")"
+if [ "$quality_failed" -eq 0 ]; then
+ exit 2
+fi
+exit 1
diff --git a/tests/test_eval_health_notify.bats b/tests/test_eval_health_notify.bats
index b6d1d6c10..dfea9bdb9 100644
--- a/tests/test_eval_health_notify.bats
+++ b/tests/test_eval_health_notify.bats
@@ -39,6 +39,10 @@ JSON
# The script's de-dup looks up the single open tracking issue by stable title.
# Keep this title in lock-step with notify-eval-health.sh (per-skill TITLE).
TITLE='Skill eval health — `triage` (holdout)'
+ # The infra-error tracker is a SEPARATE issue (distinct title + label) so a
+ # throttle/outage (outcome=error, #920) never opens or updates the held-out
+ # regression tracker above.
+ TITLE_INFRA='Skill eval infra error — `triage` (holdout)'
# JSON the stub returns for `issue list`: empty by default; tests opt into an
# "existing open issue" by setting GH_LIST_JSON to a matching-title array.
EXISTING_JSON="$(jq -nc --arg t "$TITLE" '[{"number":42,"title":$t}]')"
@@ -81,6 +85,8 @@ run_notify() {
# A `gh issue list` JSON payload with one matching-title open issue numbered $1.
mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; }
+# Same, but titled as the infra-error tracker (#920).
+mklist_infra() { jq -nc --arg t "$TITLE_INFRA" --argjson n "$1" '[{number:$n,title:$t}]'; }
# --- regression: open / update / de-dupe ------------------------------------
@@ -118,23 +124,44 @@ mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; }
grep -q "actions/runs/999" "$GH_LOG"
}
-# --- error: still visible (never silent) ------------------------------------
+# --- error: separate infra tracker, never the regression tracker (#920) ------
-@test "error outcome opens the tracking issue even with an unparseable report" {
+@test "error opens a SEPARATE infra tracker (eval-infra label), not the regression tracker (#920)" {
echo 'not json' >"$REPORT"
run_notify OUTCOME="error"
[ "$status" -eq 0 ]
grep -q "ARGS: issue create" "$GH_LOG"
+ # Opened under the infra label; the regression label is never used on an error.
+ grep -q -- "--label eval-infra" "$GH_LOG"
+ ! grep -q -- "--label eval-health" "$GH_LOG"
}
-@test "error outcome with an existing issue updates it (de-dupe, not silent)" {
+@test "error with an existing infra issue updates it in place (de-dupe, not silent)" {
echo 'not json' >"$REPORT"
- run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist 7)"
+ run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist_infra 7)"
[ "$status" -eq 0 ]
grep -q "ARGS: issue edit 7" "$GH_LOG"
! grep -q "ARGS: issue create" "$GH_LOG"
}
+@test "error never edits or closes an open held-out REGRESSION tracker (#920)" {
+ echo 'not json' >"$REPORT"
+ # A regression tracker (#99) is open; an infra error must leave it untouched
+ # and instead open its own infra tracker.
+ run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist 99)"
+ [ "$status" -eq 0 ]
+ ! grep -q "ARGS: issue edit 99" "$GH_LOG"
+ ! grep -q "ARGS: issue close 99" "$GH_LOG"
+ grep -q "ARGS: issue create" "$GH_LOG"
+}
+
+@test "error body labels the outcome as infra, not a regression (#920)" {
+ echo 'not json' >"$REPORT"
+ run_notify OUTCOME="error"
+ [ "$status" -eq 0 ]
+ grep -qi "infra" "$GH_LOG"
+}
+
# --- pass: recovery closes; healthy makes no noise --------------------------
@test "pass with a prior open issue comments and closes it (recovery)" {
@@ -152,6 +179,13 @@ mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; }
! grep -q "ARGS: issue close" "$GH_LOG"
}
+@test "pass closes an open infra tracker too (outage recovery, #920)" {
+ run_notify OUTCOME="pass" "GH_LIST_JSON=$(mklist_infra 88)"
+ [ "$status" -eq 0 ]
+ grep -q "ARGS: issue comment 88" "$GH_LOG"
+ grep -q "ARGS: issue close 88" "$GH_LOG"
+}
+
# --- guards ------------------------------------------------------------------
@test "missing REPO is a hard error" {
diff --git a/tests/test_skill_evals.bats b/tests/test_skill_evals.bats
index 7a98d4a71..6ce491d03 100644
--- a/tests/test_skill_evals.bats
+++ b/tests/test_skill_evals.bats
@@ -57,6 +57,35 @@ echo 'This PR looks fine to me, I would approve it.'
SH
chmod +x "$STUB_PROSE"
+ # Stub engine that always THROTTLES: exits non-zero after emitting no parseable
+ # output — mimics every model in engine.sh's fallback chain being throttled
+ # (#920). The non-zero EXIT is the infra signal the scorer must read so a
+ # transient outage scores as an infra error (exit 2), not a regression.
+ STUB_THROTTLE="$TMP/stub_throttle.sh"
+ cat >"$STUB_THROTTLE" <<'SH'
+#!/usr/bin/env bash
+echo '::warning::[claude] all models throttled (rc=1)' >&2
+exit 1
+SH
+ chmod +x "$STUB_THROTTLE"
+
+ # Stub engine for a MIXED run: answers the approve case correctly (rc 0) but
+ # throttles the escalate case (rc!=0, no output). Used to pin the deterministic
+ # rule for a partially-throttled run.
+ STUB_MIXED="$TMP/stub_mixed.sh"
+ cat >"$STUB_MIXED" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+prompt="$1"
+if grep -q MARKER_APPROVE "$prompt"; then
+ echo '{"escalate": false, "risk": "LOW", "signals": [], "summary": "docs only"}'
+ exit 0
+fi
+echo '::warning::[claude] all models throttled (rc=1)' >&2
+exit 1
+SH
+ chmod +x "$STUB_MIXED"
+
# Stub engine that wraps its (correct) JSON decision in a ```json markdown
# fence — the live Haiku-tier output the strict parser used to reject, which
# produced the production triage 0/5 (every case `got null`, #762).
@@ -138,6 +167,60 @@ teardown() { rm -rf "$TMP"; }
[ "$(jq -r '.actual.risk' <<<"$esc")" = "HIGH" ]
}
+# --- infra failure vs. quality regression (throttle classification, #920) ------
+#
+# A throttle (every model in the engine fallback chain exits non-zero) means the
+# skill was never actually scored — it must NOT read as a held-out regression.
+# The scorer captures the engine's per-case exit status (engine_rc) and, when
+# every failing case is an infra failure, exits 2 (-> outcome=error) instead of 1
+# (-> regression). A genuine quality miss (engine exits 0, output wrong) is
+# unchanged: exit 1.
+
+@test "all-throttled run (every engine call exits non-zero) is an infra error -> exit 2 (#920)" {
+ EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_THROTTLE" \
+ run --separate-stderr bash "$SCORER" triage
+ [ "$status" -eq 2 ]
+ # Both cases failed and every failing case carries a non-zero engine_rc.
+ [ "$(jq '.failed' <<<"$output")" -eq 2 ]
+ [ "$(jq '[.cases[] | select(.engine_rc != 0)] | length' <<<"$output")" -eq 2 ]
+}
+
+@test "mixed run (one answered, one throttled) classifies as infra error -> exit 2 (#920)" {
+ EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_MIXED" \
+ run --separate-stderr bash "$SCORER" triage
+ [ "$status" -eq 2 ]
+ ap="$(jq -c '.cases[] | select(.id=="case-approve")' <<<"$output")"
+ [ "$(jq -r '.pass' <<<"$ap")" = "true" ]
+ [ "$(jq -r '.engine_rc' <<<"$ap")" = "0" ]
+ esc="$(jq -c '.cases[] | select(.id=="case-escalate")' <<<"$output")"
+ [ "$(jq -r '.pass' <<<"$esc")" = "false" ]
+ [ "$(jq -r '.engine_rc' <<<"$esc")" != "0" ]
+}
+
+@test "a genuine quality miss (engine exits 0, wrong answer) stays a regression -> exit 1 (#920)" {
+ EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_WRONG" \
+ run --separate-stderr bash "$SCORER" triage
+ [ "$status" -eq 1 ]
+ # The failing case ran the engine successfully (rc 0) — a quality miss, not infra.
+ esc="$(jq -c '.cases[] | select(.id=="case-escalate")' <<<"$output")"
+ [ "$(jq -r '.engine_rc' <<<"$esc")" = "0" ]
+}
+
+@test "unparseable output with a clean engine exit is a regression, not infra -> exit 1 (#920)" {
+ EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_PROSE" \
+ run --separate-stderr bash "$SCORER" triage
+ [ "$status" -eq 1 ]
+ # Engine exited 0 for every case (the output was just wrong/unparseable).
+ [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ]
+}
+
+@test "passing cases carry engine_rc 0 (clean response wiring, #920)" {
+ EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_OK" \
+ run --separate-stderr bash "$SCORER" triage
+ [ "$status" -eq 0 ]
+ [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ]
+}
+
@test "aggregate score is passed/total across a mixed set" {
EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_WRONG" \
run --separate-stderr bash "$SCORER" triage
@@ -259,6 +342,9 @@ SH
# Each case carries a numeric score (AC #3).
[ "$(jq '.cases[0].score' <<<"$output")" = "1" ]
[ "$(jq '.cases[].score | numbers' <<<"$output" | wc -l)" -eq 2 ]
+ # …and the engine exit status, so the throttle classification (#920) works in
+ # llm-judge mode too. The skill stub answered cleanly here -> engine_rc 0.
+ [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ]
}
@test "llm-judge mode: a sub-threshold score fails the case (exit 1)" {