From efc85c6c26c3d3c70ee6d6163778d19b5ed312cd Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Thu, 25 Jun 2026 03:03:39 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20implement=20issue=20#920=20?= =?UTF-8?q?=E2=80=94=20Skill=20eval:=20API=20throttling=20masquerades=20as?= =?UTF-8?q?=20a=20held-out=20regression=20(infra=20failure=20scored=20as?= =?UTF-8?q?=20`regression`,=20not=20`error`)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/evals/notify-eval-health.sh | 150 ++++++++++++++++++---------- scripts/evals/run-eval.sh | 66 +++++++++--- tests/test_eval_health_notify.bats | 42 +++++++- tests/test_skill_evals.bats | 86 ++++++++++++++++ 4 files changed, 270 insertions(+), 74 deletions(-) diff --git a/scripts/evals/notify-eval-health.sh b/scripts/evals/notify-eval-health.sh index 0ce2f5d1b..3f8c0ed7c 100644 --- a/scripts/evals/notify-eval-health.sh +++ b/scripts/evals/notify-eval-health.sh @@ -9,14 +9,22 @@ set -euo pipefail # step summary / artifact / annotation — effectively invisible. This notifier # makes the outcome visible WITHOUT making it blocking: # -# outcome=regression | error -> open OR update a single `eval-health` tracking -# issue (idempotent: the same issue is rewritten -# in place, never a new one per day). -# outcome=pass -> if a tracking issue is currently open, comment -# + close it (recovery); otherwise do NOTHING -# (a healthy run must create zero noise). +# outcome=regression -> open OR update the single `eval-health` REGRESSION +# tracker (idempotent: rewritten in place, never a new +# one per day). +# outcome=error -> open OR update a SEPARATE `eval-infra` INFRA-ERROR +# tracker. An infra error (e.g. every model throttled, +# #920, or bad usage / missing tooling) means the skill +# was never scored — it must NOT open or update the +# held-out regression tracker, or a transient outage +# reads as a skill regression. It stays visible via its +# own clearly-labeled issue. +# outcome=pass -> a green run clears either condition: comment + close +# whichever tracker(s) (regression and/or infra) are +# currently open. With none open, do NOTHING (a healthy +# run must create zero noise). # -# De-dup key: the single open issue carrying $EVAL_HEALTH_LABEL whose title +# De-dup key: the single open issue carrying the matching label whose title # matches the stable per-skill title below. Stable title => one issue, rewritten. # # This script only adds visibility; it never changes the scorer's pass/fail @@ -28,7 +36,8 @@ set -euo pipefail # SKILL skill whose holdout was scored (default: triage) # REPORT_PATH eval-report.json from the scorer (default: ./eval-report.json) # RUN_URL link back to the workflow run (optional but recommended) -# EVAL_HEALTH_LABEL tracking-issue label (default: eval-health) +# EVAL_HEALTH_LABEL regression tracking-issue label (default: eval-health) +# EVAL_INFRA_LABEL infra-error tracking-issue label (default: eval-infra) # DRY_RUN "1"/"true" => render + log intent, never mutate via gh # # Exit codes: 0 success (including the no-op healthy path); 1 bad usage / gh error. @@ -39,40 +48,49 @@ SKILL="${SKILL:-triage}" REPORT_PATH="${REPORT_PATH:-eval-report.json}" RUN_URL="${RUN_URL:-}" EVAL_HEALTH_LABEL="${EVAL_HEALTH_LABEL:-eval-health}" +EVAL_INFRA_LABEL="${EVAL_INFRA_LABEL:-eval-infra}" case "$OUTCOME" in pass|regression|error) ;; *) echo "::error::notify-eval-health: unknown OUTCOME '$OUTCOME' (expected pass|regression|error)" >&2; exit 1 ;; esac -# Stable, per-skill title so the lookup always lands on the same issue. -TITLE="Skill eval health — \`${SKILL}\` (holdout)" +# Stable, per-skill titles so each lookup always lands on the same issue. The +# regression tracker and the infra-error tracker are SEPARATE issues with +# distinct labels (#920): a throttle/outage must not touch the regression +# tracker, so each is de-duped independently. +TITLE_REGRESSION="Skill eval health — \`${SKILL}\` (holdout)" +TITLE_INFRA="Skill eval infra error — \`${SKILL}\` (holdout)" _is_dry_run() { [[ "${DRY_RUN:-0}" == "1" || "${DRY_RUN:-0}" == "true" ]]; } -# Locate the single open tracking issue (scoped by label) whose title matches. -# gh emits the candidate list as JSON; system jq does the title de-dup so the -# expression is unambiguous (no reliance on gh's embedded jq arg passing). +# find_existing <label> — locate the single open tracking issue (scoped +# by label) whose title matches. gh emits the candidate list as JSON; system jq +# does the title de-dup so the expression is unambiguous (no reliance on gh's +# embedded jq arg passing). find_existing() { + local title="$1" label="$2" if _is_dry_run; then printf '%s' "${DRY_RUN_EXISTING:-}" return 0 fi - gh issue list --repo "$REPO" --label "$EVAL_HEALTH_LABEL" --state open \ + gh issue list --repo "$REPO" --label "$label" --state open \ --json number,title 2>/dev/null \ - | jq -r --arg t "$TITLE" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \ + | jq -r --arg t "$title" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \ || true } # Render the tracking-issue body for a regression/error outcome to stdout. render_body() { - echo "## Skill eval health — \`${SKILL}\` (holdout)" - echo "" if [ "$OUTCOME" = "error" ]; then - echo "**Outcome:** \`error\` — the scorer hard-errored (bad usage / missing tooling)." + echo "## ${TITLE_INFRA}" + echo "" + echo "**Outcome:** \`error\` — the eval could not score the skill (infrastructure failure: e.g. every model in the fallback chain throttled, or bad usage / missing tooling)." echo "" - echo "_The eval run could not produce a score. This is report-only (Phase 1): it does not block any merge — but it does mean the held-out set is currently un-scored and needs a look._" + echo "_This is an **infra** signal, **not** a skill regression: the model never produced a scorable answer, so the held-out set is currently un-scored and needs a look. Report-only (Phase 1): it does not block any merge._" else + echo "## ${TITLE_REGRESSION}" + echo "" echo "**Outcome:** \`regression\` — at least one held-out case regressed." echo "" echo "_Report-only (Phase 1): a regression does not fail CI or block any merge._" @@ -105,48 +123,72 @@ render_body() { echo "<sub>Auto-managed by \`notify-eval-health.sh\` — this issue is updated in place each run and closed automatically on recovery.</sub>" } -existing="$(find_existing)" +# _recover <issue_number> — comment a recovery note on the issue and close it. +_recover() { + local num="$1" comment + comment="$(mktemp)" + { + echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again." + echo "" + [ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})" + } >"$comment" + if _is_dry_run; then + echo "[dry-run] would comment on and close issue #${num}" + rm -f "$comment" + return 0 + fi + gh issue comment "$num" --repo "$REPO" --body-file "$comment" >/dev/null + gh issue close "$num" --repo "$REPO" >/dev/null + rm -f "$comment" + echo "closed tracking issue #${num} (recovered)" +} + +# _open_or_update <title> <label> <label_description> <label_color> — open the +# tracker (de-duped by title+label) or rewrite it in place if already open. +_open_or_update() { + local title="$1" label="$2" desc="$3" color="$4" existing body url + existing="$(find_existing "$title" "$label")" + body="$(mktemp)" + render_body >"$body" + if _is_dry_run; then + echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) issue '${title}' (${label})" + rm -f "$body" + return 0 + fi + if [ -n "$existing" ]; then + gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null + echo "updated tracking issue #${existing} (${label})" + else + # Ensure the label exists (ignore failure if it already does). + gh label create "$label" --repo "$REPO" --description "$desc" --color "$color" 2>/dev/null || true + url="$(gh issue create --repo "$REPO" --title "$title" --label "$label" --body-file "$body")" + echo "created tracking issue ${url} (${label})" + fi + rm -f "$body" +} case "$OUTCOME" in pass) - if [ -z "$existing" ]; then + # A green run clears either condition — close whichever tracker(s) are open. + reg="$(find_existing "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL")" + inf="$(find_existing "$TITLE_INFRA" "$EVAL_INFRA_LABEL")" + if [ -z "$reg" ] && [ -z "$inf" ]; then echo "eval health pass: no open tracking issue — nothing to do." exit 0 fi - comment="$(mktemp)" - trap 'rm -f "$comment"' EXIT - { - echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again." - echo "" - [ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})" - } >"$comment" - if _is_dry_run; then - echo "[dry-run] would comment on and close eval-health issue #${existing}" - exit 0 - fi - gh issue comment "$existing" --repo "$REPO" --body-file "$comment" >/dev/null - gh issue close "$existing" --repo "$REPO" >/dev/null - echo "closed eval-health issue #${existing} (recovered)" + if [ -n "$reg" ]; then _recover "$reg"; fi + if [ -n "$inf" ]; then _recover "$inf"; fi ;; - regression|error) - body="$(mktemp)" - trap 'rm -f "$body"' EXIT - render_body >"$body" - if _is_dry_run; then - echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) eval-health issue '${TITLE}'" - exit 0 - fi - if [ -n "$existing" ]; then - gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null - echo "updated eval-health issue #${existing}" - else - # Ensure the label exists (ignore failure if it already does). - gh label create "$EVAL_HEALTH_LABEL" --repo "$REPO" \ - --description "Skill-eval holdout health tracking" --color d93f0b 2>/dev/null || true - url="$(gh issue create --repo "$REPO" --title "$TITLE" \ - --label "$EVAL_HEALTH_LABEL" --body-file "$body")" - echo "created eval-health issue ${url}" - fi + regression) + _open_or_update "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL" \ + "Skill-eval holdout health tracking" "d93f0b" + ;; + + error) + # Distinct infra tracker (#920): an infra error never touches the regression + # tracker, so a transient throttle cannot read as a held-out regression. + _open_or_update "$TITLE_INFRA" "$EVAL_INFRA_LABEL" \ + "Skill-eval holdout infra-error tracking" "fbca04" ;; esac diff --git a/scripts/evals/run-eval.sh b/scripts/evals/run-eval.sh index 28681c3c0..2a41defdc 100755 --- a/scripts/evals/run-eval.sh +++ b/scripts/evals/run-eval.sh @@ -51,8 +51,23 @@ set -euo pipefail # # Exit codes: # 0 run completed, every case passed -# 1 run completed, at least one case failed -# 2 hard error (bad usage / missing prompt or cases file / missing tooling) +# 1 run completed, at least one case failed on QUALITY (a genuine regression) +# 2 hard error — bad usage / missing prompt or cases file / missing tooling, +# OR an all-infra run where the skill was never actually scored (see below) +# +# Infra failure vs. quality regression (#920): each case records the engine's +# exit status (engine_rc). A failing case whose engine call exited NON-ZERO is an +# INFRA failure (e.g. every model in engine.sh's fallback chain throttled) — the +# model never answered, so there is no answer to call right or wrong. A failing +# case whose engine call exited ZERO is a QUALITY miss (the model answered, but +# the output was wrong or unparseable). The aggregate exit code, when >=1 case +# fails, is: +# - every failing case is an infra failure (no quality miss anywhere) -> 2 +# (outcome=error). The held-out set is currently UN-scored; a transient +# throttle must NOT open a false held-out regression tracker. +# - otherwise (>=1 quality miss) -> 1 (regression), unchanged. +# A MIXED run with some cases answered (pass) and the rest throttled therefore +# exits 2: there is no parseable-but-wrong answer to call a regression. # # Cases are read-only here — the scorer never writes evals/<skill>/holdout/cases.jsonl. @@ -128,12 +143,14 @@ trap 'rm -f "$results" "$work_prompt" "$work_judge"' EXIT # lines, exit 1) from tripping `set -e`. extract_json() { grep -v $'^[[:space:]]*\x60\x60\x60' <<<"$1" || true; } -# score_deterministic <expected_json> <raw_output> <case_id> +# score_deterministic <expected_json> <raw_output> <case_id> <engine_rc> # Equality scorer (triage). Parses the model output; a non-JSON-object output # yields a null actual decision (an automatic failure). Every result carries a -# numeric `score` (1 pass / 0 fail) so the report shape matches the judge mode. +# numeric `score` (1 pass / 0 fail) plus the engine exit status `engine_rc` (so +# an all-infra run can be told apart from a quality regression, #920) so the +# report shape matches the judge mode. score_deterministic() { - local expected="$1" raw="$2" cid="$3" + local expected="$1" raw="$2" cid="$3" eng_rc="${4:-0}" local exp_esc exp_risk actual pass score cleaned raw_excerpt="" exp_esc="$(jq -r '.escalate' <<<"$expected")" exp_risk="$(jq -r '.risk' <<<"$expected")" @@ -169,9 +186,10 @@ score_deterministic() { # raw_excerpt is attached ONLY when the output failed to parse (actual=null), so # passing/mismatch cases keep the lean shape the report and tests expect. jq -cn --arg id "$cid" --argjson pass "$pass" --argjson score "$score" \ + --argjson engine_rc "$eng_rc" \ --argjson exp_esc "$exp_esc" --arg exp_risk "$exp_risk" \ --argjson actual "$actual" --arg raw_excerpt "$raw_excerpt" \ - '{id:$id, score:$score, pass:$pass, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual} + '{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual} + (if $raw_excerpt == "" then {} else {raw_excerpt: $raw_excerpt} end)' } @@ -182,7 +200,7 @@ score_deterministic() { # judge engine (default run_triage), and parses its numeric `score`. Judge output # that is not a JSON object with a numeric `score` scores 0 (an automatic failure). score_llm_judge() { - local expected="$1" candidate="$2" cid="$3" + local expected="$1" candidate="$2" cid="$3" eng_rc="${4:-0}" local judge_raw judge_obj score pass { @@ -216,9 +234,10 @@ score_llm_judge() { fi jq -cn --arg id "$cid" --argjson score "$score" --argjson pass "$pass" \ + --argjson engine_rc "$eng_rc" \ --argjson expected "$expected" --argjson judge "$judge_obj" \ --arg candidate "$candidate" \ - '{id:$id, score:$score, pass:$pass, expected:$expected, judge:$judge, candidate:$candidate}' + '{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:$expected, judge:$judge, candidate:$candidate}' } while IFS= read -r line; do @@ -236,15 +255,18 @@ while IFS= read -r line; do } >"$work_prompt" # Invoke the skill through the engine command (default run_triage). Capture - # stdout (the model decision); engine diagnostics flow to our stderr. A - # non-zero engine exit leaves $raw empty -> scored as a failure. - raw="" - raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || raw="" + # stdout (the model decision); engine diagnostics flow to our stderr. Preserve + # the engine's exit status (eng_rc): a NON-ZERO exit means the call failed for + # infra reasons (e.g. every model in the fallback chain throttled) — the only + # signal that distinguishes a throttle from a model that answered wrong (#920). + raw=""; eng_rc=0 + raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || eng_rc=$? - # Dispatch on the per-skill scorer mode — never on the skill name. + # Dispatch on the per-skill scorer mode — never on the skill name. eng_rc is + # threaded through so each case result records whether the engine actually ran. case "$SCORER_MODE" in - deterministic) score_deterministic "$expected" "$raw" "$cid" >>"$results" ;; - llm-judge) score_llm_judge "$expected" "$raw" "$cid" >>"$results" ;; + deterministic) score_deterministic "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;; + llm-judge) score_llm_judge "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;; esac done <"$cases_file" @@ -259,5 +281,17 @@ jq -s --arg skill "$skill" ' cases: . }' "$results" +# Decide the exit code. Classify failing cases as infra (engine exited non-zero — +# throttle/outage, the model never answered) vs. quality (engine exited zero but +# the answer was wrong/unparseable). If EVERY failing case is infra, the skill +# was never actually scored: exit 2 -> outcome=error, so a transient throttle +# cannot open a false held-out regression (#920). Any quality miss keeps exit 1. failed="$(jq -s 'map(select(.pass | not)) | length' "$results")" -[ "$failed" -eq 0 ] +if [ "$failed" -eq 0 ]; then + exit 0 +fi +quality_failed="$(jq -s 'map(select((.pass | not) and ((.engine_rc // 0) == 0))) | length' "$results")" +if [ "$quality_failed" -eq 0 ]; then + exit 2 +fi +exit 1 diff --git a/tests/test_eval_health_notify.bats b/tests/test_eval_health_notify.bats index b6d1d6c10..dfea9bdb9 100644 --- a/tests/test_eval_health_notify.bats +++ b/tests/test_eval_health_notify.bats @@ -39,6 +39,10 @@ JSON # The script's de-dup looks up the single open tracking issue by stable title. # Keep this title in lock-step with notify-eval-health.sh (per-skill TITLE). TITLE='Skill eval health — `triage` (holdout)' + # The infra-error tracker is a SEPARATE issue (distinct title + label) so a + # throttle/outage (outcome=error, #920) never opens or updates the held-out + # regression tracker above. + TITLE_INFRA='Skill eval infra error — `triage` (holdout)' # JSON the stub returns for `issue list`: empty by default; tests opt into an # "existing open issue" by setting GH_LIST_JSON to a matching-title array. EXISTING_JSON="$(jq -nc --arg t "$TITLE" '[{"number":42,"title":$t}]')" @@ -81,6 +85,8 @@ run_notify() { # A `gh issue list` JSON payload with one matching-title open issue numbered $1. mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; } +# Same, but titled as the infra-error tracker (#920). +mklist_infra() { jq -nc --arg t "$TITLE_INFRA" --argjson n "$1" '[{number:$n,title:$t}]'; } # --- regression: open / update / de-dupe ------------------------------------ @@ -118,23 +124,44 @@ mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; } grep -q "actions/runs/999" "$GH_LOG" } -# --- error: still visible (never silent) ------------------------------------ +# --- error: separate infra tracker, never the regression tracker (#920) ------ -@test "error outcome opens the tracking issue even with an unparseable report" { +@test "error opens a SEPARATE infra tracker (eval-infra label), not the regression tracker (#920)" { echo 'not json' >"$REPORT" run_notify OUTCOME="error" [ "$status" -eq 0 ] grep -q "ARGS: issue create" "$GH_LOG" + # Opened under the infra label; the regression label is never used on an error. + grep -q -- "--label eval-infra" "$GH_LOG" + ! grep -q -- "--label eval-health" "$GH_LOG" } -@test "error outcome with an existing issue updates it (de-dupe, not silent)" { +@test "error with an existing infra issue updates it in place (de-dupe, not silent)" { echo 'not json' >"$REPORT" - run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist 7)" + run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist_infra 7)" [ "$status" -eq 0 ] grep -q "ARGS: issue edit 7" "$GH_LOG" ! grep -q "ARGS: issue create" "$GH_LOG" } +@test "error never edits or closes an open held-out REGRESSION tracker (#920)" { + echo 'not json' >"$REPORT" + # A regression tracker (#99) is open; an infra error must leave it untouched + # and instead open its own infra tracker. + run_notify OUTCOME="error" "GH_LIST_JSON=$(mklist 99)" + [ "$status" -eq 0 ] + ! grep -q "ARGS: issue edit 99" "$GH_LOG" + ! grep -q "ARGS: issue close 99" "$GH_LOG" + grep -q "ARGS: issue create" "$GH_LOG" +} + +@test "error body labels the outcome as infra, not a regression (#920)" { + echo 'not json' >"$REPORT" + run_notify OUTCOME="error" + [ "$status" -eq 0 ] + grep -qi "infra" "$GH_LOG" +} + # --- pass: recovery closes; healthy makes no noise -------------------------- @test "pass with a prior open issue comments and closes it (recovery)" { @@ -152,6 +179,13 @@ mklist() { jq -nc --arg t "$TITLE" --argjson n "$1" '[{number:$n,title:$t}]'; } ! grep -q "ARGS: issue close" "$GH_LOG" } +@test "pass closes an open infra tracker too (outage recovery, #920)" { + run_notify OUTCOME="pass" "GH_LIST_JSON=$(mklist_infra 88)" + [ "$status" -eq 0 ] + grep -q "ARGS: issue comment 88" "$GH_LOG" + grep -q "ARGS: issue close 88" "$GH_LOG" +} + # --- guards ------------------------------------------------------------------ @test "missing REPO is a hard error" { diff --git a/tests/test_skill_evals.bats b/tests/test_skill_evals.bats index 7a98d4a71..6ce491d03 100644 --- a/tests/test_skill_evals.bats +++ b/tests/test_skill_evals.bats @@ -57,6 +57,35 @@ echo 'This PR looks fine to me, I would approve it.' SH chmod +x "$STUB_PROSE" + # Stub engine that always THROTTLES: exits non-zero after emitting no parseable + # output — mimics every model in engine.sh's fallback chain being throttled + # (#920). The non-zero EXIT is the infra signal the scorer must read so a + # transient outage scores as an infra error (exit 2), not a regression. + STUB_THROTTLE="$TMP/stub_throttle.sh" + cat >"$STUB_THROTTLE" <<'SH' +#!/usr/bin/env bash +echo '::warning::[claude] all models throttled (rc=1)' >&2 +exit 1 +SH + chmod +x "$STUB_THROTTLE" + + # Stub engine for a MIXED run: answers the approve case correctly (rc 0) but + # throttles the escalate case (rc!=0, no output). Used to pin the deterministic + # rule for a partially-throttled run. + STUB_MIXED="$TMP/stub_mixed.sh" + cat >"$STUB_MIXED" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +prompt="$1" +if grep -q MARKER_APPROVE "$prompt"; then + echo '{"escalate": false, "risk": "LOW", "signals": [], "summary": "docs only"}' + exit 0 +fi +echo '::warning::[claude] all models throttled (rc=1)' >&2 +exit 1 +SH + chmod +x "$STUB_MIXED" + # Stub engine that wraps its (correct) JSON decision in a ```json markdown # fence — the live Haiku-tier output the strict parser used to reject, which # produced the production triage 0/5 (every case `got null`, #762). @@ -138,6 +167,60 @@ teardown() { rm -rf "$TMP"; } [ "$(jq -r '.actual.risk' <<<"$esc")" = "HIGH" ] } +# --- infra failure vs. quality regression (throttle classification, #920) ------ +# +# A throttle (every model in the engine fallback chain exits non-zero) means the +# skill was never actually scored — it must NOT read as a held-out regression. +# The scorer captures the engine's per-case exit status (engine_rc) and, when +# every failing case is an infra failure, exits 2 (-> outcome=error) instead of 1 +# (-> regression). A genuine quality miss (engine exits 0, output wrong) is +# unchanged: exit 1. + +@test "all-throttled run (every engine call exits non-zero) is an infra error -> exit 2 (#920)" { + EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_THROTTLE" \ + run --separate-stderr bash "$SCORER" triage + [ "$status" -eq 2 ] + # Both cases failed and every failing case carries a non-zero engine_rc. + [ "$(jq '.failed' <<<"$output")" -eq 2 ] + [ "$(jq '[.cases[] | select(.engine_rc != 0)] | length' <<<"$output")" -eq 2 ] +} + +@test "mixed run (one answered, one throttled) classifies as infra error -> exit 2 (#920)" { + EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_MIXED" \ + run --separate-stderr bash "$SCORER" triage + [ "$status" -eq 2 ] + ap="$(jq -c '.cases[] | select(.id=="case-approve")' <<<"$output")" + [ "$(jq -r '.pass' <<<"$ap")" = "true" ] + [ "$(jq -r '.engine_rc' <<<"$ap")" = "0" ] + esc="$(jq -c '.cases[] | select(.id=="case-escalate")' <<<"$output")" + [ "$(jq -r '.pass' <<<"$esc")" = "false" ] + [ "$(jq -r '.engine_rc' <<<"$esc")" != "0" ] +} + +@test "a genuine quality miss (engine exits 0, wrong answer) stays a regression -> exit 1 (#920)" { + EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_WRONG" \ + run --separate-stderr bash "$SCORER" triage + [ "$status" -eq 1 ] + # The failing case ran the engine successfully (rc 0) — a quality miss, not infra. + esc="$(jq -c '.cases[] | select(.id=="case-escalate")' <<<"$output")" + [ "$(jq -r '.engine_rc' <<<"$esc")" = "0" ] +} + +@test "unparseable output with a clean engine exit is a regression, not infra -> exit 1 (#920)" { + EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_PROSE" \ + run --separate-stderr bash "$SCORER" triage + [ "$status" -eq 1 ] + # Engine exited 0 for every case (the output was just wrong/unparseable). + [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ] +} + +@test "passing cases carry engine_rc 0 (clean response wiring, #920)" { + EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_OK" \ + run --separate-stderr bash "$SCORER" triage + [ "$status" -eq 0 ] + [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ] +} + @test "aggregate score is passed/total across a mixed set" { EVALS_DIR="$TMP/evals" EVAL_ENGINE_CMD="$STUB_WRONG" \ run --separate-stderr bash "$SCORER" triage @@ -259,6 +342,9 @@ SH # Each case carries a numeric score (AC #3). [ "$(jq '.cases[0].score' <<<"$output")" = "1" ] [ "$(jq '.cases[].score | numbers' <<<"$output" | wc -l)" -eq 2 ] + # …and the engine exit status, so the throttle classification (#920) works in + # llm-judge mode too. The skill stub answered cleanly here -> engine_rc 0. + [ "$(jq '[.cases[] | select(.engine_rc == 0)] | length' <<<"$output")" -eq 2 ] } @test "llm-judge mode: a sub-threshold score fails the case (exit 1)" { From d7cc5bb2704e62d399f7909deec3402807f27887 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Thu, 25 Jun 2026 03:10:47 +0000 Subject: [PATCH 2/2] fix(reviews): address review comments [skip ci-relay] --- scripts/evals/notify-eval-health.sh | 19 +++++++++++++++++-- scripts/evals/run-eval.sh | 6 ++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/evals/notify-eval-health.sh b/scripts/evals/notify-eval-health.sh index 3f8c0ed7c..bee0c3e70 100644 --- a/scripts/evals/notify-eval-health.sh +++ b/scripts/evals/notify-eval-health.sh @@ -50,6 +50,9 @@ RUN_URL="${RUN_URL:-}" EVAL_HEALTH_LABEL="${EVAL_HEALTH_LABEL:-eval-health}" EVAL_INFRA_LABEL="${EVAL_INFRA_LABEL:-eval-infra}" +command -v jq >/dev/null 2>&1 || { echo "::error::notify-eval-health: jq is required but not installed" >&2; exit 1; } +command -v gh >/dev/null 2>&1 || { echo "::error::notify-eval-health: gh is required but not installed" >&2; exit 1; } + case "$OUTCOME" in pass|regression|error) ;; *) echo "::error::notify-eval-health: unknown OUTCOME '$OUTCOME' (expected pass|regression|error)" >&2; exit 1 ;; @@ -69,6 +72,10 @@ _is_dry_run() { [[ "${DRY_RUN:-0}" == "1" || "${DRY_RUN:-0}" == "true" ]]; } # does the title de-dup so the expression is unambiguous (no reliance on gh's # embedded jq arg passing). find_existing() { + if [ "$#" -lt 2 ]; then + echo "::error::find_existing requires title and label" >&2 + exit 1 + fi local title="$1" label="$2" if _is_dry_run; then printf '%s' "${DRY_RUN_EXISTING:-}" @@ -125,8 +132,12 @@ render_body() { # _recover <issue_number> — comment a recovery note on the issue and close it. _recover() { + if [ "$#" -lt 1 ]; then + echo "::error::_recover requires an issue number" >&2 + exit 1 + fi local num="$1" comment - comment="$(mktemp)" + comment="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; } { echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again." echo "" @@ -146,9 +157,13 @@ _recover() { # _open_or_update <title> <label> <label_description> <label_color> — open the # tracker (de-duped by title+label) or rewrite it in place if already open. _open_or_update() { + if [ "$#" -lt 4 ]; then + echo "::error::_open_or_update requires title, label, description, and color" >&2 + exit 1 + fi local title="$1" label="$2" desc="$3" color="$4" existing body url existing="$(find_existing "$title" "$label")" - body="$(mktemp)" + body="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; } render_body >"$body" if _is_dry_run; then echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) issue '${title}' (${label})" diff --git a/scripts/evals/run-eval.sh b/scripts/evals/run-eval.sh index 2a41defdc..cf277a102 100755 --- a/scripts/evals/run-eval.sh +++ b/scripts/evals/run-eval.sh @@ -150,6 +150,9 @@ extract_json() { grep -v $'^[[:space:]]*\x60\x60\x60' <<<"$1" || true; } # an all-infra run can be told apart from a quality regression, #920) so the # report shape matches the judge mode. score_deterministic() { + if [ "$#" -lt 3 ]; then + die "score_deterministic requires expected, raw, and cid arguments" + fi local expected="$1" raw="$2" cid="$3" eng_rc="${4:-0}" local exp_esc exp_risk actual pass score cleaned raw_excerpt="" exp_esc="$(jq -r '.escalate' <<<"$expected")" @@ -200,6 +203,9 @@ score_deterministic() { # judge engine (default run_triage), and parses its numeric `score`. Judge output # that is not a JSON object with a numeric `score` scores 0 (an automatic failure). score_llm_judge() { + if [ "$#" -lt 3 ]; then + die "score_llm_judge requires expected, candidate, and cid arguments" + fi local expected="$1" candidate="$2" cid="$3" eng_rc="${4:-0}" local judge_raw judge_obj score pass