Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 111 additions & 54 deletions scripts/evals/notify-eval-health.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,22 @@ set -euo pipefail
# step summary / artifact / annotation — effectively invisible. This notifier
# makes the outcome visible WITHOUT making it blocking:
#
# outcome=regression | error -> open OR update a single `eval-health` tracking
# issue (idempotent: the same issue is rewritten
# in place, never a new one per day).
# outcome=pass -> if a tracking issue is currently open, comment
# + close it (recovery); otherwise do NOTHING
# (a healthy run must create zero noise).
# outcome=regression -> open OR update the single `eval-health` REGRESSION
# tracker (idempotent: rewritten in place, never a new
# one per day).
# outcome=error -> open OR update a SEPARATE `eval-infra` INFRA-ERROR
# tracker. An infra error (e.g. every model throttled,
# #920, or bad usage / missing tooling) means the skill
# was never scored — it must NOT open or update the
# held-out regression tracker, or a transient outage
# reads as a skill regression. It stays visible via its
# own clearly-labeled issue.
# outcome=pass -> a green run clears either condition: comment + close
# whichever tracker(s) (regression and/or infra) are
# currently open. With none open, do NOTHING (a healthy
# run must create zero noise).
#
# De-dup key: the single open issue carrying $EVAL_HEALTH_LABEL whose title
# De-dup key: the single open issue carrying the matching label whose title
# matches the stable per-skill title below. Stable title => one issue, rewritten.
#
# This script only adds visibility; it never changes the scorer's pass/fail
Expand All @@ -28,7 +36,8 @@ set -euo pipefail
# SKILL skill whose holdout was scored (default: triage)
# REPORT_PATH eval-report.json from the scorer (default: ./eval-report.json)
# RUN_URL link back to the workflow run (optional but recommended)
# EVAL_HEALTH_LABEL tracking-issue label (default: eval-health)
# EVAL_HEALTH_LABEL regression tracking-issue label (default: eval-health)
# EVAL_INFRA_LABEL infra-error tracking-issue label (default: eval-infra)
# DRY_RUN "1"/"true" => render + log intent, never mutate via gh
#
# Exit codes: 0 success (including the no-op healthy path); 1 bad usage / gh error.
Expand All @@ -39,40 +48,56 @@ SKILL="${SKILL:-triage}"
REPORT_PATH="${REPORT_PATH:-eval-report.json}"
RUN_URL="${RUN_URL:-}"
EVAL_HEALTH_LABEL="${EVAL_HEALTH_LABEL:-eval-health}"
EVAL_INFRA_LABEL="${EVAL_INFRA_LABEL:-eval-infra}"
Comment thread
don-petry marked this conversation as resolved.
Resolved

command -v jq >/dev/null 2>&1 || { echo "::error::notify-eval-health: jq is required but not installed" >&2; exit 1; }
command -v gh >/dev/null 2>&1 || { echo "::error::notify-eval-health: gh is required but not installed" >&2; exit 1; }

case "$OUTCOME" in
pass|regression|error) ;;
*) echo "::error::notify-eval-health: unknown OUTCOME '$OUTCOME' (expected pass|regression|error)" >&2; exit 1 ;;
esac

# Stable, per-skill title so the lookup always lands on the same issue.
TITLE="Skill eval health — \`${SKILL}\` (holdout)"
# Stable, per-skill titles so each lookup always lands on the same issue. The
# regression tracker and the infra-error tracker are SEPARATE issues with
# distinct labels (#920): a throttle/outage must not touch the regression
# tracker, so each is de-duped independently.
TITLE_REGRESSION="Skill eval health — \`${SKILL}\` (holdout)"
TITLE_INFRA="Skill eval infra error — \`${SKILL}\` (holdout)"

_is_dry_run() { [[ "${DRY_RUN:-0}" == "1" || "${DRY_RUN:-0}" == "true" ]]; }

# Locate the single open tracking issue (scoped by label) whose title matches.
# gh emits the candidate list as JSON; system jq does the title de-dup so the
# expression is unambiguous (no reliance on gh's embedded jq arg passing).
# find_existing <title> <label> — locate the single open tracking issue (scoped
# by label) whose title matches. gh emits the candidate list as JSON; system jq
# does the title de-dup so the expression is unambiguous (no reliance on gh's
# embedded jq arg passing).
find_existing() {
if [ "$#" -lt 2 ]; then
echo "::error::find_existing requires title and label" >&2
exit 1
fi
local title="$1" label="$2"
Comment thread
don-petry marked this conversation as resolved.
Resolved
if _is_dry_run; then
printf '%s' "${DRY_RUN_EXISTING:-}"
return 0
fi
gh issue list --repo "$REPO" --label "$EVAL_HEALTH_LABEL" --state open \
gh issue list --repo "$REPO" --label "$label" --state open \
--json number,title 2>/dev/null \
| jq -r --arg t "$TITLE" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \
| jq -r --arg t "$title" 'map(select(.title == $t)) | (.[0].number // "")' 2>/dev/null \
|| true
}

# Render the tracking-issue body for a regression/error outcome to stdout.
render_body() {
echo "## Skill eval health — \`${SKILL}\` (holdout)"
echo ""
if [ "$OUTCOME" = "error" ]; then
echo "**Outcome:** \`error\` — the scorer hard-errored (bad usage / missing tooling)."
echo "## ${TITLE_INFRA}"
echo ""
echo "**Outcome:** \`error\` — the eval could not score the skill (infrastructure failure: e.g. every model in the fallback chain throttled, or bad usage / missing tooling)."
echo ""
echo "_The eval run could not produce a score. This is report-only (Phase 1): it does not block any merge — but it does mean the held-out set is currently un-scored and needs a look._"
echo "_This is an **infra** signal, **not** a skill regression: the model never produced a scorable answer, so the held-out set is currently un-scored and needs a look. Report-only (Phase 1): it does not block any merge._"
else
echo "## ${TITLE_REGRESSION}"
echo ""
echo "**Outcome:** \`regression\` — at least one held-out case regressed."
echo ""
echo "_Report-only (Phase 1): a regression does not fail CI or block any merge._"
Expand Down Expand Up @@ -105,48 +130,80 @@ render_body() {
echo "<sub>Auto-managed by \`notify-eval-health.sh\` — this issue is updated in place each run and closed automatically on recovery.</sub>"
}

existing="$(find_existing)"
# _recover <issue_number> — comment a recovery note on the issue and close it.
_recover() {
if [ "$#" -lt 1 ]; then
echo "::error::_recover requires an issue number" >&2
exit 1
fi
local num="$1" comment
comment="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; }
{
echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again."
echo ""
[ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})"
} >"$comment"
if _is_dry_run; then
echo "[dry-run] would comment on and close issue #${num}"
rm -f "$comment"
return 0
fi
gh issue comment "$num" --repo "$REPO" --body-file "$comment" >/dev/null
gh issue close "$num" --repo "$REPO" >/dev/null
rm -f "$comment"
echo "closed tracking issue #${num} (recovered)"
}

# _open_or_update <title> <label> <label_description> <label_color> — open the
# tracker (de-duped by title+label) or rewrite it in place if already open.
_open_or_update() {
if [ "$#" -lt 4 ]; then
echo "::error::_open_or_update requires title, label, description, and color" >&2
exit 1
fi
local title="$1" label="$2" desc="$3" color="$4" existing body url
existing="$(find_existing "$title" "$label")"
body="$(mktemp)" || { echo "::error::Failed to create temp file" >&2; exit 1; }
render_body >"$body"
if _is_dry_run; then
echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) issue '${title}' (${label})"
rm -f "$body"
return 0
fi
if [ -n "$existing" ]; then
gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null
echo "updated tracking issue #${existing} (${label})"
else
# Ensure the label exists (ignore failure if it already does).
gh label create "$label" --repo "$REPO" --description "$desc" --color "$color" 2>/dev/null || true
url="$(gh issue create --repo "$REPO" --title "$title" --label "$label" --body-file "$body")"
echo "created tracking issue ${url} (${label})"
fi
rm -f "$body"
}

case "$OUTCOME" in
pass)
if [ -z "$existing" ]; then
# A green run clears either condition — close whichever tracker(s) are open.
reg="$(find_existing "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL")"
inf="$(find_existing "$TITLE_INFRA" "$EVAL_INFRA_LABEL")"
if [ -z "$reg" ] && [ -z "$inf" ]; then
echo "eval health pass: no open tracking issue — nothing to do."
exit 0
fi
comment="$(mktemp)"
trap 'rm -f "$comment"' EXIT
{
echo "✅ **Recovered** — the \`${SKILL}\` holdout eval passed; all held-out cases are green again."
echo ""
[ -n "$RUN_URL" ] && echo "[View the workflow run](${RUN_URL})"
} >"$comment"
if _is_dry_run; then
echo "[dry-run] would comment on and close eval-health issue #${existing}"
exit 0
fi
gh issue comment "$existing" --repo "$REPO" --body-file "$comment" >/dev/null
gh issue close "$existing" --repo "$REPO" >/dev/null
echo "closed eval-health issue #${existing} (recovered)"
if [ -n "$reg" ]; then _recover "$reg"; fi
if [ -n "$inf" ]; then _recover "$inf"; fi
;;

regression|error)
body="$(mktemp)"
trap 'rm -f "$body"' EXIT
render_body >"$body"
if _is_dry_run; then
echo "[dry-run] would $( [ -n "$existing" ] && echo "update #${existing}" || echo "create" ) eval-health issue '${TITLE}'"
exit 0
fi
if [ -n "$existing" ]; then
gh issue edit "$existing" --repo "$REPO" --body-file "$body" >/dev/null
echo "updated eval-health issue #${existing}"
else
# Ensure the label exists (ignore failure if it already does).
gh label create "$EVAL_HEALTH_LABEL" --repo "$REPO" \
--description "Skill-eval holdout health tracking" --color d93f0b 2>/dev/null || true
url="$(gh issue create --repo "$REPO" --title "$TITLE" \
--label "$EVAL_HEALTH_LABEL" --body-file "$body")"
echo "created eval-health issue ${url}"
fi
regression)
_open_or_update "$TITLE_REGRESSION" "$EVAL_HEALTH_LABEL" \
"Skill-eval holdout health tracking" "d93f0b"
;;

error)
# Distinct infra tracker (#920): an infra error never touches the regression
# tracker, so a transient throttle cannot read as a held-out regression.
_open_or_update "$TITLE_INFRA" "$EVAL_INFRA_LABEL" \
"Skill-eval holdout infra-error tracking" "fbca04"
;;
esac
72 changes: 56 additions & 16 deletions scripts/evals/run-eval.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,23 @@ set -euo pipefail
#
# Exit codes:
# 0 run completed, every case passed
# 1 run completed, at least one case failed
# 2 hard error (bad usage / missing prompt or cases file / missing tooling)
# 1 run completed, at least one case failed on QUALITY (a genuine regression)
# 2 hard error — bad usage / missing prompt or cases file / missing tooling,
# OR an all-infra run where the skill was never actually scored (see below)
#
# Infra failure vs. quality regression (#920): each case records the engine's
# exit status (engine_rc). A failing case whose engine call exited NON-ZERO is an
# INFRA failure (e.g. every model in engine.sh's fallback chain throttled) — the
# model never answered, so there is no answer to call right or wrong. A failing
# case whose engine call exited ZERO is a QUALITY miss (the model answered, but
# the output was wrong or unparseable). The aggregate exit code, when >=1 case
# fails, is:
# - every failing case is an infra failure (no quality miss anywhere) -> 2
# (outcome=error). The held-out set is currently UN-scored; a transient
# throttle must NOT open a false held-out regression tracker.
# - otherwise (>=1 quality miss) -> 1 (regression), unchanged.
# A MIXED run with some cases answered (pass) and the rest throttled therefore
# exits 2: there is no parseable-but-wrong answer to call a regression.
#
# Cases are read-only here — the scorer never writes evals/<skill>/holdout/cases.jsonl.

Expand Down Expand Up @@ -128,12 +143,17 @@ trap 'rm -f "$results" "$work_prompt" "$work_judge"' EXIT
# lines, exit 1) from tripping `set -e`.
extract_json() { grep -v $'^[[:space:]]*\x60\x60\x60' <<<"$1" || true; }

# score_deterministic <expected_json> <raw_output> <case_id>
# score_deterministic <expected_json> <raw_output> <case_id> <engine_rc>
# Equality scorer (triage). Parses the model output; a non-JSON-object output
# yields a null actual decision (an automatic failure). Every result carries a
# numeric `score` (1 pass / 0 fail) so the report shape matches the judge mode.
# numeric `score` (1 pass / 0 fail) plus the engine exit status `engine_rc` (so
# an all-infra run can be told apart from a quality regression, #920) so the
# report shape matches the judge mode.
score_deterministic() {
local expected="$1" raw="$2" cid="$3"
if [ "$#" -lt 3 ]; then
die "score_deterministic requires expected, raw, and cid arguments"
fi
local expected="$1" raw="$2" cid="$3" eng_rc="${4:-0}"
Comment thread
don-petry marked this conversation as resolved.
Resolved
local exp_esc exp_risk actual pass score cleaned raw_excerpt=""
exp_esc="$(jq -r '.escalate' <<<"$expected")"
exp_risk="$(jq -r '.risk' <<<"$expected")"
Expand Down Expand Up @@ -169,9 +189,10 @@ score_deterministic() {
# raw_excerpt is attached ONLY when the output failed to parse (actual=null), so
# passing/mismatch cases keep the lean shape the report and tests expect.
jq -cn --arg id "$cid" --argjson pass "$pass" --argjson score "$score" \
--argjson engine_rc "$eng_rc" \
--argjson exp_esc "$exp_esc" --arg exp_risk "$exp_risk" \
--argjson actual "$actual" --arg raw_excerpt "$raw_excerpt" \
'{id:$id, score:$score, pass:$pass, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual}
'{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:{escalate:$exp_esc, risk:$exp_risk}, actual:$actual}
+ (if $raw_excerpt == "" then {} else {raw_excerpt: $raw_excerpt} end)'
}

Expand All @@ -182,7 +203,10 @@ score_deterministic() {
# judge engine (default run_triage), and parses its numeric `score`. Judge output
# that is not a JSON object with a numeric `score` scores 0 (an automatic failure).
score_llm_judge() {
local expected="$1" candidate="$2" cid="$3"
if [ "$#" -lt 3 ]; then
die "score_llm_judge requires expected, candidate, and cid arguments"
fi
local expected="$1" candidate="$2" cid="$3" eng_rc="${4:-0}"
Comment thread
don-petry marked this conversation as resolved.
Resolved
local judge_raw judge_obj score pass

{
Expand Down Expand Up @@ -216,9 +240,10 @@ score_llm_judge() {
fi

jq -cn --arg id "$cid" --argjson score "$score" --argjson pass "$pass" \
--argjson engine_rc "$eng_rc" \
--argjson expected "$expected" --argjson judge "$judge_obj" \
--arg candidate "$candidate" \
'{id:$id, score:$score, pass:$pass, expected:$expected, judge:$judge, candidate:$candidate}'
'{id:$id, score:$score, pass:$pass, engine_rc:$engine_rc, expected:$expected, judge:$judge, candidate:$candidate}'
}

while IFS= read -r line; do
Expand All @@ -236,15 +261,18 @@ while IFS= read -r line; do
} >"$work_prompt"

# Invoke the skill through the engine command (default run_triage). Capture
# stdout (the model decision); engine diagnostics flow to our stderr. A
# non-zero engine exit leaves $raw empty -> scored as a failure.
raw=""
raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || raw=""
# stdout (the model decision); engine diagnostics flow to our stderr. Preserve
# the engine's exit status (eng_rc): a NON-ZERO exit means the call failed for
# infra reasons (e.g. every model in the fallback chain throttled) — the only
# signal that distinguishes a throttle from a model that answered wrong (#920).
raw=""; eng_rc=0
raw="$("$EVAL_ENGINE_CMD" "$work_prompt")" || eng_rc=$?

# Dispatch on the per-skill scorer mode — never on the skill name.
# Dispatch on the per-skill scorer mode — never on the skill name. eng_rc is
# threaded through so each case result records whether the engine actually ran.
case "$SCORER_MODE" in
deterministic) score_deterministic "$expected" "$raw" "$cid" >>"$results" ;;
llm-judge) score_llm_judge "$expected" "$raw" "$cid" >>"$results" ;;
deterministic) score_deterministic "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;;
llm-judge) score_llm_judge "$expected" "$raw" "$cid" "$eng_rc" >>"$results" ;;
esac
done <"$cases_file"

Expand All @@ -259,5 +287,17 @@ jq -s --arg skill "$skill" '
cases: .
}' "$results"

# Decide the exit code. Classify failing cases as infra (engine exited non-zero —
# throttle/outage, the model never answered) vs. quality (engine exited zero but
# the answer was wrong/unparseable). If EVERY failing case is infra, the skill
# was never actually scored: exit 2 -> outcome=error, so a transient throttle
# cannot open a false held-out regression (#920). Any quality miss keeps exit 1.
failed="$(jq -s 'map(select(.pass | not)) | length' "$results")"
[ "$failed" -eq 0 ]
if [ "$failed" -eq 0 ]; then
exit 0
fi
quality_failed="$(jq -s 'map(select((.pass | not) and ((.engine_rc // 0) == 0))) | length' "$results")"
if [ "$quality_failed" -eq 0 ]; then
exit 2
fi
exit 1
Loading
Loading