From f9a2cabf74266d0bcec339b45dee3ad9a07402fb Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Fri, 12 Jun 2026 01:43:08 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#582=20?= =?UTF-8?q?=E2=80=94=20[Phase=201]=20Define=20held-out=20eval-case=20forma?= =?UTF-8?q?t=20+=20seed=20prompts/triage.md=20case=20set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/CODEOWNERS | 5 ++ .github/workflows/lint.yml | 4 +- skills/evals/README.md | 64 ++++++++++++++++++++++++++ skills/evals/case.schema.json | 52 +++++++++++++++++++++ skills/evals/triage/cases.jsonl | 8 ++++ skills/evals/validate-cases.py | 79 ++++++++++++++++++++++++++++++++ tests/test_evals_cases.bats | 81 +++++++++++++++++++++++++++++++++ 7 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 skills/evals/README.md create mode 100644 skills/evals/case.schema.json create mode 100644 skills/evals/triage/cases.jsonl create mode 100644 skills/evals/validate-cases.py create mode 100644 tests/test_evals_cases.bats diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 03ca81662..9bbf44ce9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,7 @@ # Default owner for all files * @petry-projects/org-leads + +# Held-out eval cases are the trust anchor for skill-quality measurement. +# They MUST stay separately owned: a future automated skill proposer may only +# edit the skill markdown, never this test set (reward-hacking guard, #582). +/skills/evals/ @petry-projects/org-leads diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5b7e0ced5..f44af8cdd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,6 +7,7 @@ on: - '.github/workflows/**' - 'tests/**' - 'agents/**' + - 'skills/**' - '.gitleaks.toml' workflow_dispatch: @@ -56,7 +57,8 @@ jobs: tests/test_idea_enhancer.bats \ tests/test_oversized_pr.bats \ tests/test_apply_repo_settings.bats \ - tests/test_sweep_stuck_reviews.bats + tests/test_sweep_stuck_reviews.bats \ + tests/test_evals_cases.bats validate-agent-profiles: runs-on: ubuntu-latest diff --git a/skills/evals/README.md b/skills/evals/README.md new file mode 100644 index 000000000..7b68f6ba8 --- /dev/null +++ b/skills/evals/README.md @@ -0,0 +1,64 @@ +# Held-out eval cases + +This directory holds the **held-out eval-case sets** used to measure prompt-skill +quality against fixed, reward-hack-resistant expected outputs. It is the trust +anchor of the skill-quality initiative (Discussion #572): cases live here, on +their own, and are validated structurally — there is **no scorer and no model +invocation** in this directory. + +## Layout + +``` +skills/evals/ +├── case.schema.json # JSON-Schema (draft 2020-12) for one eval case +├── validate-cases.py # validate a .jsonl set against the schema +└── /cases.jsonl # one case per line for that skill (e.g. triage/) +``` + +Each `/cases.jsonl` is [JSON Lines](https://jsonlines.org/): one JSON +object per line, each conforming to `case.schema.json`. + +## Case format + +A case pairs a single triage **input** with the **expected** decision the skill +must emit. Required fields (see `case.schema.json` for the authoritative +contract — it is `additionalProperties: false`, so unknown keys are rejected): + +| Field | Type | Notes | +| ------------------- | ------------------------- | -------------------------------------------------------------------------------------- | +| `id` | string (kebab-case) | Stable, unique identifier for the case. | +| `input` | string | The pre-fetched PR context exactly as `prompts/triage.md` consumes under `## Pre-fetched PR context`. Self-contained; triage has no tools. | +| `expected.escalate` | boolean | Whether triage must escalate (`true`) or may approve (`false`). | +| `expected.risk` | `LOW` \| `MEDIUM` \| `HIGH` | Expected risk classification. Any `HIGH` signal must pair with `escalate: true`. | +| `description` | string (optional) | What the case exercises and which `triage.md` criterion grounds the expected decision. | +| `tags` | string[] (optional) | Coverage labels, e.g. the high-risk trigger a case exercises (`auth-secrets`, `db-migration`, `security-anti-pattern`). | + +Expected decisions are **grounded strictly in the numbered decision criteria of +`prompts/triage.md`** — not invented behavior. `triage.md` emits a deterministic +JSON object with no tools, so the expected `escalate`/`risk` can be fixed values. + +## Validation + +Validation uses the same `python jsonschema` pattern that +`scripts/initiative-planner/plan.schema.json` uses: + +```sh +pip install 'jsonschema>=4' +python3 skills/evals/validate-cases.py skills/evals/triage/cases.jsonl +``` + +The validator checks every line against `case.schema.json` and enforces the one +cross-line invariant the set depends on: **case ids must be unique**. It runs in +CI via `tests/test_evals_cases.bats`. + +## Separately owned — reward-hacking guard + +This directory is **separately owned** via `.github/CODEOWNERS`. That is a +deliberate guard: a future automated skill *proposer* may only ever edit the +skill markdown (e.g. `prompts/triage.md`), **never** this held-out set. If the +proposer could edit the cases, it could optimize the skill against a moving +target — reward hacking. Keeping the cases CODEOWNER-gated, apart from any +scorer or proposer, is what makes a measurement against them trustworthy. + +Any change to `skills/evals/**` therefore requires review from the designated +code owners. diff --git a/skills/evals/case.schema.json b/skills/evals/case.schema.json new file mode 100644 index 000000000..98e316eee --- /dev/null +++ b/skills/evals/case.schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Triage Eval Case", + "description": "One held-out eval case: a single triage input paired with its expected decision. Each line of skills/evals//cases.jsonl is one object of this shape. This is the trust anchor for skill-quality measurement and is CODEOWNER-gated separately from any scorer/proposer.", + "type": "object", + "additionalProperties": false, + "required": ["id", "input", "expected"], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$", + "minLength": 3, + "description": "Stable, unique kebab-case identifier for the case (e.g. 'triage-auth-token')." + }, + "description": { + "type": "string", + "minLength": 5, + "description": "Human-readable note on what this case exercises and which triage.md criterion grounds the expected decision." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 2 + }, + "uniqueItems": true, + "description": "Optional labels for coverage tracking, e.g. the high-risk trigger a case exercises ('auth-secrets', 'db-migration', 'security-anti-pattern')." + }, + "input": { + "type": "string", + "minLength": 1, + "description": "The pre-fetched PR context exactly as triage.md consumes it under '## Pre-fetched PR context'. A self-contained string; triage has no tools and reads nothing else." + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["escalate", "risk"], + "description": "The fixed, reward-hack-resistant expected decision fields triage.md must emit.", + "properties": { + "escalate": { + "type": "boolean", + "description": "Whether triage must escalate (true) or may approve (false)." + }, + "risk": { + "type": "string", + "enum": ["LOW", "MEDIUM", "HIGH"], + "description": "Expected risk classification. Any HIGH signal must pair with escalate=true." + } + } + } + } +} diff --git a/skills/evals/triage/cases.jsonl b/skills/evals/triage/cases.jsonl new file mode 100644 index 000000000..458b1b942 --- /dev/null +++ b/skills/evals/triage/cases.jsonl @@ -0,0 +1,8 @@ +{"id": "triage-docs-typo", "description": "Docs-only typo fix; touches no high-risk area (criterion 1) and has no anti-patterns (criterion 3) -> approve.", "input": "Title: Fix typo in README quick-start\n\nFiles changed (1):\n README.md (+1/-1)\n\nDiff:\n--- a/README.md\n+++ b/README.md\n@@\n-Run the the installer.\n+Run the installer.\n\nLinked issue: none. No unresolved review threads. CI: all green.", "expected": {"escalate": false, "risk": "LOW"}} +{"id": "triage-test-only", "description": "Test-only addition that addresses its linked issue; no high-risk files (criterion 1), all gates pass -> approve.", "input": "Title: Add unit tests for date formatter\n\nFiles changed (1):\n src/utils/format_date.test.ts (+42/-0)\n\nDiff adds test cases only; no production code is touched.\n\nLinked issue: #210 (add coverage for the formatter); the diff addresses it. No unresolved review threads. CI: all green.", "expected": {"escalate": false, "risk": "LOW"}} +{"id": "triage-clean-logic-fix", "description": "Non-trivial but clean logic fix; no high-risk area, no anti-pattern, linked issue addressed, all gates pass -> approve at MEDIUM.", "input": "Title: Cap retry backoff at 30s\n\nFiles changed (1):\n src/net/retry.ts (+6/-2)\n\nDiff clamps exponential backoff to a 30s ceiling. No auth, secrets, DB, or workflow files touched and no security anti-patterns. Linked issue #233 (runaway backoff) is addressed. Clear title and scope. No unresolved review threads. CI green.", "expected": {"escalate": false, "risk": "MEDIUM"}} +{"id": "triage-auth-token", "description": "Touches authentication/session tokens under src/auth/** (criterion 1, high-risk area) -> always escalate, HIGH.", "tags": ["auth-secrets"], "input": "Title: Rotate session tokens on privilege change\n\nFiles changed (2):\n src/auth/session.ts (+58/-12)\n src/auth/tokens.ts (+20/-4)\n\nDiff reworks session-token issuance and rotation in the authentication layer; both files live under src/auth/. Linked issue #240. CI green.", "expected": {"escalate": true, "risk": "HIGH"}} +{"id": "triage-db-migration", "description": "Adds a SQL migration under migrations/ altering the users schema (criterion 1, high-risk area) -> always escalate, HIGH.", "tags": ["db-migration"], "input": "Title: Add nullable phone column to users\n\nFiles changed (2):\n migrations/0007_add_user_phone.sql (+9/-0)\n src/models/user.ts (+3/-0)\n\nDiff adds a new SQL migration under migrations/ that alters the users table schema. Linked issue #251. CI green.", "expected": {"escalate": true, "risk": "HIGH"}} +{"id": "triage-sql-concat", "description": "Builds a SQL query by string concatenation of input (criterion 3, security anti-pattern) -> escalate, HIGH.", "tags": ["security-anti-pattern"], "input": "Title: Add user search endpoint\n\nFiles changed (1):\n src/api/search.py (+18/-0)\n\nDiff:\n+def search(user_id):\n+ q = 'SELECT * FROM users WHERE id = ' + user_id\n+ return db.execute(q)\n\nNo auth or migration files are touched, but the query is assembled by string concatenation of the input. Linked issue #260.", "expected": {"escalate": true, "risk": "HIGH"}} +{"id": "triage-workflow-pr-target", "description": "GitHub Actions workflow using pull_request_target with a secret and PR-head checkout (criterion 1, workflow/secrets) -> escalate, HIGH.", "tags": ["workflow-secrets"], "input": "Title: Add PR labeler workflow\n\nFiles changed (1):\n .github/workflows/labeler.yml (+30/-0)\n\nDiff adds a GitHub Actions workflow triggered on pull_request_target that checks out the PR head and uses a repository secret. Linked issue #266.", "expected": {"escalate": true, "risk": "HIGH"}} +{"id": "triage-unresolved-thread", "description": "No high-risk area or anti-pattern, but an unresolved review thread requests changes (criterion 2) -> escalate at MEDIUM.", "tags": ["unresolved-threads"], "input": "Title: Refactor pagination helper\n\nFiles changed (1):\n src/api/paginate.ts (+24/-18)\n\nNo high-risk areas are touched and there are no security anti-patterns. However an unresolved review thread requests changes to an off-by-one in the page-size clamp, not yet addressed. Linked issue #270.", "expected": {"escalate": true, "risk": "MEDIUM"}} diff --git a/skills/evals/validate-cases.py b/skills/evals/validate-cases.py new file mode 100644 index 000000000..a43991427 --- /dev/null +++ b/skills/evals/validate-cases.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Validate a held-out eval-case set (JSONL) against case.schema.json. + +Each line of the .jsonl file is one case object. Beyond per-line JSON-Schema +structural checks this enforces the one cross-line invariant the held-out set +depends on: + + - case ids are unique (a duplicate id silently shadows a case) + +This is intentionally a *validator only* — no scorer, no model invocation. The +held-out cases are the trust anchor of skill-quality measurement; this tool +exists so the set can be checked in CI the same way scripts/initiative-planner +checks plan.schema.json (python jsonschema, draft 2020-12). + +Usage: validate-cases.py [schema.json] +Exit 0 on success; non-zero with a message on failure. +""" +import json +import sys +from pathlib import Path +from typing import NoReturn + + +def fail(msg: str) -> NoReturn: + # stdout (not stderr) so the reason is visible in workflow logs and capturable + # by tests; the ::error:: prefix still renders as a GitHub annotation. + print(f"::error::eval cases invalid: {msg}") + sys.exit(1) + + +def main() -> None: + if len(sys.argv) < 2: + fail("usage: validate-cases.py [schema.json]") + cases_path = Path(sys.argv[1]) + schema_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(__file__).with_name("case.schema.json") + + try: + import jsonschema + except ImportError: + fail("jsonschema not installed (pip install 'jsonschema>=4')") + + try: + schema = json.loads(schema_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + fail(f"could not read/parse schema {schema_path}: {exc}") + + try: + raw = cases_path.read_text() + except OSError as exc: + fail(f"could not read {cases_path}: {exc}") + + validator = jsonschema.Draft202012Validator(schema) + ids: dict[str, int] = {} + count = 0 + for lineno, line in enumerate(raw.splitlines(), start=1): + if not line.strip(): + continue # tolerate blank separator lines + try: + case = json.loads(line) + except json.JSONDecodeError as exc: + fail(f"line {lineno}: not valid JSON: {exc}") + error = next(validator.iter_errors(case), None) + if error is not None: + loc = "/".join(str(p) for p in error.absolute_path) or "" + fail(f"line {lineno}: schema violation at {loc}: {error.message}") + case_id = case["id"] + if case_id in ids: + fail(f"line {lineno}: duplicate case id '{case_id}' (first seen on line {ids[case_id]})") + ids[case_id] = lineno + count += 1 + + if count == 0: + fail(f"{cases_path} contains no cases") + + print(f"cases OK: {count} case(s), unique ids, schema-valid.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_evals_cases.bats b/tests/test_evals_cases.bats new file mode 100644 index 000000000..d789a2e90 --- /dev/null +++ b/tests/test_evals_cases.bats @@ -0,0 +1,81 @@ +#!/usr/bin/env bats +# Tests for the held-out triage eval-case set (skills/evals/). +# validate-cases.py performs JSON-Schema validation only (no scorer, no model +# invocation) so the whole suite runs offline. + +setup() { + ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd)" + EVALS_DIR="$ROOT/skills/evals" + VALIDATOR="$EVALS_DIR/validate-cases.py" + SCHEMA="$EVALS_DIR/case.schema.json" + CASES="$EVALS_DIR/triage/cases.jsonl" + TMP="$(mktemp -d)" +} + +teardown() { rm -rf "$TMP"; } + +@test "validate-cases accepts the seed triage case set" { + run python3 "$VALIDATOR" "$CASES" + [ "$status" -eq 0 ] + [[ "$output" == *"cases OK"* ]] +} + +@test "validate-cases rejects a case missing expected.risk" { + jq -c 'del(.expected.risk)' <(head -1 "$CASES") >"$TMP/bad.jsonl" + run python3 "$VALIDATOR" "$TMP/bad.jsonl" + [ "$status" -ne 0 ] + [[ "$output" == *"risk"* ]] +} + +@test "validate-cases rejects an unknown top-level property" { + jq -c '.surprise = "x"' <(head -1 "$CASES") >"$TMP/bad.jsonl" + run python3 "$VALIDATOR" "$TMP/bad.jsonl" + [ "$status" -ne 0 ] + [[ "$output" == *"additional"* || "$output" == *"surprise"* ]] +} + +@test "validate-cases rejects an invalid risk enum value" { + jq -c '.expected.risk = "CATASTROPHIC"' <(head -1 "$CASES") >"$TMP/bad.jsonl" + run python3 "$VALIDATOR" "$TMP/bad.jsonl" + [ "$status" -ne 0 ] +} + +@test "validate-cases rejects a non-boolean escalate" { + jq -c '.expected.escalate = "yes"' <(head -1 "$CASES") >"$TMP/bad.jsonl" + run python3 "$VALIDATOR" "$TMP/bad.jsonl" + [ "$status" -ne 0 ] +} + +@test "validate-cases rejects duplicate case ids" { + head -1 "$CASES" >"$TMP/dup.jsonl" + head -1 "$CASES" >>"$TMP/dup.jsonl" + run python3 "$VALIDATOR" "$TMP/dup.jsonl" + [ "$status" -ne 0 ] + [[ "$output" == *"unique"* || "$output" == *"duplicate"* ]] +} + +@test "case schema validates with the same python jsonschema pattern as plan.schema.json" { + run python3 -c "import json,jsonschema,sys; jsonschema.Draft202012Validator.check_schema(json.load(open('$SCHEMA')))" + [ "$status" -eq 0 ] +} + +@test "seed cases cover both escalate outcomes" { + has_true="$(jq -s 'any(.[]; .expected.escalate == true)' "$CASES")" + has_false="$(jq -s 'any(.[]; .expected.escalate == false)' "$CASES")" + [ "$has_true" = "true" ] + [ "$has_false" = "true" ] +} + +@test "seed cases include a case for each named high-risk trigger" { + tags="$(jq -r 'select(.tags != null) | .tags[]' "$CASES" | sort -u)" + grep -qx 'auth-secrets' <<<"$tags" + grep -qx 'db-migration' <<<"$tags" + grep -qx 'security-anti-pattern' <<<"$tags" +} + +@test "every high-risk (HIGH) seed case escalates" { + # A HIGH classification must never be paired with escalate=false: triage.md + # treats any HIGH signal as an always-escalate condition. + bad="$(jq -s '[.[] | select(.expected.risk == "HIGH" and .expected.escalate == false)] | length' "$CASES")" + [ "$bad" -eq 0 ] +} From d2b56ba8b1d1bb305d6d975f0995c7485c07c8d7 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Fri, 12 Jun 2026 01:49:28 +0000 Subject: [PATCH 2/3] chore: apply manual instructions [skip ci-relay] --- skills/evals/case.schema.json | 10 ++++++++++ skills/evals/validate-cases.py | 4 ++-- tests/test_evals_cases.bats | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/skills/evals/case.schema.json b/skills/evals/case.schema.json index 98e316eee..188bc4508 100644 --- a/skills/evals/case.schema.json +++ b/skills/evals/case.schema.json @@ -46,6 +46,16 @@ "enum": ["LOW", "MEDIUM", "HIGH"], "description": "Expected risk classification. Any HIGH signal must pair with escalate=true." } + }, + "if": { + "properties": { + "risk": { "const": "HIGH" } + } + }, + "then": { + "properties": { + "escalate": { "const": true } + } } } } diff --git a/skills/evals/validate-cases.py b/skills/evals/validate-cases.py index a43991427..f42c59418 100644 --- a/skills/evals/validate-cases.py +++ b/skills/evals/validate-cases.py @@ -40,12 +40,12 @@ def main() -> None: fail("jsonschema not installed (pip install 'jsonschema>=4')") try: - schema = json.loads(schema_path.read_text()) + schema = json.loads(schema_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: fail(f"could not read/parse schema {schema_path}: {exc}") try: - raw = cases_path.read_text() + raw = cases_path.read_text(encoding="utf-8") except OSError as exc: fail(f"could not read {cases_path}: {exc}") diff --git a/tests/test_evals_cases.bats b/tests/test_evals_cases.bats index d789a2e90..c85a6de90 100644 --- a/tests/test_evals_cases.bats +++ b/tests/test_evals_cases.bats @@ -79,3 +79,9 @@ teardown() { rm -rf "$TMP"; } bad="$(jq -s '[.[] | select(.expected.risk == "HIGH" and .expected.escalate == false)] | length' "$CASES")" [ "$bad" -eq 0 ] } + +@test "validate-cases rejects a case with HIGH risk but escalate false" { + jq -c '.expected.risk = "HIGH" | .expected.escalate = false' <(head -1 "$CASES") >"$TMP/bad.jsonl" + run python3 "$VALIDATOR" "$TMP/bad.jsonl" + [ "$status" -ne 0 ] +} From 2ea5b778a59a045f4b7f415c09db78cbd0357d98 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 02:57:48 +0000 Subject: [PATCH 3/3] refactor(evals): relocate held-out eval set to repo-root evals/ per maintainer decision A1 Epic #581 open-question A1 ruled the held-out eval cases live at evals//cases.jsonl at the repo root, not under a new skills/ tree (skills live in prompts/ and frameworks/). This story (#582) had landed the set under skills/evals/; move it to the decided location and update every reference. - git mv skills/evals -> evals (schema, validator, README, triage cases) - .github/CODEOWNERS: own /evals/ instead of /skills/evals/ - .github/workflows/lint.yml: trigger path skills/** -> evals/** - tests/test_evals_cases.bats: EVALS_DIR -> $ROOT/evals - README + schema description: skills/evals -> evals validate-cases.py resolves its schema via __file__, so the move needs no code change. All 11 bats tests pass. https://claude.ai/code/session_01GtKqCFHPEShgswWynw5bbD --- .github/CODEOWNERS | 4 ++-- .github/workflows/lint.yml | 2 +- {skills/evals => evals}/README.md | 6 +++--- {skills/evals => evals}/case.schema.json | 2 +- {skills/evals => evals}/triage/cases.jsonl | 0 {skills/evals => evals}/validate-cases.py | 0 tests/test_evals_cases.bats | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) rename {skills/evals => evals}/README.md (95%) rename {skills/evals => evals}/case.schema.json (90%) rename {skills/evals => evals}/triage/cases.jsonl (100%) rename {skills/evals => evals}/validate-cases.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9bbf44ce9..c16a9cef7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,5 +3,5 @@ # Held-out eval cases are the trust anchor for skill-quality measurement. # They MUST stay separately owned: a future automated skill proposer may only -# edit the skill markdown, never this test set (reward-hacking guard, #582). -/skills/evals/ @petry-projects/org-leads +# edit the skill markdown, never this held-out set (reward-hacking guard, #582). +/evals/ @petry-projects/org-leads diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f44af8cdd..3841432a6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ on: - '.github/workflows/**' - 'tests/**' - 'agents/**' - - 'skills/**' + - 'evals/**' - '.gitleaks.toml' workflow_dispatch: diff --git a/skills/evals/README.md b/evals/README.md similarity index 95% rename from skills/evals/README.md rename to evals/README.md index 7b68f6ba8..e7b6968da 100644 --- a/skills/evals/README.md +++ b/evals/README.md @@ -9,7 +9,7 @@ invocation** in this directory. ## Layout ``` -skills/evals/ +evals/ ├── case.schema.json # JSON-Schema (draft 2020-12) for one eval case ├── validate-cases.py # validate a .jsonl set against the schema └── /cases.jsonl # one case per line for that skill (e.g. triage/) @@ -44,7 +44,7 @@ Validation uses the same `python jsonschema` pattern that ```sh pip install 'jsonschema>=4' -python3 skills/evals/validate-cases.py skills/evals/triage/cases.jsonl +python3 evals/validate-cases.py evals/triage/cases.jsonl ``` The validator checks every line against `case.schema.json` and enforces the one @@ -60,5 +60,5 @@ proposer could edit the cases, it could optimize the skill against a moving target — reward hacking. Keeping the cases CODEOWNER-gated, apart from any scorer or proposer, is what makes a measurement against them trustworthy. -Any change to `skills/evals/**` therefore requires review from the designated +Any change to `evals/**` therefore requires review from the designated code owners. diff --git a/skills/evals/case.schema.json b/evals/case.schema.json similarity index 90% rename from skills/evals/case.schema.json rename to evals/case.schema.json index 188bc4508..3ef27fd5a 100644 --- a/skills/evals/case.schema.json +++ b/evals/case.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Triage Eval Case", - "description": "One held-out eval case: a single triage input paired with its expected decision. Each line of skills/evals//cases.jsonl is one object of this shape. This is the trust anchor for skill-quality measurement and is CODEOWNER-gated separately from any scorer/proposer.", + "description": "One held-out eval case: a single triage input paired with its expected decision. Each line of evals//cases.jsonl is one object of this shape. This is the trust anchor for skill-quality measurement and is CODEOWNER-gated separately from any scorer/proposer.", "type": "object", "additionalProperties": false, "required": ["id", "input", "expected"], diff --git a/skills/evals/triage/cases.jsonl b/evals/triage/cases.jsonl similarity index 100% rename from skills/evals/triage/cases.jsonl rename to evals/triage/cases.jsonl diff --git a/skills/evals/validate-cases.py b/evals/validate-cases.py similarity index 100% rename from skills/evals/validate-cases.py rename to evals/validate-cases.py diff --git a/tests/test_evals_cases.bats b/tests/test_evals_cases.bats index c85a6de90..0a7e7f7c7 100644 --- a/tests/test_evals_cases.bats +++ b/tests/test_evals_cases.bats @@ -1,11 +1,11 @@ #!/usr/bin/env bats -# Tests for the held-out triage eval-case set (skills/evals/). +# Tests for the held-out triage eval-case set (evals/). # validate-cases.py performs JSON-Schema validation only (no scorer, no model # invocation) so the whole suite runs offline. setup() { ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd)" - EVALS_DIR="$ROOT/skills/evals" + EVALS_DIR="$ROOT/evals" VALIDATOR="$EVALS_DIR/validate-cases.py" SCHEMA="$EVALS_DIR/case.schema.json" CASES="$EVALS_DIR/triage/cases.jsonl"