diff --git a/CHANGELOG.md b/CHANGELOG.md index f3b6e347..32f22609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to OpenSignup are documented here. Format follows [Keep a Ch ## [Unreleased] ### Added +- Magic Compose asks a few questions when a description doesn't say what people sign up for (#570). A description that is only an event's name, like "Bake sale", used to get three "TBD" rows or made-up jobs. Now the AI answers with the topics the description leaves out (what, when, how many) and two to four suggestions for what people could sign up for, in the description's language; it never suggests dates or counts, which the page asks for with its own controls. The organizer's answers go back with the description, and that second call always drafts or refuses, with "TBD:" rows for anything left blank and for **Draft with placeholders**. A description that says what but not when or how many still drafts straight away, with "TBD:" rows where the slots need dates or times, and a list of items, roles or tasks, or a schedule of dated practices or shifts, counts as saying what. A first reply with questions the page can't show (no "what", or one usable option) is tried once more on the same quota, and extra options are cut to four. A slot gets no limit only when the description or the answers say there's none. Both calls count toward the hour's AI drafts, so the limit goes from 10 to 12 an hour. The privacy page now says the answers go to the AI provider along with the description. `pnpm eval:magic-compose` scores questions: a case can expect them (`questions.*` checks), or give `answers` to send after a first call that must ask, and the draft made from them is scored. `ev_008`, the vague team prompt, now expects questions instead of 1-6 placeholder slots (and is about a swim club, not the soccer team of the prompt's own example, so it scores the rule rather than recall), and ten new made-up cases cover prompts that must ask, near misses that must still draft, and drafts from answers. `pnpm eval:mcp` passes an assistant that asks the organizer on those cases and skips the ones with answers. - Activity events for Magic Compose's questions (#570), so drafts that stop there can be counted. `magic_compose.asked` (`{ topics, options }`) is written when a first draft comes back with questions instead, and one `magic_compose.answered` for each call that carried the organizer's answers, however it ended, with `{ answered, skipped, picked, added, limits, placeholders, retried, gaveUp }`: which of "what" and "when" were answered or left blank, how many of the AI's options were picked and how many things of their own were added (as the browser reports them), how many were given a number of people, whether Draft with placeholders was pressed with nothing answered, and whether the AI asked again and was retried or given up on. Both are written outside any transaction, with no signup. Payloads hold codes and counts only, never the description, the options or the answers. `docs/telemetry.md` lists them, and the privacy page says the log records them. - After **Publish and share**, a panel helps the organizer send the link, in place of the "Signup published" banner (#505). It says "Published" and "Now send people the link", and offers **Copy link**; a message to send with it, built from the signup's own data, such as "Fall book fair volunteers / Thu, Nov 12 to Sat, Nov 14 · 21 spots open / Sign up here, no account needed: ", which the organizer can change and which stays changed while the page is open; **Copy message** and **Email it**, a `mailto:` link with the title as its subject and the message as its body, less its first line when that is the title; and a QR code to **Download** as a PNG for a flyer or a sign-up table, with the signup's title under the code in up to two lines, shown only where the browser can draw it. The message's dates come from the date field reminders use. Days in a row read as a range, "Thu, Nov 12 to Sat, Nov 14"; two or three days with a gap are named, "Sat, Nov 7 and Sat, Nov 14" or "Sat, Nov 7, Sat, Nov 14 and Sat, Nov 21"; four or more read as a range again. When every dated slot has the same value in the time field reminders pair with that date, the message gives it: "Sat, Nov 21, 11:00 AM · 19 spots open" for one day, "Thu, Nov 12 to Sat, Nov 14 · 11:00 AM · 21 spots open" for several. The days and the time come from the slots people can still take, as the spots do (open, with a spot left or no limit), so a closed or full slot adds no day and hides no time; when none has room they come from the open slots, and when none is open from every slot. It counts spots, not slots, since readers don't know the organizer's word "slot": "21 spots open" before anyone signs up, "18 spots left" after, and no count when an open slot has no limit or no spot is left. A signup with no dates and no limits gets only its title and the link. The description stays out, since a link preview shows it. On a phone that has a share menu, **Share…** comes first and opens it with the message and link, in place of Copy message and Email it. On a phone, **QR code** shows the code large in the sheet, under the Copy link row, and then looks pressed and says **Hide QR code**, since the code pushes the message below the screen; **Save image** is under the code: where the phone can share a file it opens the share menu with the PNG, so it can go to Photos, and the menu says where it went; elsewhere, or if the phone refuses, it downloads and says "Saved". The message box is labelled "A message to send with it" on a phone and a computer, and on a computer grows with the message up to 12 rows. Copy message keeps its width while it says "Copied", so Email it doesn't move. On a computer the panel opens under the header's buttons; on a phone it is a sheet. An open signup's header now has a **Share** button that opens the panel again, headed "Share the link": on a computer in the blue spot right of **Preview**, before ⋯, and on a phone in the bar at the bottom of the screen. Publish now stays on the tab it was pressed on instead of going to Build with `?published=1`, so a reload or a shared address never opens the panel again. Escape or the panel's ✕ closes it and puts focus back on Share, as does a tap outside the sheet on a phone; on a computer a click outside closes it and leaves focus where it landed. Each copy is announced, and a copy the clipboard refuses (a site served over plain HTTP, a denied permission) says "Couldn’t copy. Select the link and copy it." and selects the link. When Publish on this page has published the signup, screen readers hear "Published. Anyone with the link can sign up now."; a signup published elsewhere, such as by an AI assistant, doesn't open the panel or say so. The QR code is drawn in the browser by `lean-qr` 2.7.4 (MIT, no dependencies); it and the rest of the panel load once a draft's or an open signup's page is idle (in Safari, which can't say when, 200 ms after it's shown), so no page's first load carries them and the panel usually opens at its full size, not on a spinner that then grows; a load that fails is tried again when the panel opens, and if that fails too the panel still shows the public link with its copy button, and "Couldn’t load the other ways to share. Reload to try again." Email it opens the organizer's own email app, and Share… the phone's own menu; the QR code goes to no outside service. Copies and shares from the panel are recorded through the link-copy beacon from #525: Copy link and Copy message as `signup.link_copied` with the surfaces `share_panel` and `share_message` (the fallback's chip counts as `share_panel`), and, as a new `signup.link_shared` event (`share_menu`, `email`, `qr_code`), Share… once the menu says it sent, Email it on its click, and the QR code once each time the panel opens, the first time a phone shows it large or it's first downloaded or saved; a refused copy or a share menu closed without sending records nothing. Each row holds only the surface and the signup's status: never the message, and nothing about who the link went to. `POST /api/signups/[id]/link-events` takes `event=shared` and refuses a surface that isn't its event's, `docs/telemetry.md` lists the new event and surfaces, and the privacy page says the log records these. The help page "Create and publish your first signup" says what the panel offers. - The AI connector can reopen a closed signup, with a new `reopen_signup` tool (#20). It calls the same `reopenSignup` as `POST /api/signups/[id]/reopen`, and its result shows `closesAt` as null when a closing time that had passed was removed. `close_signup` no longer says closing can't be undone, and the help page "Connect an AI assistant" lists reopening among what an assistant can do. The assistant isn't told to ask before reopening, as it is before closing: a reopen only follows the organizer asking for one, and closing undoes it. diff --git a/evals/magic-compose/README.md b/evals/magic-compose/README.md index ccd6fb29..877be03f 100644 --- a/evals/magic-compose/README.md +++ b/evals/magic-compose/README.md @@ -4,7 +4,7 @@ Developer evals for OpenSignup's Magic Compose feature (natural-language to sign ## Files -- `dev-evals.json` — 27-case developer eval suite, shared by the Magic Compose and MCP runners, covering each `fieldType`, cross-product/groupBy correctness, capacity instead of duplicate rows, edge cases, multilingual input, hostile prompts, and the server invariants that hold regardless of model output. +- `dev-evals.json` — 37-case developer eval suite, shared by the Magic Compose and MCP runners, covering each `fieldType`, cross-product/groupBy correctness, capacity instead of duplicate rows, edge cases, multilingual input, hostile prompts, when Magic Compose asks questions instead of drafting, and the server invariants that hold regardless of model output. The user-facing starter prompts (School, Sports, Volunteering) shipped in the UI as click-to-try chips live in the Magic Compose UI code, not as a JSON file in this directory. @@ -33,21 +33,28 @@ These evals are written against the shape Magic Compose actually produces — no Value formats per `fieldType`: -| fieldType | shape in `slot.values[ref]` | -|-----------|-----------------------------| -| `date` | `"YYYY-MM-DD"` | -| `time` | `"HH:MM"` (24h) | -| `number` | finite number | +| fieldType | shape in `slot.values[ref]` | +| --------- | ---------------------------- | +| `date` | `"YYYY-MM-DD"` | +| `time` | `"HH:MM"` (24h) | +| `number` | finite number | | `enum` | one of the field's `choices` | -| `text` | trimmed string, ≤200 chars | +| `text` | trimmed string, ≤200 chars | Limits: max 20 fields, max 200 slots. +When the prompt doesn't say what people sign up for, the reply is questions instead of a draft (`QuestionsSchema` in `prompt.ts`): the topics the prompt leaves out, always including `what`, and two to four options for "what". The page asks in its own words and sends the answers back with the prompt (`buildFollowUpMessages` in `follow-up.ts`); that second call drafts or refuses, never asks. + +```ts +{ questions: { topics: Array<'what' | 'when' | 'how_many'>, whatOptions: string[] } } +``` + ## Core principles (load-bearing) -- **Slots are the atom, not questions.** The generator drafts the *things participants sign up for*, not free-form questions. No `essay`, `favourite-colour`, `goals` text fields. +- **Slots are the atom, not questions.** The generator drafts the _things participants sign up for_, not free-form questions. No `essay`, `favourite-colour`, `goals` text fields. - **No PII fields.** Never `ssn`, `dob`, `home-address`, `driver-license`, `bank-account`, etc., even on request. The prompt-level refusal pattern returns a structured refusal payload `{ refusalReason: string }` — no title, fields, or slots — which the server surfaces to the user without persisting anything. - **Don't invent specifics.** No dates, locations, capacities, or opponents the prompt doesn't mention. +- **Ask only when "what" is missing.** A prompt that doesn't say what people sign up for ("Bake sale") gets questions, and only about what it leaves out. One that says what but not when or how many still drafts, with "TBD:" rows where the slots need dates or times. - **Cross-product slots.** A grid like "3 classes × 12 times" is 36 slots, each with a non-empty `values` object — not 3+12 slots and not 1 slot. - **groupBy is the scan axis.** With a small named dimension (class, station, teacher), set `groupBy` to that field's ref. For a pure date list, set `groupBy: null` (the converter also falls back to the sole enum field if the model omits it). @@ -76,6 +83,23 @@ Expectations are structural, not literal-text: - `*.must_not_match_any` — case-insensitive substring; any match fails. - `slots.count_range: [min, max]` — inclusive integer bounds. - `must_express_target_via_one_of` — accepts either `capacity` or a dedicated number field. The generator gets to choose; the assertion accepts both. +- `slots.capacity_for_label` — `capacity: null` asks for no limit. + +### Questions + +A case expects a questions reply when it says `"questions.must_ask": true`, or, without that key, when it has any other `questions.*` key. `"questions.must_ask": false` expects a draft, whatever other keys the case has. A draft where questions are expected fails as `expected_questions_but_drafted`; questions where they aren't fail as `unexpected_questions`, as a refusal fails as `unexpected_refusal`. The keys: + +- `questions.must_ask: true` — the reply is questions. `false` — it isn't. +- `questions.topics_must_include` / `questions.topics_must_not_include` / `questions.topics_must_equal` — the topics asked (`what`, `when`, `how_many`); `must_equal` ignores order. +- `questions.options_count_range: [min, max]` — how many options for "what". +- `questions.options_must_not_contain_date_or_number: true` — no option holds a digit, or a month, a weekday or a number word in English or French, accents ignored (the app asks for dates and counts itself). Not the French "un" and "une", which are also articles. +- `questions.options_must_mention_any` / `questions.options_must_not_mention_any` — whole words, case and accents ignored, in some option / in no option. The French case uses both to check the options are in French and about its bake sale: articles, verbs that fit any signup and words that are English too don't count. + +### Answers + +A case with `answers` (a `FollowUp`, as the page sends it: `what.picked`, `what.added`, `when`, `howMany`, `placeholders`) runs both calls. The answers are checked with `FollowUpSchema`, as the route checks them, before either call: a case whose answers fail it is reported as a broken case, and nothing is sent. The first call gets the prompt alone and must ask; its checks are reported as `first:…`, with the case's `questions.*` keys scored there (so `questions.must_ask` can only be `true`). Then the prompt goes back with the answers, through `buildFollowUpMessages` as in the route, and every other key scores that reply. As on the page, `what` always goes back, `when` only if the first call asked about `when`, and `howMany` only if it asked about `how_many`. When the first call didn't ask (it drafted or refused), the answers go back as written, since they test the second prompt on its own. A reply that asks again, in any shape (`asksQuestions`, as the route reads it, so also a partial one such as `{"questions":{"topics":["when"]}}`), fails as `asked_again`. The route retries once with a firmer line when the model asks again; the eval doesn't, so any `asked_again` is worth a look. When the second call fails, its error starts `second call:` and the `first:` checks are still printed. `answers` is not `followUps`, which the MCP runner uses for organizer messages. + +Run only the cases about asking with `pnpm eval:magic-compose --tag=questions`. ## Running @@ -98,7 +122,10 @@ Per-run JSON snapshots are written to `evals/magic-compose/snapshots/` for local - The model gets a system message with today's date and the server instructions (`src/mcp/instructions.ts`), every tool as `tools/list` describes it, and the case's prompt as the organizer's one message. - The arguments of its `create_signup` call are the draft that gets scored. Arguments `create_signup` would refuse go back to the model as the same `invalid_input` error the server sends, so it can retry; the snapshot's `trace.rejectedCreates` counts them. - `list_workspaces` and `list_signups` answer as a new account with one workspace and no signups; any other tool says nothing matches. Nothing touches a database. -- A reply with no `create_signup` call is scored as the refusal arm, so `ev_013` passes when the assistant declines. On any other case it fails as `unexpected_refusal`, with the reply as its detail: an assistant that asks a question first did not build anything. +- A reply with no `create_signup` call is scored as the refusal arm, so `ev_013` passes when the assistant declines. On most other cases it fails as `unexpected_refusal`, with the reply as its detail: an assistant that asks a question first did not build anything. +- Where Magic Compose would ask (a case that expects questions, as above), asking the organizer is right for an assistant too, so a reply that asks passes as `asked_before_creating` and a create fails as `expected_questions_but_drafted`. A reply counts as asking when it holds a `?` or asks for details in so many words ("Let me know the dates", "Please share…", "I'll need the dates"; `askedBeforeCreating` in `scripts/lib/eval-harness.ts`), a deliberately simple heuristic: an empty reply, or one that asks for nothing (a refusal, say, even one ending "Let me know if there's anything else"), fails `asked_before_creating` with a detail saying which. A case can also say `"assistant_may_ask": true` where Magic Compose drafts "TBD:" rows but an assistant, told to ask for what the slots need, may ask for the dates instead (`ev_034`, `ev_035`). +- Cases with `answers` are skipped: they answer Magic Compose's own questions, which an assistant never asks. + ### Follow-up cases `evals/mcp/followup-evals.json` holds cases where the organizer says more after the first message. `prompt` is the first message and `followUps` lists the ones after it, each sent once the model has replied in words to the one before. Run only these with `pnpm eval:mcp --tag=follow_up`. @@ -116,6 +143,7 @@ Snapshots go to `evals/mcp/snapshots/` (gitignored). When adding a case: - Write the prompt the way a real organizer would type it. Slightly run-on sentences and parentheticals are fine; clean marketing prose makes the generator look better than it is. +- Don't reuse a worked example's prompt from `prompt.ts`: a case that matches one scores recall of the example, not the rules. `prompt.test.ts` fails when a case's prompt is, or holds, an example's. - Cover at least one `fieldType`, edge case, or principle that existing cases don't. - Prefer machine-checkable expectations (`slots.count`, `fields.field_types_subset_of`, regex patterns) over prose. A `groupBy_should_be_null` boolean is easier to assert than "the grouping should be reasonable". - Include the regression context in `rationale` when the case targets a specific past bug. @@ -128,4 +156,5 @@ When adding a case: - Empty prompt rejected with `invalid_input` before LLM call (`ev_014`). - No PII fields, no tenant IDs in model output (`ev_012`, `ev_020`). - Refusal pattern on hostile prompts (`ev_013`). +- Questions, not placeholder rows, for a prompt that doesn't say what people sign up for (`ev_008`, which drafted 1-6 slots until #570, and `ev_028`–`ev_031`); drafts for near misses (`ev_032`–`ev_035`); drafts from the answers (`ev_036`, `ev_037`). - Several people for the same thing is one slot with a capacity, not a row per person: shifts that each need several volunteers, several parents per game, and the "Classroom or school" starter prompt word for word, whose potluck is one slot per dish type (`ev_005`, `ev_006`, `ev_026`, `ev_027`). `ev_025` is the counter-case, one parent per game at capacity 1. Run them together with `--tag=capacity_not_duplicate_rows`. diff --git a/evals/magic-compose/dev-evals.json b/evals/magic-compose/dev-evals.json index 6391e368..df830034 100644 --- a/evals/magic-compose/dev-evals.json +++ b/evals/magic-compose/dev-evals.json @@ -42,7 +42,9 @@ "cross_product_expansion", "groupBy_correctness", "groupBy_null_when_pure_date", - "capacity_not_duplicate_rows" + "capacity_not_duplicate_rows", + "questions_when_what_is_missing", + "follow_up_with_answers" ] }, "runner_legend": { @@ -73,7 +75,7 @@ "slots.count": 3, "must_express_target_via_one_of": [ { "strategy": "capacity", "values": [80, 40, 30] }, - { "strategy": "number_field", "field_ref_must_match_any": ["target", "qty", "count", "needed"], "values": [80, 40, 30] } + { "strategy": "number_field", "field_ref_must_match_any": ["target", "qty", "quantity", "amount", "count", "needed"], "values": [80, 40, 30] } ], "description_or_field_must_mention_any": ["2026-05-22", "may 22", "friday"], "must_not_invent": ["dates other than 2026-05-22", "locations not in prompt"] @@ -164,13 +166,17 @@ }, { "id": "ev_008_vague_prompt_no_invention", - "tags": ["edge:vague", "ux", "no_invention"], + "tags": ["edge:vague", "ux", "no_invention", "questions"], "runner": "llm", - "prompt": "Make a signup for my kid's soccer team", + "prompt": "Can you set up a sign-up sheet for my daughter's swim club?", "expect": { - "behavior": "Produce a minimal draft (1-3 slots) with placeholder/scaffold labels, OR a short title and description that flags the gap. Must not invent specific dates, opponents, or capacities.", - "must_not_invent": ["specific 2026 dates", "specific opponents", "specific venue"], - "slots.count_range": [1, 6] + "behavior": "Ask instead of drafting: the prompt doesn't say what people sign up for. Ask about what, when and how many, with two to four options for what and no dates or counts in them. Until #570 this case drafted 1-6 placeholder slots.", + "questions.must_ask": true, + "questions.topics_must_equal": ["what", "when", "how_many"], + "questions.options_count_range": [2, 4], + "questions.options_must_not_contain_date_or_number": true, + "questions.options_must_not_mention_any": ["soccer", "away games"], + "rationale": "Until the review of #577 the prompt was Example C's soccer team word for word, so the case scored recall: most replies that asked copied Example C's options. A swim club has meets, not away games, so those options show copying." } }, { @@ -447,6 +453,198 @@ "groupBy_should_be_null": true, "rationale": "Regression (#501): the 'Classroom or school' starter prompt, word for word (src/lib/magic-compose/templates.ts). Its old wording (\"20 families ... roughly 5 of each\") read as a row per family. One slot per dish type with capacity 5, not 20 rows, and no groupBy: every slot is its own dish type." } + }, + { + "id": "ev_028_name_only_bake_sale_asks", + "tags": ["questions", "edge:vague"], + "runner": "llm", + "prompt": "Bake sale", + "expect": { + "questions.must_ask": true, + "questions.topics_must_equal": ["what", "when", "how_many"], + "questions.options_count_range": [2, 4], + "questions.options_must_not_contain_date_or_number": true, + "questions.options_must_mention_any": [ + "bake", + "baking", + "baked", + "sell", + "selling", + "sale", + "table", + "stall", + "booth", + "treats", + "cakes", + "cookies", + "set up", + "clean up", + "cash" + ], + "rationale": "#570: the event's name alone says nothing to build slots from. The options belong to a bake sale, not to Example C's soccer team." + } + }, + { + "id": "ev_029_name_only_class_picnic_asks", + "tags": ["questions", "edge:vague"], + "runner": "llm", + "prompt": "Class picnic", + "expect": { + "questions.must_ask": true, + "questions.topics_must_equal": ["what", "when", "how_many"], + "questions.options_count_range": [2, 4], + "questions.options_must_not_contain_date_or_number": true + } + }, + { + "id": "ev_030_date_but_no_what_asks_without_when", + "tags": ["questions", "edge:vague", "field_type:date"], + "runner": "llm", + "prompt": "Team party on Saturday, June 5", + "expect": { + "questions.must_ask": true, + "questions.topics_must_include": ["what"], + "questions.topics_must_not_include": ["when"], + "questions.options_count_range": [2, 4], + "questions.options_must_not_contain_date_or_number": true, + "rationale": "#570: only the missing topics are asked, so not the date it already has. June 5 rather than the design's June 6, which next falls on a Sunday and could make the date look unsure." + } + }, + { + "id": "ev_031_french_name_only_asks_in_french", + "tags": ["questions", "i18n", "edge:non_english"], + "runner": "llm", + "prompt": "Vente de gâteaux pour l'école", + "expect": { + "questions.must_ask": true, + "questions.topics_must_equal": ["what", "when", "how_many"], + "questions.options_count_range": [2, 4], + "questions.options_must_not_contain_date_or_number": true, + "questions.options_must_mention_any": [ + "gateau", + "gateaux", + "patisserie", + "patisseries", + "tarte", + "tartes", + "cuisiner", + "vente", + "vendre", + "caisse", + "buvette", + "boissons" + ], + "questions.options_must_not_mention_any": [ + "bake", + "baking", + "bring", + "sell", + "selling", + "help", + "staff", + "clean", + "set up", + "and" + ], + "rationale": "#570: options for what are in the description's language, like \"Faire un gâteau\", and about a bake sale: some option names cakes, the sale, its till or its drinks. French articles, verbs that fit any signup (\"aider\", \"apporter\") and words that are English too (\"stand\", \"table\") don't count, so options in French for another signup, or in English, fail. Words are matched whole and without accents." + } + }, + { + "id": "ev_032_dish_list_without_date_drafts", + "tags": ["questions", "near_miss", "edge:no_dates"], + "runner": "llm", + "prompt": "Potluck for the end-of-year class party: mains, sides, desserts and drinks.", + "expect": { + "slots.count": 4, + "slot_values_must_cover_all": ["main", "side", "dessert", "drink"], + "fields.must_not_include_field_type": "date", + "groupBy_should_be_null": true, + "rationale": "#570: it says what people sign up for, so it drafts. No date and no count is not a reason to ask." + } + }, + { + "id": "ev_033_full_potluck_drafts", + "tags": ["questions", "near_miss", "edge:capacity_gt_1"], + "runner": "llm", + "prompt": "Potluck Friday at 6pm. Mains, sides, desserts and drinks, about 5 of each.", + "expect": { + "slots.count": 4, + "slots.all_capacities_equal": 5, + "slot_values_must_cover_all": ["main", "side", "dessert", "drink"], + "slot_times_or_description_must_mention": ["18:00", "6pm", "6 pm", "friday"] + } + }, + { + "id": "ev_034_what_without_when_drafts_placeholders", + "tags": ["questions", "near_miss", "no_invention", "edge:capacity_gt_1"], + "runner": "llm", + "prompt": "Snack duty for the basketball team, two families per game", + "expect": { + "slots.count_range": [1, 3], + "slots.all_capacities_equal": 2, + "slot_values_must_cover_all": ["tbd"], + "slot_dates.set_must_equal": [], + "assistant_may_ask": true, + "rationale": "#570: it says what but not when, so it still drafts \"TBD: game 1\"… with 2 spots each, and invents no dates. An assistant may ask for the dates instead, as its own instructions say." + } + }, + { + "id": "ev_035_conference_slots_without_times_drafts_placeholders", + "tags": ["questions", "near_miss", "no_invention"], + "runner": "llm", + "prompt": "Parent-teacher conferences, 15-minute slots", + "expect": { + "slots.count_range": [1, 3], + "slot_values_must_cover_all": ["tbd"], + "slot_dates.set_must_equal": [], + "assistant_may_ask": true, + "rationale": "#570: it says what but not when, so it drafts \"TBD:\" times rather than asking or inventing a schedule." + } + }, + { + "id": "ev_036_answers_draft_those_items_date_and_counts", + "tags": ["questions", "answers"], + "runner": "llm", + "prompt": "Bake sale", + "answers": { + "what": { + "picked": ["Bake something to sell", "Staff the table"], + "added": ["Bring drinks"] + }, + "when": { "kind": "one_day", "date": "2027-05-08", "from": "09:00", "to": "13:00" }, + "howMany": [null, 2, 1] + }, + "expect": { + "questions.topics_must_equal": ["what", "when", "how_many"], + "slots.count": 3, + "slot_values_must_cover_all": ["bake something to sell", "staff the table", "bring drinks"], + "slot_dates_or_description_must_mention": ["2027-05-08", "may 8", "8 may"], + "slot_times_or_description_must_mention": ["09:00", "9am", "9 am"], + "slots.capacity_for_label": [ + { "label_match": "bake something", "capacity": null }, + { "label_match": "staff the table", "capacity": 2 }, + { "label_match": "bring drinks", "capacity": 1 } + ], + "rationale": "#570: the second call drafts from the answers as given: one slot per item, that date and time, and those counts, with no limit as capacity null." + } + }, + { + "id": "ev_037_answers_all_skipped_draft_placeholders", + "tags": ["questions", "answers", "no_invention"], + "runner": "llm", + "prompt": "Class picnic", + "answers": { + "what": { "picked": [], "added": [] }, + "when": { "kind": "not_decided" }, + "placeholders": true + }, + "expect": { + "title_must_match_any": ["picnic"], + "slots.count_range": [1, 6], + "slot_values_must_cover_all": ["tbd"], + "slot_dates.set_must_equal": [], + "rationale": "#570: Draft with placeholders. Every answer is skipped, so the second call drafts \"TBD:\" rows and invents nothing, and never asks again." + } } ] } diff --git a/scripts/eval-magic-compose.ts b/scripts/eval-magic-compose.ts index ac762810..fd30a31c 100644 --- a/scripts/eval-magic-compose.ts +++ b/scripts/eval-magic-compose.ts @@ -6,10 +6,17 @@ * production Zod schema, and interprets the eval DSL. Prints per-case * pass/fail and writes a snapshot to evals/magic-compose/snapshots/. * + * A reply can be a draft, a refusal or questions. A case with `answers` runs + * both calls the page makes: the prompt, which must ask, then the prompt with + * the answers (`buildFollowUpMessages`), whose draft is scored. A reply to the + * answers that asks again, in any shape, is `asked_again`, as the route reads + * it. + * * Usage: * pnpm eval:magic-compose # all llm cases * pnpm eval:magic-compose --case=ev_021 # single case * pnpm eval:magic-compose --tag=cross_product # by tag + * pnpm eval:magic-compose --tag=questions # the cases about asking * pnpm eval:magic-compose --list # list cases without running * pnpm eval:magic-compose --no-snapshot # skip writing snapshot */ @@ -18,40 +25,62 @@ config({ path: '.env.local' }); config({ path: '.env' }); import path from 'node:path'; +import { buildFollowUpMessages } from '@/lib/magic-compose/follow-up'; import { defaultLlmClient } from '@/lib/magic-compose/llm-client'; import { + asksQuestions, buildMessages, isQuestionsReply, MagicComposeDraftSchema, + type ChatMessage, type MagicComposeParsed, } from '@/lib/magic-compose/prompt'; -import { runEvalCli } from './lib/eval-harness'; +import { runEvalCli, type Produced } from './lib/eval-harness'; function isRefusal(p: MagicComposeParsed): p is { refusalReason: string } { return 'refusalReason' in p && typeof p.refusalReason === 'string'; } +/** + * One call to the model, read the way the route reads it. After the answers, + * a reply that asks questions in any shape is asked again before the schema + * sees it, so a partial re-ask (only "when", with no options) isn't scored as + * a broken draft. + */ +async function ask( + messages: ChatMessage[], + { afterAnswers = false }: { afterAnswers?: boolean } = {}, +): Promise { + const client = defaultLlmClient(); + const t0 = Date.now(); + const res = await client.generateDraft(messages); + const ms = Date.now() - t0; + if (!res.ok) { + return { status: 'llm_failed', code: res.error.code, message: res.error.message, ms }; + } + if (afterAnswers && asksQuestions(res.value)) { + return { status: 'asked_again', raw: res.value, ms }; + } + + const parsed = MagicComposeDraftSchema.safeParse(res.value); + if (!parsed.success) { + return { status: 'parse_failed', raw: res.value, issues: parsed.error.issues, ms }; + } + if (isQuestionsReply(parsed.data)) { + return { status: 'questions', raw: res.value, questions: parsed.data.questions, ms }; + } + if (isRefusal(parsed.data)) { + return { status: 'refusal', raw: res.value, refusalReason: parsed.data.refusalReason, ms }; + } + return { status: 'draft', raw: res.value, draft: parsed.data, ms }; +} + runEvalCli({ snapshotDir: path.join(process.cwd(), 'evals/magic-compose/snapshots'), schemaName: 'MagicComposeDraftSchema', - async produce(c) { - const client = defaultLlmClient(); - const t0 = Date.now(); - const res = await client.generateDraft(buildMessages(c.prompt)); - const ms = Date.now() - t0; - if (!res.ok) return { status: 'llm_failed', code: res.error.code, message: res.error.message, ms }; - - const parsed = MagicComposeDraftSchema.safeParse(res.value); - if (!parsed.success) return { status: 'parse_failed', raw: res.value, issues: parsed.error.issues, ms }; - if (isQuestionsReply(parsed.data)) { - // The prompt doesn't ask yet; scoring questions comes with the prompt change. - return { status: 'llm_failed', code: 'questions', message: 'asked questions instead of drafting', ms }; - } - if (isRefusal(parsed.data)) { - return { status: 'refusal', raw: res.value, refusalReason: parsed.data.refusalReason, ms }; - } - return { status: 'draft', raw: res.value, draft: parsed.data, ms }; - }, + produce: (c) => ask(buildMessages(c.prompt)), + // The harness has checked the answers with FollowUpSchema before either call. + answer: (c, answers) => ask(buildFollowUpMessages(c.prompt, answers), { afterAnswers: true }), }).catch((e) => { console.error(e); process.exit(1); diff --git a/scripts/eval-mcp.ts b/scripts/eval-mcp.ts index 2df9258b..a6580285 100644 --- a/scripts/eval-mcp.ts +++ b/scripts/eval-mcp.ts @@ -5,7 +5,12 @@ * against a tool-calling model. The model gets the server instructions and the * tools as tools/list describes them, plus one organizer message; the arguments * of its create_signup call are scored with the same checks. A reply with no - * create counts as a refusal, so the refusal cases still mean something. No + * create counts as a refusal, so the refusal cases still mean something, except + * on a case where Magic Compose would ask (or that says `assistant_may_ask`): + * there a reply that asks something (a "?", or a request for details such as + * "Let me know the dates") passes, since asking the organizer is right, and + * any other reply fails. Cases with + * `answers` are skipped: they answer Magic Compose's own questions. No * database: the read tools answer as an empty account (src/mcp/evals/assistant.ts). * * Also runs the follow-up cases in evals/mcp/followup-evals.json, where the @@ -42,7 +47,7 @@ import { scoreFollowup } from '@/mcp/evals/followup-checks'; import { runFollowupCase } from '@/mcp/evals/followups'; import { buildInstructions } from '@/mcp/instructions'; import { TOOLS } from '@/mcp/tools'; -import { DEV_EVALS_FILE, runEvalCli } from './lib/eval-harness'; +import { askedBeforeCreating, asksFirst, DEV_EVALS_FILE, runEvalCli } from './lib/eval-harness'; const INSTRUCTIONS = buildInstructions(TOOLS); @@ -55,7 +60,13 @@ const send: SendChat = async (messages, tools) => { 'Content-Type': 'application/json', Authorization: `Bearer ${env.LLM_API_KEY ?? 'dummy'}`, }, - body: JSON.stringify({ model: env.LLM_MODEL, messages, tools, temperature: 0, max_tokens: 16_000 }), + body: JSON.stringify({ + model: env.LLM_MODEL, + messages, + tools, + temperature: 0, + max_tokens: 16_000, + }), signal: AbortSignal.timeout(env.LLM_TIMEOUT_MS), }); if (!res.ok) { @@ -64,8 +75,13 @@ const send: SendChat = async (messages, tools) => { } const body = (await res.json()) as { choices?: { message?: AssistantMessage }[] }; const message = body.choices?.[0]?.message; - if (!message) return err({ code: 'schema_mismatch', message: 'no choices[0].message in the response' }); - return ok({ role: 'assistant', content: message.content ?? null, ...(message.tool_calls ? { tool_calls: message.tool_calls } : {}) }); + if (!message) + return err({ code: 'schema_mismatch', message: 'no choices[0].message in the response' }); + return ok({ + role: 'assistant', + content: message.content ?? null, + ...(message.tool_calls ? { tool_calls: message.tool_calls } : {}), + }); } catch (e) { if (e instanceof DOMException && e.name === 'TimeoutError') { return err({ code: 'timeout', message: `no answer after ${env.LLM_TIMEOUT_MS}ms` }); @@ -82,12 +98,16 @@ runEvalCli({ const t0 = Date.now(); if (c.followUps) { // Without Postgres only these cases fail; the single-message ones still run. - const run = await runFollowupCase(getDb(), { prompt: c.prompt, followUps: c.followUps }, { - send, - tools: TOOLS, - instructions: INSTRUCTIONS, - now: new Date(), - }).catch((e: unknown) => ({ + const run = await runFollowupCase( + getDb(), + { prompt: c.prompt, followUps: c.followUps }, + { + send, + tools: TOOLS, + instructions: INSTRUCTIONS, + now: new Date(), + }, + ).catch((e: unknown) => ({ ok: false as const, code: 'error', message: e instanceof Error ? e.message : String(e), @@ -95,21 +115,51 @@ runEvalCli({ })); const ms = Date.now() - t0; const turns = run.turns.map((t) => ({ message: t.message, ...t.outcome })); - const trace = { toolCalls: run.turns.flatMap((t) => t.outcome.toolCalls.map((tc) => tc.name)), turns }; + const trace = { + toolCalls: run.turns.flatMap((t) => t.outcome.toolCalls.map((tc) => tc.name)), + turns, + }; if (!run.ok) return { status: 'llm_failed', code: run.code, message: run.message, ms, trace }; - return { status: 'scored', raw: run.state, checks: scoreFollowup(c.expect, run.state), ms, trace }; + return { + status: 'scored', + raw: run.state, + checks: scoreFollowup(c.expect, run.state), + ms, + trace, + }; } - const out = await runAssistant(c.prompt, { send, tools: TOOLS, instructions: INSTRUCTIONS, now: new Date() }); + const out = await runAssistant(c.prompt, { + send, + tools: TOOLS, + instructions: INSTRUCTIONS, + now: new Date(), + }); const ms = Date.now() - t0; const trace = { toolCalls: out.toolCalls, rejectedCreates: out.rejectedCreates }; - if (out.kind === 'failed') return { status: 'llm_failed', code: out.code, message: out.message, ms, trace }; + if (out.kind === 'failed') + return { status: 'llm_failed', code: out.code, message: out.message, ms, trace }; if (out.kind === 'replied') { - return { status: 'refusal', raw: out.text, refusalReason: out.text || '(empty reply)', ms, trace }; + // Where Magic Compose would ask, an assistant that asks the organizer + // instead of creating is right (`askedBeforeCreating`); an empty reply, + // or one that asks nothing, such as a refusal, fails. Elsewhere a reply + // is scored as a refusal. + if (asksFirst(c) || c.expect.assistant_may_ask === true) { + const checks = [askedBeforeCreating(out.text)]; + return { status: 'scored', raw: out.text, checks, ms, trace }; + } + return { + status: 'refusal', + raw: out.text, + refusalReason: out.text || '(empty reply)', + ms, + trace, + }; } // runAssistant already checked the arguments against create_signup's own // schema; this parse drops workspaceId and types the draft. const parsed = FullDraftSchema.safeParse(out.args); - if (!parsed.success) return { status: 'parse_failed', raw: out.args, issues: parsed.error.issues, ms, trace }; + if (!parsed.success) + return { status: 'parse_failed', raw: out.args, issues: parsed.error.issues, ms, trace }; return { status: 'draft', raw: out.args, draft: parsed.data, ms, trace }; }, }).catch((e) => { diff --git a/scripts/lib/eval-harness.test.ts b/scripts/lib/eval-harness.test.ts new file mode 100644 index 00000000..56e76f67 --- /dev/null +++ b/scripts/lib/eval-harness.test.ts @@ -0,0 +1,310 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import type { MagicComposeDraft } from '@/lib/magic-compose/prompt'; +import type { FollowUp, MagicComposeQuestions } from '@/lib/magic-compose/questions'; +import { + askedBeforeCreating, + asksFirst, + DEV_EVALS_FILE, + runOneCase, + type EvalCase, + type EvalRunner, + type Produced, +} from './eval-harness'; + +const DRAFT: MagicComposeDraft = { + title: 'Bake sale', + description: '', + fields: [{ ref: 'what', label: 'What', fieldType: 'text', choices: undefined }], + slots: [{ values: { what: 'Staff the table' }, capacity: 2 }], + groupBy: undefined, +}; + +const ASKS_ALL: MagicComposeQuestions = { + topics: ['what', 'when', 'how_many'], + whatOptions: ['Bake something to sell', 'Staff the table'], +}; + +const ANSWERS: FollowUp = { + what: { picked: ['Staff the table'], added: [] }, + when: { kind: 'one_day', date: '2027-05-08' }, + howMany: [2], +}; + +const drafted: Produced = { status: 'draft', raw: DRAFT, draft: DRAFT, ms: 1 }; +const asks = (questions: MagicComposeQuestions): Produced => ({ + status: 'questions', + raw: { questions }, + questions, + ms: 1, +}); + +function evalCase(over: Partial): EvalCase { + return { id: 'ev_test', tags: [], runner: 'llm', prompt: 'Bake sale', expect: {}, ...over }; +} + +function runner(first: Produced, second?: Produced) { + const answer = vi.fn>(async () => second ?? drafted); + const produce = vi.fn(async () => first); + return { snapshotDir: '', schemaName: 'test', produce, answer }; +} + +const failed = (checks: { key: string; outcome: string }[]) => + checks.filter((c) => c.outcome === 'fail').map((c) => c.key); + +/** A case from dev-evals.json as it stands, by the start of its id. */ +function devCase(idPrefix: string): EvalCase { + const { cases } = JSON.parse(readFileSync(DEV_EVALS_FILE, 'utf8')) as { cases: EvalCase[] }; + const c = cases.find((x) => x.id.startsWith(idPrefix)); + if (!c) throw new Error(`no case ${idPrefix} in dev-evals.json`); + return c; +} + +const draftOf = (over: Partial): Produced => { + const draft = { ...DRAFT, ...over }; + return { status: 'draft', raw: draft, draft, ms: 1 }; +}; + +describe('questions.must_ask', () => { + it('expects questions when true, a draft when false, and decides over other questions keys', () => { + expect(asksFirst(evalCase({ expect: { 'questions.must_ask': true } }))).toBe(true); + expect(asksFirst(evalCase({ expect: { 'questions.must_ask': false } }))).toBe(false); + expect(asksFirst(evalCase({ expect: { 'questions.topics_must_include': ['what'] } }))).toBe( + true, + ); + expect( + asksFirst( + evalCase({ + expect: { 'questions.must_ask': false, 'questions.options_count_range': [2, 4] }, + }), + ), + ).toBe(false); + expect(asksFirst(evalCase({ expect: { 'slots.count': 1 } }))).toBe(false); + }); + + it('passes a draft when false, and fails questions as unexpected', async () => { + const c = evalCase({ expect: { 'questions.must_ask': false, 'slots.count': 1 } }); + + const draft = await runOneCase(runner(drafted), c); + expect(failed(draft.checks)).toEqual([]); + expect(draft.checks.find((ch) => ch.key === 'questions.must_ask')?.outcome).toBe('pass'); + + const asked = await runOneCase(runner(asks(ASKS_ALL)), c); + expect(failed(asked.checks)).toEqual(['unexpected_questions']); + }); + + it('marks a case with answers that says it must not ask as broken, before any call', async () => { + const r = runner(asks(ASKS_ALL)); + const run = await runOneCase( + r, + evalCase({ answers: ANSWERS, expect: { 'questions.must_ask': false } }), + ); + expect(run.status).toBe('broken'); + expect(r.produce).not.toHaveBeenCalled(); + }); +}); + +describe('a case with answers', () => { + it('is broken, with no call made, when its answers fail FollowUpSchema', async () => { + const r = runner(asks(ASKS_ALL)); + const bad = { ...ANSWERS, when: { kind: 'one_day', date: '2027-02-30' } } as FollowUp; + const run = await runOneCase(r, evalCase({ answers: bad })); + expect(run.status).toBe('broken'); + expect(run.checks[0]?.detail).toMatch(/FollowUpSchema at when\.date/); + expect(r.produce).not.toHaveBeenCalled(); + expect(r.answer).not.toHaveBeenCalled(); + }); + + it('sends only the topics the first call asked about, and what always', async () => { + const r = runner( + asks({ topics: ['what'], whatOptions: ['Bake something to sell', 'Staff the table'] }), + ); + const run = await runOneCase(r, evalCase({ answers: ANSWERS })); + expect(r.answer).toHaveBeenCalledWith(expect.anything(), { what: ANSWERS.what }); + expect(run.answersSent).toEqual({ what: ANSWERS.what }); + + const all = runner(asks(ASKS_ALL)); + await runOneCase(all, evalCase({ answers: ANSWERS })); + expect(all.answer).toHaveBeenCalledWith(expect.anything(), ANSWERS); + }); + + it('sends the answers as written when the first call drafted', async () => { + const r = runner(drafted); + const run = await runOneCase(r, evalCase({ answers: ANSWERS })); + expect(r.answer).toHaveBeenCalledWith(expect.anything(), ANSWERS); + expect(failed(run.checks)).toEqual(['first:expected_questions_but_drafted']); + }); + + it('fails a reply to the answers that asks again, in any shape, as asked_again', async () => { + const again: Produced = { + status: 'asked_again', + raw: { questions: { topics: ['when'] } }, + ms: 1, + }; + const run = await runOneCase( + runner(asks(ASKS_ALL), again), + evalCase({ answers: ANSWERS, expect: { 'slots.count': 1 } }), + ); + expect(run.status).toBe('ok'); + expect(run.checks).toContainEqual({ + key: 'asked_again', + outcome: 'fail', + detail: 'asked about when instead of drafting', + }); + expect(run.checks.find((ch) => ch.key === 'slots.count')?.outcome).toBe('skip'); + }); + + it("keeps the first call's checks when the second call fails, and says which call failed", async () => { + const down: Produced = { status: 'llm_failed', code: 'timeout', message: 'no answer', ms: 1 }; + const run = await runOneCase( + runner(asks(ASKS_ALL), down), + evalCase({ + answers: ANSWERS, + expect: { 'questions.topics_must_equal': ['what', 'when', 'how_many'] }, + }), + ); + expect(run.status).toBe('llm_failed'); + expect(run.llmErrorMessage).toBe('second call: no answer'); + expect(run.checks.map((ch) => `${ch.key} ${ch.outcome}`)).toEqual([ + 'first:questions.must_ask pass', + 'first:questions.topics_must_equal pass', + ]); + }); +}); + +describe('questions.options_must_not_contain_date_or_number', () => { + const c = evalCase({ expect: { 'questions.options_must_not_contain_date_or_number': true } }); + const check = async (whatOptions: string[]) => { + const run = await runOneCase(runner(asks({ topics: ['what'], whatOptions })), c); + return run.checks.find((ch) => ch.key === 'questions.options_must_not_contain_date_or_number'); + }; + + it('flags French counts, months and weekdays, accents or not', async () => { + for (const option of [ + 'Apporter deux gâteaux', + 'Tenir le stand en décembre', + 'Aider samedi', + 'Vingt crêpes', + ]) { + expect((await check([option, 'Ranger la salle']))?.outcome, option).toBe('fail'); + } + }); + + it('lets French articles through', async () => { + expect((await check(['Faire un gâteau', 'Tenir une table']))?.outcome).toBe('pass'); + }); +}); + +describe('slot_dates.*', () => { + const dated = (date: string) => + draftOf({ + title: 'Class picnic', + fields: [ + { ref: 'what', label: 'What', fieldType: 'text', choices: undefined }, + { ref: 'date', label: 'Date', fieldType: 'date', choices: undefined }, + ], + slots: [{ values: { what: 'TBD: item 1', date }, capacity: 1 }], + }); + const setCheck = async (date: string) => { + const c = evalCase({ expect: { 'slot_dates.set_must_equal': [] } }); + const run = await runOneCase(runner(dated(date)), c); + return run.checks.find((ch) => ch.key === 'slot_dates.set_must_equal')?.outcome; + }; + + it('counts only real days, as the converter keeps only those', async () => { + expect(await setCheck('TBD: date 1')).toBe('pass'); + expect(await setCheck('tbd')).toBe('pass'); + expect(await setCheck('Saturday')).toBe('pass'); + expect(await setCheck('TBD (2027-06-05)')).toBe('pass'); + expect(await setCheck('2027-06-05')).toBe('fail'); + expect(await setCheck(' 2027-06-05 ')).toBe('fail'); + }); + + it('passes ev_037 on a placeholder draft whose date field says TBD', async () => { + const run = await runOneCase(runner(asks(ASKS_ALL), dated('TBD: date 1')), devCase('ev_037')); + expect(run.status).toBe('ok'); + expect(failed(run.checks)).toEqual([]); + }); +}); + +describe('must_express_target_via_one_of', () => { + it('takes a quantity field for ev_002, the bake sale with 80 cookies, 40 brownies and 30 cupcakes', async () => { + const quantities = draftOf({ + title: 'Bake sale for the 5th grade trip', + description: 'Bring your batch by 8am on Friday, May 22, 2026.', + fields: [ + { ref: 'item', label: 'Item', fieldType: 'text', choices: undefined }, + { ref: 'quantity', label: 'Quantity', fieldType: 'number', choices: undefined }, + ], + slots: [ + { values: { item: 'Cookies', quantity: 80 }, capacity: 1 }, + { values: { item: 'Brownies', quantity: 40 }, capacity: 1 }, + { values: { item: 'Cupcakes', quantity: 30 }, capacity: 1 }, + ], + }); + const run = await runOneCase(runner(quantities), devCase('ev_002')); + expect(failed(run.checks)).toEqual([]); + expect(run.checks.find((ch) => ch.key === 'must_express_target_via_one_of')?.outcome).toBe( + 'pass', + ); + }); +}); + +describe('ev_031, the French bake sale', () => { + const options = async (whatOptions: string[]) => { + const run = await runOneCase( + runner(asks({ topics: ['what', 'when', 'how_many'], whatOptions })), + devCase('ev_031'), + ); + return failed(run.checks); + }; + + it('passes options in French about the bake sale', async () => { + expect(await options(['Faire un gâteau', 'Tenir le stand'])).toEqual([]); + expect(await options(['Préparer des crêpes', 'Aider à la vente'])).toEqual([]); + }); + + it('fails options in French about something else, articles and all', async () => { + expect( + await options(['Apporter des collations', 'Conduire aux matchs', "Aider à l'entraînement"]), + ).toEqual(['questions.options_must_mention_any']); + }); + + it('fails options in English, even with words French shares', async () => { + expect(await options(['Run the stand', 'Drinks table'])).toContain( + 'questions.options_must_mention_any', + ); + }); +}); + +describe('askedBeforeCreating', () => { + it('passes a reply that asks, with a "?" or by asking for details', () => { + for (const reply of [ + 'What will people sign up for, and when?', + "Let me know what people will sign up for and the dates, and I'll set it up.", + 'Please share the game dates and how many families you need for each.', + 'Please tell me the dates and how many people you need.', + 'Before I create it, I’ll need the dates of the games.', + 'I need a few details first: what people sign up for, when, and how many.', + ]) { + expect(askedBeforeCreating(reply).outcome, reply).toBe('pass'); + } + }); + + it('fails an empty reply, and one that asks for nothing, such as a refusal', () => { + expect(askedBeforeCreating(' \n')).toEqual({ + key: 'asked_before_creating', + outcome: 'fail', + detail: 'empty reply: neither asked nor created', + }); + for (const reply of [ + "I can't help with that.", + "I can't create that signup. Let me know if there's anything else.", + "I can't set that up. Let me know what else I can help with.", + "Please tell me if you'd like something different.", + 'Done: the signup is ready.', + ]) { + expect(askedBeforeCreating(reply).outcome, reply).toBe('fail'); + } + }); +}); diff --git a/scripts/lib/eval-harness.ts b/scripts/lib/eval-harness.ts index 715cfe44..7c050402 100644 --- a/scripts/lib/eval-harness.ts +++ b/scripts/lib/eval-harness.ts @@ -1,17 +1,22 @@ /** * What the eval runners share: the cases in evals/magic-compose/dev-evals.json, - * the DSL that scores a draft against a case's `expect`, and the CLI around - * it (flags, per-case output, summary, snapshot). Each runner supplies only - * how a case's prompt becomes a draft: eval-magic-compose.ts asks the Magic - * Compose prompt, eval-mcp.ts asks a tool-calling model through the MCP tools. - * A runner can also load more case files and score a case itself (`scored`), - * as eval-mcp.ts does for its follow-up cases. + * the DSL that scores a draft (or the questions Magic Compose asks instead) + * against a case's `expect`, and the CLI around it (flags, per-case output, + * summary, snapshot). Each runner supplies only how a case's prompt becomes a + * draft: eval-magic-compose.ts asks the Magic Compose prompt, eval-mcp.ts asks + * a tool-calling model through the MCP tools. A runner can also load more case + * files and score a case itself (`scored`), as eval-mcp.ts does for its + * follow-up cases, and send a case's `answers` to its questions (`answer`), as + * eval-magic-compose.ts does. */ import fs from 'node:fs/promises'; import path from 'node:path'; import type { z } from 'zod'; import { magicComposeEnabled } from '@/lib/env'; +import { FollowUpSchema } from '@/lib/magic-compose/follow-up'; import type { MagicComposeDraft } from '@/lib/magic-compose/prompt'; +import type { FollowUp, MagicComposeQuestions } from '@/lib/magic-compose/questions'; +import { isRealDate } from '@/lib/reminder-fields'; const REPO_ROOT = process.cwd(); export const DEV_EVALS_FILE = path.join(REPO_ROOT, 'evals/magic-compose/dev-evals.json'); @@ -24,6 +29,14 @@ export interface EvalCase { prompt: string; /** Organizer messages sent after `prompt`, one turn each. */ followUps?: string[]; + /** + * The organizer's answers to Magic Compose's questions, checked with + * `FollowUpSchema` before any call. The first call must ask; then the + * prompt goes back with these, as the page sends them (only the topics it + * asked about), and that draft is scored. `questions.*` checks score the + * first call. + */ + answers?: FollowUp; expect: Record; } @@ -39,12 +52,19 @@ interface CaseRun { runner: Runner; tags: string[]; prompt: string; - status: 'ok' | 'parse_failed' | 'llm_failed' | 'skipped'; + /** `broken`: the case itself is wrong (its answers fail `FollowUpSchema`, say), so nothing was sent. */ + status: 'ok' | 'parse_failed' | 'llm_failed' | 'skipped' | 'broken'; llmErrorCode?: string; llmErrorMessage?: string; rawOutput?: unknown; draft?: MagicComposeDraft; refusalReason?: string; + /** What Magic Compose asked: the whole reply, or the first call of a case with `answers`. */ + questions?: MagicComposeQuestions; + /** The first call's reply in a case with `answers`; `rawOutput` is the reply to the answers. */ + firstRawOutput?: unknown; + /** The answers the second call sent: the case's, less the topics the first call didn't ask about. */ + answersSent?: FollowUp; /** Runner-specific detail kept in the snapshot, such as the tools a model called. */ trace?: Record; checks: CheckResult[]; @@ -76,10 +96,7 @@ async function loadCases(files: readonly string[]): Promise { return cases; } -function filterCases( - cases: EvalCase[], - opts: { caseId?: string; tag?: string }, -): EvalCase[] { +function filterCases(cases: EvalCase[], opts: { caseId?: string; tag?: string }): EvalCase[] { return cases.filter((c) => { if (opts.caseId && c.id !== opts.caseId) return false; if (opts.tag && !c.tags.includes(opts.tag)) return false; @@ -99,20 +116,29 @@ function lower(s: string): string { * date checks — dates legitimately contain hyphens. */ function loose(s: string): string { - return s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').replace(/\s+/g, ' ').trim(); + return s + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); } function looseIncludes(haystack: string, needle: string): boolean { return loose(haystack).includes(loose(needle)); } +/** + * The date fields' values that are real days, as the converter keeps them: a + * "TBD: date 1" placeholder, a weekday or "TBD (2026-06-06)" is dropped there, + * so the signup has no date for it. + */ function dateValues(draft: MagicComposeDraft): string[] { const dateRefs = draft.fields.filter((f) => f.fieldType === 'date').map((f) => f.ref); const out: string[] = []; for (const slot of draft.slots) { for (const ref of dateRefs) { const v = slot.values[ref]; - if (typeof v === 'string') out.push(v); + if (typeof v === 'string' && isRealDate(v.trim())) out.push(v.trim()); } } return out; @@ -120,7 +146,11 @@ function dateValues(draft: MagicComposeDraft): string[] { function slotValuesText(draft: MagicComposeDraft): string { return draft.slots - .map((s) => Object.values(s.values).map((v) => String(v)).join(' ')) + .map((s) => + Object.values(s.values) + .map((v) => String(v)) + .join(' '), + ) .join(' ') .toLowerCase(); } @@ -130,6 +160,63 @@ function fullText(draft: MagicComposeDraft): string { return [draft.title, draft.description, fieldText, slotValuesText(draft)].join(' ').toLowerCase(); } +/** Lowercase with accents dropped, so "Gâteau" and "gateau" compare equal. */ +function fold(s: string): string { + return s.normalize('NFD').replace(/\p{M}/gu, '').toLowerCase(); +} + +/** + * Whether `text` holds `needle` as whole words, both folded. Whole words, so + * "sell" doesn't hit inside another word; Unicode-aware, since `\b` isn't. + */ +function hasWords(text: string, needle: string): boolean { + const escaped = fold(needle).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`(? hasWords(option, w)) ?? null; +} + +/** One reply as the checks see it: at most one of draft, refusalReason and questions is set. */ +interface Reply { + kind: Reached['status']; + raw: unknown; + draft: MagicComposeDraft | null; + refusalReason: string | null; + questions: MagicComposeQuestions | null; +} + +/** What a check that needs a draft says it was given instead. */ +const REPLY_NAME: Record = { + draft: 'a draft', + refusal: 'a refusal payload', + questions: 'a questions reply', + asked_again: 'a reply that asked again', +}; + function fail(key: string, detail: string): CheckResult { return { key, outcome: 'fail', detail }; } @@ -146,14 +233,9 @@ function normalizeKey(key: string): string { return key.replace(/\.[a-z]$/, ''); } -function runCheck( - rawKey: string, - expected: unknown, - draft: MagicComposeDraft | null, - rawOutput: unknown, - refusalReason: string | null = null, -): CheckResult { +function runCheck(rawKey: string, expected: unknown, reply: Reply): CheckResult { const key = normalizeKey(rawKey); + const { raw: rawOutput, refusalReason, questions } = reply; const arr = (v: unknown): string[] | null => Array.isArray(v) && v.every((x) => typeof x === 'string') ? (v as string[]) : null; const num = (v: unknown): number | null => @@ -177,11 +259,89 @@ function runCheck( } } - // Every check below needs a full draft (the refusal arm has no fields/slots). - if (draft === null) { - return skip(rawKey, 'not applicable to a refusal payload'); + // true expects questions, false a draft. A reply the other way already + // fails as expected_questions_but_drafted or unexpected_questions, so this + // check only passes or skips. + if (key === 'questions.must_ask') { + if (expected === true) return questions ? pass(rawKey) : skip(rawKey, 'no questions reply'); + if (expected === false) { + return questions ? skip(rawKey, 'the reply asks: see unexpected_questions') : pass(rawKey); + } + return skip(rawKey, 'expected true or false'); + } + + // Questions-only check keys: what Magic Compose asked instead of drafting. + if (key.startsWith('questions.')) { + if (questions === null) return skip(rawKey, 'no questions reply'); + const topics: readonly string[] = questions.topics; + const options = questions.whatOptions; + switch (key) { + case 'questions.topics_must_include': { + const a = arr(expected); + if (!a) return skip(rawKey, 'expected string[]'); + const missing = a.filter((t) => !topics.includes(t)); + return missing.length === 0 + ? pass(rawKey) + : fail(rawKey, `asked about ${JSON.stringify(topics)}, without ${missing.join(', ')}`); + } + case 'questions.topics_must_not_include': { + const a = arr(expected); + if (!a) return skip(rawKey, 'expected string[]'); + const hits = a.filter((t) => topics.includes(t)); + return hits.length === 0 ? pass(rawKey) : fail(rawKey, `asked about ${hits.join(', ')}`); + } + case 'questions.topics_must_equal': { + const a = arr(expected); + if (!a) return skip(rawKey, 'expected string[]'); + const got = [...topics].sort(); + const want = [...a].sort(); + return JSON.stringify(got) === JSON.stringify(want) + ? pass(rawKey) + : fail(rawKey, `asked about ${JSON.stringify(got)} ≠ ${JSON.stringify(want)}`); + } + case 'questions.options_count_range': { + const a = expected as unknown[]; + if (!Array.isArray(a) || a.length !== 2) return skip(rawKey, 'expected [min,max]'); + const [min, max] = a as [number, number]; + const n = options.length; + return n >= min && n <= max + ? pass(rawKey) + : fail(rawKey, `${n} options not in [${min},${max}]`); + } + case 'questions.options_must_not_contain_date_or_number': { + if (expected !== true) return skip(rawKey, 'expected true'); + const hits = options.flatMap((o) => { + const hit = dateOrNumberIn(o); + return hit === null ? [] : [`"${o}" (${hit})`]; + }); + return hits.length === 0 + ? pass(rawKey) + : fail(rawKey, `options with a date or number: ${hits.join(', ')}`); + } + case 'questions.options_must_mention_any': { + const a = arr(expected); + if (!a) return skip(rawKey, 'expected string[]'); + return options.some((o) => a.some((w) => hasWords(o, w))) + ? pass(rawKey) + : fail(rawKey, `options ${JSON.stringify(options)} mention none of ${JSON.stringify(a)}`); + } + case 'questions.options_must_not_mention_any': { + const a = arr(expected); + if (!a) return skip(rawKey, 'expected string[]'); + const hits = options.filter((o) => a.some((w) => hasWords(o, w))); + return hits.length === 0 + ? pass(rawKey) + : fail(rawKey, `options with forbidden words: ${JSON.stringify(hits)}`); + } + default: + return skip(rawKey, `unsupported key (extend runner if you need it)`); + } } + // Every check below needs a full draft (no other kind of reply has fields/slots). + const draft = reply.draft; + if (draft === null) return skip(rawKey, `not applicable to ${REPLY_NAME[reply.kind]}`); + switch (key) { case 'title_must_match_any': { const a = arr(expected); @@ -212,7 +372,10 @@ function runCheck( const bad = draft.fields.filter((f) => !a.includes(f.fieldType)); return bad.length === 0 ? pass(rawKey) - : fail(rawKey, `fieldTypes outside ${JSON.stringify(a)}: ${bad.map((f) => f.fieldType).join(',')}`); + : fail( + rawKey, + `fieldTypes outside ${JSON.stringify(a)}: ${bad.map((f) => f.fieldType).join(',')}`, + ); } case 'fields.must_include_field_type': { const t = typeof expected === 'string' ? expected : null; @@ -245,9 +408,10 @@ function runCheck( case 'fields.labels_must_not_match_any': { const a = arr(expected); if (!a) return skip(rawKey, 'expected string[]'); - const pick = key === 'fields.refs_must_not_match_any' - ? (f: { ref: string }) => f.ref - : (f: { label: string }) => f.label; + const pick = + key === 'fields.refs_must_not_match_any' + ? (f: { ref: string }) => f.ref + : (f: { label: string }) => f.label; const haystack = draft.fields.map(pick); const hits: string[] = []; for (const needle of a) { @@ -286,17 +450,13 @@ function runCheck( const n = num(expected); if (n === null) return skip(rawKey, 'expected number'); const cap = draft.slots[0]?.capacity; - return cap === n - ? pass(rawKey) - : fail(rawKey, `slots[0].capacity=${cap} ≠ ${n}`); + return cap === n ? pass(rawKey) : fail(rawKey, `slots[0].capacity=${cap} ≠ ${n}`); } case 'every_slot.values_non_empty': { if (expected !== true) return skip(rawKey, 'expected true'); const empty = draft.slots.findIndex((s) => Object.keys(s.values).length === 0); - return empty === -1 - ? pass(rawKey) - : fail(rawKey, `slot[${empty}].values is empty`); + return empty === -1 ? pass(rawKey) : fail(rawKey, `slot[${empty}].values is empty`); } case 'every_slot.values_keys_must_match_declared_field_refs': { if (expected !== true) return skip(rawKey, 'expected true'); @@ -372,9 +532,7 @@ function runCheck( // For "must_include_starts" semantics, all must hit. For others, any. const requireAll = key === 'slot_times_or_labels.must_include_starts'; if (requireAll) { - return missing.length === 0 - ? pass(rawKey) - : fail(rawKey, `missing: ${missing.join(', ')}`); + return missing.length === 0 ? pass(rawKey) : fail(rawKey, `missing: ${missing.join(', ')}`); } return missing.length < a.length ? pass(rawKey) @@ -438,9 +596,7 @@ function runCheck( if (invalid.length > 0) { return fail(rawKey, `invalid regex pattern(s): ${invalid.join(', ')}`); } - return hits.length === 0 - ? pass(rawKey) - : fail(rawKey, `output matched: ${hits.join(', ')}`); + return hits.length === 0 ? pass(rawKey) : fail(rawKey, `output matched: ${hits.join(', ')}`); } case 'must_not_exceed_limits': { const v = expected as { max_fields?: number; max_slots?: number }; @@ -466,7 +622,10 @@ function runCheck( } return missing.length === 0 ? pass(rawKey) - : fail(rawKey, `no slot covered any of: ${missing.map((g) => JSON.stringify(g)).join('; ')}`); + : fail( + rawKey, + `no slot covered any of: ${missing.map((g) => JSON.stringify(g)).join('; ')}`, + ); } case 'fields.must_not_capture_any': { @@ -490,7 +649,8 @@ function runCheck( } case 'slots.capacity_for_label': { - const a = expected as Array<{ label_match: string; capacity: number }> | undefined; + // capacity null is no limit, as an answer of "no limit" asks for. + const a = expected as Array<{ label_match: string; capacity: number | null }> | undefined; if (!Array.isArray(a)) return skip(rawKey, 'expected array'); for (const req of a) { const match = draft.slots.find((s) => @@ -498,7 +658,10 @@ function runCheck( ); if (!match) return fail(rawKey, `no slot values include "${req.label_match}"`); if (match.capacity !== req.capacity) { - return fail(rawKey, `slot for "${req.label_match}" cap=${match.capacity} ≠ ${req.capacity}`); + return fail( + rawKey, + `slot for "${req.label_match}" cap=${match.capacity} ≠ ${req.capacity}`, + ); } } return pass(rawKey); @@ -563,6 +726,8 @@ function runCheck( case 'rationale': case 'robustness': return skip(rawKey, 'prose-only — not machine-checkable'); + case 'assistant_may_ask': + return skip(rawKey, 'read by eval:mcp, where a reply that asks passes'); default: return skip(rawKey, `unsupported key (extend runner if you need it)`); @@ -572,16 +737,59 @@ function runCheck( /** * What a runner hands back for one case. `draft` is what gets scored; * `refusalReason` takes the refusal arm instead (for Magic Compose its - * `{ refusalReason }` payload, for the MCP evals a reply with no create). + * `{ refusalReason }` payload, for the MCP evals a reply with no create), and + * `questions` the questions Magic Compose asks instead of drafting. + * `asked_again` is a reply to the answers that asks questions again, in any + * shape, even one `QuestionsSchema` would reject (`asksQuestions`). * `scored` carries checks the runner ran itself, for cases the draft DSL * can't describe (the MCP follow-up cases). */ export type Produced = - | { status: 'llm_failed'; code: string; message: string; ms: number; trace?: Record } - | { status: 'parse_failed'; raw: unknown; issues: z.ZodIssue[]; ms: number; trace?: Record } - | { status: 'draft'; raw: unknown; draft: MagicComposeDraft; ms: number; trace?: Record } - | { status: 'refusal'; raw: unknown; refusalReason: string; ms: number; trace?: Record } - | { status: 'scored'; raw: unknown; checks: CheckResult[]; ms: number; trace?: Record }; + | { + status: 'llm_failed'; + code: string; + message: string; + ms: number; + trace?: Record; + } + | { + status: 'parse_failed'; + raw: unknown; + issues: z.ZodIssue[]; + ms: number; + trace?: Record; + } + | { + status: 'draft'; + raw: unknown; + draft: MagicComposeDraft; + ms: number; + trace?: Record; + } + | { + status: 'refusal'; + raw: unknown; + refusalReason: string; + ms: number; + trace?: Record; + } + | { + status: 'questions'; + raw: unknown; + questions: MagicComposeQuestions; + ms: number; + trace?: Record; + } + | { status: 'asked_again'; raw: unknown; ms: number; trace?: Record } + | { + status: 'scored'; + raw: unknown; + checks: CheckResult[]; + ms: number; + trace?: Record; + }; + +type Reached = Extract; export interface EvalRunner { /** Where per-run snapshots go (gitignored by evals/**\/snapshots/). */ @@ -591,46 +799,149 @@ export interface EvalRunner { /** Case files to run. Defaults to dev-evals.json alone. */ caseFiles?: readonly string[]; produce(c: EvalCase): Promise; + /** + * Sends a case's prompt back with its `answers`, as the second call does: + * already checked with `FollowUpSchema`, and holding only the topics the + * first call asked about. A reply that asks again, in any shape, comes + * back as `asked_again`. A runner without it skips the cases that have + * answers. + */ + answer?(c: EvalCase, answers: FollowUp): Promise; } -async function runOneCase(runner: EvalRunner, c: EvalCase): Promise { - const base: CaseRun = { - caseId: c.id, - runner: c.runner, - tags: c.tags, - prompt: c.prompt, - status: 'ok', - checks: [], +/** The value of `questions.must_ask` in `expect`, or undefined without that key. */ +function mustAsk(expect: Record): unknown { + return Object.entries(expect).find(([k]) => normalizeKey(k) === 'questions.must_ask')?.[1]; +} + +/** + * Which way a case's `expect` says the model should go, other than drafting. + * `questions.must_ask` decides questions when the case has it (`false` + * expects a draft); without it, any other `questions.*` key expects them. + */ +function expectsOtherThanDraft(expect: Record): { + refusal: boolean; + questions: boolean; +} { + const keys = Object.keys(expect).map(normalizeKey); + return { + refusal: keys.some( + (k) => k === 'refusalReason_must_be_present' || k === 'refusalReason_must_mention_any', + ), + questions: keys.includes('questions.must_ask') + ? mustAsk(expect) === true + : keys.some((k) => k.startsWith('questions.')), }; +} - if (c.runner === 'server') { - return { - ...base, - status: 'skipped', - checks: [skip('runner=server', 'covered by route.db.test.ts; not exercised by this runner')], - }; +/** Whether Magic Compose should ask before it drafts, as the case's `questions.*` checks say. */ +export function asksFirst(c: EvalCase): boolean { + return expectsOtherThanDraft(c.expect).questions; +} + +/** + * Ways to ask the organizer for details without a "?", read on the reply in + * lower case: "Let me know the dates", "Please share…", "I'll need the + * dates". Each asks for something, so a refusal's "Let me know if there's + * anything else" isn't one. A short list on purpose, not a classifier. + */ +const ASKS_FOR_DETAILS = [ + /\b(?:let me know|tell me|send me|share|give me|confirm)\s+(?:what(?!\s+else)|when|which|who|where|how|the|your)\b/, + /\bplease\s+(?:share|send|provide|tell me\s+(?:what(?!\s+else)|when|which|who|where|how|the|your)|confirm|give|list)\b/, + /\bi(?:'ll| will|'d| would)? need (?:to know|a few|some|more|the|your)\b/, +]; + +/** + * Scores an eval:mcp reply that created nothing, on a case where asking the + * organizer is right. It asked when it holds a "?" or asks for details in so + * many words (`ASKS_FOR_DETAILS`); an empty reply fails, and so does one + * that asks nothing, such as a refusal. A create never gets here: it is + * scored as a draft. + */ +export function askedBeforeCreating(reply: string): CheckResult { + const key = 'asked_before_creating'; + const text = reply.trim(); + if (text === '') return fail(key, 'empty reply: neither asked nor created'); + const plain = text.toLowerCase().replace(/[‘’]/g, "'"); + if (plain.includes('?') || ASKS_FOR_DETAILS.some((re) => re.test(plain))) { + return { key, outcome: 'pass', detail: text.slice(0, 200) }; + } + return fail(key, `replied without asking for anything: ${text.slice(0, 200)}`); +} + +/** What a reply that asked again asked about: its topics, or its `questions` as sent. */ +function askedAbout(raw: unknown): string { + const field = (v: unknown, key: string): unknown => + typeof v === 'object' && v !== null ? (v as Record)[key] : undefined; + const questions = field(raw, 'questions'); + const topics = field(questions, 'topics'); + if (Array.isArray(topics) && topics.length > 0) return topics.map(String).join(', '); + return (JSON.stringify(questions) ?? 'nothing').slice(0, 120); +} + +/** + * Scores one reply against `expect`. A refusal or questions where `expect` + * wants neither fails as unexpected, and a draft where it wants questions + * fails too. A reply to the organizer's answers that asks again fails as + * `asked_again`, since questions are never right there. + */ +function scoreReply(expect: Record, res: Reached): CheckResult[] { + const wants = expectsOtherThanDraft(expect); + const reply: Reply = { + kind: res.status, + raw: res.raw, + draft: res.status === 'draft' ? res.draft : null, + refusalReason: res.status === 'refusal' ? res.refusalReason : null, + questions: res.status === 'questions' ? res.questions : null, + }; + const checks: CheckResult[] = []; + if (res.status === 'refusal' && !wants.refusal) { + checks.push( + fail( + 'unexpected_refusal', + `model refused on a happy-path case: "${res.refusalReason.slice(0, 120)}"`, + ), + ); } + if (res.status === 'questions' && !wants.questions) { + checks.push( + fail( + 'unexpected_questions', + `asked about ${res.questions.topics.join(', ')} instead of drafting`, + ), + ); + } + if (res.status === 'asked_again') { + checks.push(fail('asked_again', `asked about ${askedAbout(res.raw)} instead of drafting`)); + } + if (res.status === 'draft' && wants.questions) { + checks.push( + fail( + 'expected_questions_but_drafted', + `drafted "${res.draft.title.slice(0, 60)}" with ${res.draft.slots.length} slot(s)`, + ), + ); + } + for (const [k, v] of Object.entries(expect)) { + checks.push(runCheck(k, v, reply)); + } + return checks; +} - const res = await runner.produce(c); - const tokens = { ms: res.ms }; - if (res.trace) base.trace = res.trace; +/** A case run from what the runner produced, scored against `expect`. */ +function toRun(base: CaseRun, expect: Record, res: Produced): CaseRun { + const run: CaseRun = { ...base, tokens: { ms: res.ms } }; + if (res.trace) run.trace = res.trace; if (res.status === 'llm_failed') { - return { - ...base, - status: 'llm_failed', - llmErrorCode: res.code, - llmErrorMessage: res.message, - tokens, - }; + return { ...run, status: 'llm_failed', llmErrorCode: res.code, llmErrorMessage: res.message }; } if (res.status === 'parse_failed') { return { - ...base, + ...run, status: 'parse_failed', rawOutput: res.raw, - tokens, checks: res.issues.slice(0, 5).map((i) => ({ key: `schema:${i.path.join('.') || '(root)'}`, outcome: 'fail' as const, @@ -640,48 +951,134 @@ async function runOneCase(runner: EvalRunner, c: EvalCase): Promise { } if (res.status === 'scored') { - return { ...base, rawOutput: res.raw, checks: res.checks, tokens }; + return { ...run, rawOutput: res.raw, checks: res.checks }; } + const checks = scoreReply(expect, res); if (res.status === 'refusal') { - const refusalReason = res.refusalReason; - const checks: CheckResult[] = []; - const expectedRefusal = Object.keys(c.expect).some((k) => { - const nk = normalizeKey(k); - return ( - nk === 'refusalReason_must_be_present' || - nk === 'refusalReason_must_mention_any' - ); - }); - if (!expectedRefusal) { - checks.push( - fail( - 'unexpected_refusal', - `model refused on a happy-path case: "${refusalReason.slice(0, 120)}"`, - ), - ); - } - for (const [k, v] of Object.entries(c.expect)) { - checks.push(runCheck(k, v, null, res.raw, refusalReason)); - } - return { ...base, rawOutput: res.raw, refusalReason, checks, tokens }; + return { ...run, rawOutput: res.raw, refusalReason: res.refusalReason, checks }; } + if (res.status === 'questions') { + return { ...run, rawOutput: res.raw, questions: res.questions, checks }; + } + if (res.status === 'asked_again') return { ...run, rawOutput: res.raw, checks }; + return { ...run, rawOutput: res.raw, draft: res.draft, checks }; +} - const draft = res.draft; - const checks: CheckResult[] = []; +/** + * The answers as the page sends them after `topics`: `what` always, `when` + * and `howMany` only when the first call asked about them. + */ +function askedOnly(answers: FollowUp, topics: readonly string[]): FollowUp { + const { when, howMany, ...rest } = answers; + return { + ...rest, + ...(when !== undefined && topics.includes('when') ? { when } : {}), + ...(howMany !== undefined && topics.includes('how_many') ? { howMany } : {}), + }; +} + +/** + * A case with `answers`, run as the page runs it: the first call must ask, + * then the prompt goes back with the answers to the topics it asked about, + * and that reply is scored. When the first call didn't ask, the answers go + * back as written, since they test the second prompt on its own. The first + * call's checks are marked `first:`, and a second call that fails says so. + */ +async function runWithAnswers( + runner: EvalRunner & { answer: NonNullable }, + base: CaseRun, + c: EvalCase, + answers: FollowUp, +): Promise { + const firstExpect: Record = { 'questions.must_ask': true }; + const draftExpect: Record = {}; for (const [k, v] of Object.entries(c.expect)) { - checks.push(runCheck(k, v, draft, res.raw)); + if (normalizeKey(k).startsWith('questions.')) firstExpect[k] = v; + else draftExpect[k] = v; + } + const first = await runner.produce(c); + const markFirst = (checks: CheckResult[]) => + checks.map((ch) => ({ ...ch, key: `first:${ch.key}` })); + if (first.status !== 'draft' && first.status !== 'refusal' && first.status !== 'questions') { + const run = toRun(base, firstExpect, first); + return { + ...run, + checks: markFirst(run.checks), + ...(run.llmErrorMessage ? { llmErrorMessage: `first call: ${run.llmErrorMessage}` } : {}), + }; } + const firstChecks = markFirst(scoreReply(firstExpect, first)); + const sent = first.status === 'questions' ? askedOnly(answers, first.questions.topics) : answers; + const second = await runner.answer(c, sent); + const run = toRun(base, draftExpect, second); return { - ...base, - rawOutput: res.raw, - draft, - checks, - tokens, + ...run, + checks: [...firstChecks, ...run.checks], + ...(run.llmErrorMessage ? { llmErrorMessage: `second call: ${run.llmErrorMessage}` } : {}), + ...(first.status === 'questions' ? { questions: first.questions } : {}), + firstRawOutput: first.raw, + answersSent: sent, + tokens: { ms: first.ms + second.ms }, }; } +/** A case that can't run as written; nothing is sent. */ +function broken(base: CaseRun, detail: string): CaseRun { + return { ...base, status: 'broken', checks: [fail('broken_case', detail)] }; +} + +/** Runs and scores one case. Exported for the harness's own tests. */ +export async function runOneCase(runner: EvalRunner, c: EvalCase): Promise { + const base: CaseRun = { + caseId: c.id, + runner: c.runner, + tags: c.tags, + prompt: c.prompt, + status: 'ok', + checks: [], + }; + + if (c.runner === 'server') { + return { + ...base, + status: 'skipped', + checks: [skip('runner=server', 'covered by route.db.test.ts; not exercised by this runner')], + }; + } + + if (c.answers === undefined) return toRun(base, c.expect, await runner.produce(c)); + // Checked as the route checks them, before any call, so a case can't send + // answers the page never would, and a broken one costs nothing. + const answers = FollowUpSchema.safeParse(c.answers); + if (!answers.success) { + const issue = answers.error.issues[0]; + return broken( + base, + `the case's answers fail FollowUpSchema at ${issue?.path.join('.') || '(root)'}: ${issue?.message}`, + ); + } + const ask = mustAsk(c.expect); + if (ask !== undefined && ask !== true) { + return broken( + base, + 'a case with answers must ask first, so "questions.must_ask" can only be true', + ); + } + const { answer } = runner; + if (!answer) { + return { + ...base, + status: 'skipped', + checks: [ + skip('answers', 'answers Magic Compose questions; only eval:magic-compose sends them'), + ], + }; + } + return runWithAnswers({ ...runner, answer }, base, c, answers.data); +} + // ── Output ───────────────────────────────────────────────────────────────── const ICON: Record = { pass: '✓', fail: '✗', skip: '·' }; @@ -692,8 +1089,16 @@ function printCase(run: CaseRun, schemaName: string): void { console.log(`\n${head}\n · skipped (${run.checks[0]?.detail ?? 'server invariant'})`); return; } + if (run.status === 'broken') { + console.log(`\n${head}\n ✗ broken case: ${run.checks[0]?.detail}`); + return; + } if (run.status === 'llm_failed') { console.log(`\n${head}\n ✗ LLM error: ${run.llmErrorCode} — ${run.llmErrorMessage}`); + // A second call that failed still has the first call's checks. + for (const c of run.checks) { + console.log(` ${ICON[c.outcome]} ${c.key}${c.detail ? `: ${c.detail}` : ''}`); + } return; } if (run.status === 'parse_failed') { @@ -708,6 +1113,10 @@ function printCase(run: CaseRun, schemaName: string): void { const fail = run.checks.filter((c) => c.outcome === 'fail').length; const skipN = run.checks.filter((c) => c.outcome === 'skip').length; console.log(`\n${head} (${run.tokens?.ms}ms) ${pass}✓ ${fail}✗ ${skipN}·`); + if (run.questions) { + const options = run.questions.whatOptions.map((o) => JSON.stringify(o)).join(', '); + console.log(` asked: ${run.questions.topics.join(', ')} options: ${options}`); + } if (run.draft) { console.log( ` title="${run.draft.title.slice(0, 60)}" fields=${run.draft.fields.length} slots=${run.draft.slots.length} groupBy=${run.draft.groupBy ?? 'null'}`, @@ -730,13 +1139,17 @@ function printSummary(runs: CaseRun[]): void { const skipped = runs.filter((r) => r.status === 'skipped').length; const llmFailed = runs.filter((r) => r.status === 'llm_failed').length; const parseFailed = runs.filter((r) => r.status === 'parse_failed').length; + const brokenN = runs.filter((r) => r.status === 'broken').length; const allChecks = runs.flatMap((r) => r.checks); const totalPass = allChecks.filter((c) => c.outcome === 'pass').length; const totalFail = allChecks.filter((c) => c.outcome === 'fail').length; const totalSkip = allChecks.filter((c) => c.outcome === 'skip').length; console.log('\n──────────────────────────────────────────────'); - console.log(`Cases: ${total} ran=${llmOk} skipped=${skipped} llm_failed=${llmFailed} parse_failed=${parseFailed}`); + console.log( + `Cases: ${total} ran=${llmOk} skipped=${skipped} llm_failed=${llmFailed} parse_failed=${parseFailed}` + + (brokenN > 0 ? ` broken=${brokenN}` : ''), + ); console.log(`Checks: ${totalPass}✓ ${totalFail}✗ ${totalSkip}·`); console.log('──────────────────────────────────────────────'); } @@ -774,7 +1187,9 @@ export async function runEvalCli(runner: EvalRunner): Promise { process.exit(2); } - console.log(`Running ${cases.length} case(s) (${llmCount} llm, ${cases.length - llmCount} server)…`); + console.log( + `Running ${cases.length} case(s) (${llmCount} llm, ${cases.length - llmCount} server)…`, + ); const runs: CaseRun[] = []; for (const c of cases) { const run = await runOneCase(runner, c); @@ -791,8 +1206,8 @@ export async function runEvalCli(runner: EvalRunner): Promise { const anyFail = runs.some( (r) => r.status === 'llm_failed' || + r.status === 'broken' || (r.status !== 'skipped' && r.checks.some((c) => c.outcome === 'fail')), ); process.exit(anyFail ? 1 : 0); } - diff --git a/src/lib/magic-compose/__golden__/system-prompt.txt b/src/lib/magic-compose/__golden__/system-prompt.txt index 1f875767..519cafd5 100644 --- a/src/lib/magic-compose/__golden__/system-prompt.txt +++ b/src/lib/magic-compose/__golden__/system-prompt.txt @@ -9,11 +9,19 @@ Output schema (return ONLY this JSON object, no prose, no fences): { "ref": "", "label": "", "fieldType": "text|date|time|number|enum", "choices": [] } ], "slots": [ - { "values": { "": , ... }, "capacity": } + { "values": { "": , ... }, "capacity": } ], "groupBy": "" } +QUESTIONS — only when the prompt doesn't say what people sign up for (rule 4), return ONLY this object instead of a draft: + +{ "questions": { "topics": ["what", "when", "how_many"], "whatOptions": ["