Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions packages/llm/src/protocols/anthropic-messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,12 @@ const AnthropicStreamDelta = Schema.Struct({
const AnthropicEvent = Schema.Struct({
type: Schema.String,
index: Schema.optional(Schema.Number),
message: Schema.optional(Schema.Struct({ usage: Schema.optional(AnthropicUsage) })),
// `model` on message_start is the model that actually served the response.
// For a routed model (Fireworks FireRouter over the Anthropic-compatible
// route) the request names a route and this names the member that answered.
message: Schema.optional(
Schema.Struct({ usage: Schema.optional(AnthropicUsage), model: Schema.optional(Schema.String) }),
),
content_block: Schema.optional(AnthropicStreamBlock),
delta: Schema.optional(AnthropicStreamDelta),
usage: Schema.optional(AnthropicUsage),
Expand All @@ -222,6 +227,8 @@ type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>

interface ParserState {
readonly tools: ToolStream.State<number>
/** Model the provider reported on message_start; see AnthropicEvent.message.model. */
readonly responseModelID?: string
readonly usage?: Usage
readonly lifecycle: Lifecycle.State
}
Expand Down Expand Up @@ -651,7 +658,13 @@ const NO_EVENTS: StepResult["1"] = []

const onMessageStart = (state: ParserState, event: AnthropicEvent): StepResult => {
const usage = mapUsage(event.message?.usage)
return [usage ? { ...state, usage: mergeUsage(state.usage, usage) } : state, NO_EVENTS]
const served = event.message?.model?.trim()
const responseModelID = served && served.length > 0 ? served : state.responseModelID
const next =
usage || responseModelID !== state.responseModelID
? { ...state, usage: usage ? mergeUsage(state.usage, usage) : state.usage, responseModelID }
: state
return [next, NO_EVENTS]
}

const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepResult => {
Expand Down Expand Up @@ -785,6 +798,7 @@ const onMessageDelta = (state: ParserState, event: AnthropicEvent): StepResult =
const lifecycle = Lifecycle.finish(state.lifecycle, events, {
reason: mapFinishReason(event.delta?.stop_reason),
usage,
responseModelID: state.responseModelID,
providerMetadata: event.delta?.stop_sequence
? anthropicMetadata({ stopSequence: event.delta.stop_sequence })
: undefined,
Expand Down
9 changes: 9 additions & 0 deletions packages/llm/src/protocols/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,11 +132,16 @@ const GeminiCandidate = Schema.Struct({
const GeminiEvent = Schema.Struct({
candidates: optionalArray(GeminiCandidate),
usageMetadata: Schema.optional(GeminiUsage),
// Gemini reports the serving model as `modelVersion`, which can be more
// specific than the requested alias (a dated build behind a floating name).
modelVersion: Schema.optional(Schema.String),
})
type GeminiEvent = Schema.Schema.Type<typeof GeminiEvent>

interface ParserState {
readonly finishReason?: string
/** Serving model from the response's `modelVersion`, when reported. */
readonly responseModelID?: string
readonly hasToolCalls: boolean
readonly nextToolCallId: number
readonly usage?: Usage
Expand Down Expand Up @@ -391,15 +396,19 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> =>
Lifecycle.finish(lifecycle, events, {
reason: mapFinishReason(state.finishReason, state.hasToolCalls),
usage: state.usage,
responseModelID: state.responseModelID,
})
return events
})()
: []

const step = (state: ParserState, event: GeminiEvent) => {
const servedModel = event.modelVersion?.trim()
const nextState = {
...state,
usage: event.usageMetadata ? (mapUsage(event.usageMetadata) ?? state.usage) : state.usage,
// Later chunks may omit it; keep the first non-empty value.
responseModelID: servedModel && servedModel.length > 0 ? servedModel : state.responseModelID,
}
const candidate = event.candidates?.[0]
if (!candidate?.content)
Expand Down
20 changes: 19 additions & 1 deletion packages/llm/src/protocols/openai-chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ const OpenAIChatChoice = Schema.Struct({
const OpenAIChatEvent = Schema.Struct({
choices: Schema.Array(OpenAIChatChoice),
usage: optionalNull(OpenAIChatUsage),
// The model that actually served the response. For a ROUTED model (Fireworks
// FireRouter, Azure model-router) the request names a route and this names the
// member the router chose, so it is the only field that says who answered.
model: optionalNull(Schema.String),
})
type OpenAIChatEvent = Schema.Schema.Type<typeof OpenAIChatEvent>
type OpenAIChatRequestMessage = LLMRequest["messages"][number]
Expand All @@ -166,6 +170,8 @@ interface ParserState {
readonly usage?: Usage
readonly finishReason?: FinishReason
readonly lifecycle: Lifecycle.State
/** Model the provider reported as serving this response; see OpenAIChatEvent.model. */
readonly responseModelID?: string
}

const invalid = ProviderShared.invalidRequest
Expand Down Expand Up @@ -408,6 +414,12 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
Effect.gen(function* () {
const events: LLMEvent[] = []
const usage = mapUsage(event.usage) ?? state.usage
// Chunks after the first frequently omit `model`; keep the first non-empty
// value rather than letting a later empty chunk erase it. Trim and reject
// blanks: an empty string is not an identity, and encoding it as one would
// let a reader treat "" as a real model instead of falling back.
const servedModel = event.model?.trim()
const responseModelID = servedModel && servedModel.length > 0 ? servedModel : state.responseModelID
const choice = event.choices[0]
const finishReason = choice?.finish_reason ? mapFinishReason(choice.finish_reason) : state.finishReason
const delta = choice?.delta
Expand Down Expand Up @@ -454,6 +466,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
usage,
finishReason,
lifecycle,
responseModelID,
},
events,
] as const
Expand All @@ -465,7 +478,12 @@ const finishEvents = (state: ParserState): ReadonlyArray<LLMEvent> => {
const reason = state.finishReason === "stop" && hasToolCalls ? "tool-calls" : state.finishReason
const lifecycle = state.toolCallEvents.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
events.push(...state.toolCallEvents)
if (reason) Lifecycle.finish(lifecycle, events, { reason, usage: state.usage })
if (reason)
Lifecycle.finish(lifecycle, events, {
reason,
usage: state.usage,
responseModelID: state.responseModelID,
})
return events
}

Expand Down
5 changes: 5 additions & 0 deletions packages/llm/src/protocols/openai-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,9 @@ const OpenAIResponsesEvent = Schema.Struct({
Schema.StructWithRest(
Schema.Struct({
id: Schema.optional(Schema.String),
// The model that actually served the response. For a ROUTED model the
// request names a route and this names the member that answered.
model: optionalNull(Schema.String),
service_tier: optionalNull(Schema.String),
incomplete_details: optionalNull(Schema.Struct({ reason: Schema.String })),
usage: optionalNull(OpenAIResponsesUsage),
Expand Down Expand Up @@ -874,9 +877,11 @@ const onOutputItemDone = Effect.fn("OpenAIResponses.onOutputItemDone")(function*

const onResponseFinish = (state: ParserState, event: OpenAIResponsesEvent): StepResult => {
const events: LLMEvent[] = []
const served = event.response?.model?.trim()
const lifecycle = Lifecycle.finish(state.lifecycle, events, {
reason: mapFinishReason(event, state.hasFunctionCall),
usage: mapUsage(event.response?.usage),
responseModelID: served && served.length > 0 ? served : undefined,
providerMetadata:
event.response?.id || event.response?.service_tier
? openaiMetadata({
Expand Down
7 changes: 7 additions & 0 deletions packages/llm/src/protocols/utils/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ export const finish = (
readonly reason: FinishReason
readonly usage?: Usage
readonly providerMetadata?: ProviderMetadata
/**
* The model the provider reported as serving this response, when the wire
* format carries one. Differs from the REQUESTED model for routed models,
* where the request names a route and the provider picks a member.
*/
readonly responseModelID?: string
},
): State => {
const stepped = closeOpenBlocks(stepStart(state, events), events)
Expand All @@ -93,6 +99,7 @@ export const finish = (
reason: input.reason,
usage: input.usage,
providerMetadata: input.providerMetadata,
responseModelID: input.responseModelID,
}),
LLMEvent.finish(input),
)
Expand Down
15 changes: 15 additions & 0 deletions packages/llm/src/schema/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,19 @@ export const StepFinish = Schema.Struct({
reason: FinishReason,
usage: Schema.optional(Usage),
providerMetadata: Schema.optional(ProviderMetadata),
/**
* The model the PROVIDER reports as having served this step, when it reports one.
*
* This is NOT the requested model. For a router (Fireworks FireRouter, Azure
* model-router) the request names a route and the provider picks a member per
* request, so the two differ and only this one says who actually answered.
* A router may also pick differently between steps of a single turn, which is
* why it belongs on the step rather than only on the finish.
*
* Optional because not every provider or runtime reports it; absent means
* "not reported", never "same as requested".
*/
responseModelID: Schema.optional(Schema.String),
}).annotate({ identifier: "LLM.Event.StepFinish" })
export type StepFinish = Schema.Schema.Type<typeof StepFinish>

Expand All @@ -194,6 +207,8 @@ export const Finish = Schema.Struct({
reason: FinishReason,
usage: Schema.optional(Usage),
providerMetadata: Schema.optional(ProviderMetadata),
/** See {@link StepFinish.responseModelID}. The model serving the final step. */
responseModelID: Schema.optional(Schema.String),
}).annotate({ identifier: "LLM.Event.Finish" })
export type Finish = Schema.Schema.Type<typeof Finish>

Expand Down
64 changes: 64 additions & 0 deletions packages/llm/test/provider/openai-chat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,70 @@ describe("OpenAI Chat route", () => {
}),
)

// A ROUTED model (Fireworks FireRouter, Azure model-router) is requested by route
// and served by a member the router picks per request, so the response `model`
// field is the only thing that says who actually answered. Verified on the wire:
// `firerouter/glm-5p3/glm-5p3-flash` returns model `glm-5p3-flash`.
it.effect("reports the served model from the response when it differs from the request", () =>
Effect.gen(function* () {
const served = { id: "chatcmpl_fixture", model: "glm-5p3-flash", choices: [], usage: null }
const body = sseEvents(
served,
deltaChunk({ role: "assistant", content: "ok" }),
deltaChunk({}, "stop"),
)
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
const stepFinish = response.events.find((event) => event.type === "step-finish")
const finish = response.events.find((event) => event.type === "finish")
// The REQUESTED model is gpt-4o-mini; the SERVED model must not be assumed equal to it.
expect(model.id as string).toBe("gpt-4o-mini")
expect(stepFinish).toMatchObject({ responseModelID: "glm-5p3-flash" })
expect(finish).toMatchObject({ responseModelID: "glm-5p3-flash" })
}),
)

// Absent must mean "the provider did not report one", never "same as requested":
// a reader that cannot tell those apart will silently price a routed turn wrong.
it.effect("leaves the served model unset when the provider reports none", () =>
Effect.gen(function* () {
const body = sseEvents(deltaChunk({ role: "assistant", content: "ok" }), deltaChunk({}, "stop"))
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
const stepFinish = response.events.find((event) => event.type === "step-finish")
expect((stepFinish as { responseModelID?: string }).responseModelID).toBeUndefined()
}),
)

// An empty string is not an identity. Encoding it as one would let a reader treat
// "" as a real model and stop falling back to the requested id.
it.effect("treats an empty served model as not reported", () =>
Effect.gen(function* () {
const body = sseEvents(
{ id: "chatcmpl_fixture", model: " ", choices: [], usage: null },
deltaChunk({ role: "assistant", content: "ok" }),
deltaChunk({}, "stop"),
)
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
const stepFinish = response.events.find((event) => event.type === "step-finish")
expect((stepFinish as { responseModelID?: string }).responseModelID).toBeUndefined()
}),
)

// Only the first chunk carries `model` in practice; a later chunk without it must
// not erase what was already reported.
it.effect("keeps the served model when later chunks omit it", () =>
Effect.gen(function* () {
const body = sseEvents(
{ id: "chatcmpl_fixture", model: "glm-5p3-flash", choices: [], usage: null },
deltaChunk({ role: "assistant", content: "a" }),
deltaChunk({ content: "b" }),
deltaChunk({}, "stop"),
)
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
const stepFinish = response.events.find((event) => event.type === "step-finish")
expect(stepFinish).toMatchObject({ responseModelID: "glm-5p3-flash" })
}),
)

it.effect("parses text and usage stream fixtures", () =>
Effect.gen(function* () {
const body = sseEvents(
Expand Down
26 changes: 26 additions & 0 deletions packages/opencode/src/session/llm/ai-sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,31 @@ export function adapterState() {
currentReasoningID: undefined as string | undefined,
toolNames: {} as Record<string, string>,
copilotTotalNanoAiu: undefined as number | undefined,
// Last model the provider reported as serving a step. The AI SDK puts
// response.modelId on `finish-step` but NOT on `finish`, so `finish` can only
// report a served model by carrying the last step's value forward.
lastResponseModelID: undefined as string | undefined,
}
}

/**
* The model the provider says served this step, from the AI SDK's
* `finish-step` response metadata.
*
* Read defensively rather than off the type: a provider that omits the field, or
* returns an empty string, must yield undefined ("not reported") instead of an
* empty identity that a reader could mistake for a real model.
*/
function responseModelID(value: unknown): string | undefined {
if (!value || typeof value !== "object") return undefined
const response = (value as Record<string, unknown>).response
if (!response || typeof response !== "object") return undefined
const id = (response as Record<string, unknown>).modelId
if (typeof id !== "string") return undefined
const trimmed = id.trim()
return trimmed.length > 0 ? trimmed : undefined
}

function finishReason(value: string | undefined): FinishReason {
return Schema.is(FinishReason)(value) ? value : "unknown"
}
Expand Down Expand Up @@ -101,12 +123,15 @@ export function toLLMEvents(
},
}
state.copilotTotalNanoAiu = undefined
const served = responseModelID(event)
if (served !== undefined) state.lastResponseModelID = served
return [
LLMEvent.stepFinish({
index: state.step++,
reason: finishReason(event.finishReason),
usage: usage(event.usage),
providerMetadata: metadata,
responseModelID: served,
}),
]
})
Expand All @@ -118,6 +143,7 @@ export function toLLMEvents(
reason: finishReason(event.finishReason),
usage: usage(event.totalUsage),
providerMetadata: "providerMetadata" in event ? providerMetadata(event.providerMetadata) : undefined,
responseModelID: state.lastResponseModelID,
}),
]
// Reset so the adapter can be reused for a follow-up stream without leaking
Expand Down
10 changes: 10 additions & 0 deletions packages/opencode/src/session/processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -457,13 +457,23 @@ const layer = Layer.effect(
ctx.assistantMessage.finish = value.reason
ctx.assistantMessage.cost += usage.cost
ctx.assistantMessage.tokens = usage.tokens
// Record who actually served this step. Appended as a distinct ordered
// set rather than overwritten: a router can pick a different member per
// step, and keeping only the last would misreport a mixed turn as if one
// model had served all of it.
if (value.responseModelID !== undefined) {
const seen = ctx.assistantMessage.responseModelIDs ?? []
if (!seen.includes(value.responseModelID))
ctx.assistantMessage.responseModelIDs = [...seen, value.responseModelID]
}
yield* session.updatePart({
id: PartID.ascending(),
reason: value.reason,
snapshot: completedSnapshot,
messageID: ctx.assistantMessage.id,
sessionID: ctx.assistantMessage.sessionID,
type: "step-finish",
responseModelID: value.responseModelID,
tokens: usage.tokens,
cost: usage.cost,
})
Expand Down
21 changes: 21 additions & 0 deletions packages/schema/src/v1/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,13 @@ export const StepFinishPart = Schema.Struct({
type: Schema.Literal("step-finish"),
reason: Schema.String,
snapshot: Schema.optional(Schema.String),
/**
* The model the PROVIDER reported as serving THIS step, when it reports one.
* Distinct from the requested model: for a routed model the request names a
* route and the provider picks a member, possibly a different one per step.
* Absent means "not reported", never "same as requested".
*/
responseModelID: Schema.optional(Schema.String),
cost: Schema.Finite,
tokens: Schema.Struct({
total: Schema.optional(Schema.Finite),
Expand Down Expand Up @@ -468,6 +475,20 @@ export const Assistant = Schema.Struct({
root: Schema.String,
}),
summary: Schema.optional(Schema.Boolean),
/**
* Models the PROVIDER reported as serving this turn, in first-seen order.
*
* Distinct from `modelID`, which is what was REQUESTED. For a routed model
* (Fireworks FireRouter, Azure model-router) the request names a route and the
* provider picks a member per request, so `modelID` alone cannot say who
* answered - and pricing a routed turn from `modelID` is wrong by construction.
*
* An ARRAY because a router may pick differently between steps of one turn;
* collapsing that to a single id would silently misreport a mixed turn. Usually
* length 1. Per-step values are on each StepFinishPart. Absent means "not
* reported", never "same as requested".
*/
responseModelIDs: Schema.optional(Schema.Array(Schema.String)),
cost: Schema.Finite,
tokens: Schema.Struct({
total: Schema.optional(Schema.Finite),
Expand Down
Loading