From 98fc4d491b9d791d9bcfab1a86a1f0834d0f15f1 Mon Sep 17 00:00:00 2001 From: kartsan03 Date: Mon, 28 Sep 2026 12:51:41 +0000 Subject: [PATCH 1/3] fix(ai-anthropic, ai-gemini, ai-mistral, ai-ollama, ai-bedrock): report max-token truncation in structuredOutput --- ...ctured-output-truncation-other-adapters.md | 9 +++ packages/ai-anthropic/src/adapters/text.ts | 8 +++ .../tests/anthropic-adapter.test.ts | 36 ++++++++++++ .../ai-bedrock/src/adapters/converse-text.ts | 23 ++++++++ .../ai-bedrock/tests/converse/adapter.test.ts | 55 +++++++++++++++++++ packages/ai-gemini/src/adapters/text.ts | 8 +++ .../ai-gemini/tests/gemini-adapter.test.ts | 22 ++++++++ packages/ai-mistral/src/adapters/text.ts | 8 +++ .../ai-mistral/tests/mistral-adapter.test.ts | 22 ++++++++ packages/ai-ollama/src/adapters/text.ts | 8 +++ packages/ai-ollama/tests/text-adapter.test.ts | 17 ++++++ 11 files changed, 216 insertions(+) create mode 100644 .changeset/structured-output-truncation-other-adapters.md diff --git a/.changeset/structured-output-truncation-other-adapters.md b/.changeset/structured-output-truncation-other-adapters.md new file mode 100644 index 0000000000..a7d4b56d92 --- /dev/null +++ b/.changeset/structured-output-truncation-other-adapters.md @@ -0,0 +1,9 @@ +--- +'@tanstack/ai-anthropic': patch +'@tanstack/ai-gemini': patch +'@tanstack/ai-mistral': patch +'@tanstack/ai-ollama': patch +'@tanstack/ai-bedrock': patch +--- + +Report a structured-output response cut off at the output token limit as truncation instead of a JSON parse or schema error, as `openai-base` and `ai-openrouter` already do (#1426). diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index b84a7a65c4..edc10d4c33 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -477,6 +477,14 @@ export class AnthropicTextAdapter< }, ) + // A forced tool call cut off at the output cap carries partial input, + // which would read like a schema failure (issue #1426). + if (response.stop_reason === 'max_tokens') { + throw new Error( + 'anthropic.structuredOutput: the response was cut off because the maximum token limit was reached (stop_reason=max_tokens); raise modelOptions.max_tokens', + ) + } + // Extract the tool use content from the response let parsed: unknown = null let rawText = '' diff --git a/packages/ai-anthropic/tests/anthropic-adapter.test.ts b/packages/ai-anthropic/tests/anthropic-adapter.test.ts index b13ef7bec8..e98ffbc7bf 100644 --- a/packages/ai-anthropic/tests/anthropic-adapter.test.ts +++ b/packages/ai-anthropic/tests/anthropic-adapter.test.ts @@ -6,6 +6,7 @@ import { type AdapterYieldChunk, type UIMessage, } from '@tanstack/ai' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' import { createAnthropicChatWithClient } from '../src' import { AnthropicTextAdapter } from '../src/adapters/text' import type { AnthropicTextProviderOptions } from '../src/adapters/text' @@ -825,6 +826,41 @@ describe('Anthropic adapter option mapping', () => { expect(payload.max_tokens).toBeLessThanOrEqual(21_333) }) + it('reports a max_tokens stop as truncation in structuredOutput', async () => { + // A forced tool call cut off at the cap arrives with partial input. + mocks.betaMessagesCreate.mockResolvedValueOnce({ + id: 'msg_structured', + type: 'message', + role: 'assistant', + model: 'claude-opus-4-1', + content: [ + { + type: 'tool_use', + id: 'toolu_structured_output', + name: 'structured_output', + input: {}, + }, + ], + stop_reason: 'max_tokens', + usage: { input_tokens: 10, output_tokens: 100 }, + }) + + await expect( + createAdapter('claude-opus-4-1').structuredOutput({ + chatOptions: { + model: 'claude-opus-4-1', + messages: [{ role: 'user', content: 'recommend a guitar as json' }], + logger: resolveDebugOption(false), + }, + outputSchema: { + type: 'object', + properties: { recommendation: { type: 'string' } }, + required: ['recommendation'], + }, + }), + ).rejects.toThrow(/maximum token limit was reached/) + }) + it('native combined mode (#605): wires outputSchema into output_format alongside tools on Claude 4.5+', async () => { // Final-turn JSON the model emits when output_format is in play. const finalJson = JSON.stringify({ city: 'Berlin', temp: 18 }) diff --git a/packages/ai-bedrock/src/adapters/converse-text.ts b/packages/ai-bedrock/src/adapters/converse-text.ts index fb176f08d0..89f128e392 100644 --- a/packages/ai-bedrock/src/adapters/converse-text.ts +++ b/packages/ai-bedrock/src/adapters/converse-text.ts @@ -291,6 +291,13 @@ export class BedrockConverseTextAdapter< toolConfig: buildStructuredToolConfig(outputSchema), } const res = await this.send(input) + // A forced tool call cut off at the output cap carries partial input, + // which would read like a schema failure (issue #1426). + if (res.stopReason === 'max_tokens') { + throw new Error( + `${this.name}.structuredOutput: the response was cut off because the maximum token limit was reached (stopReason=max_tokens); raise modelOptions.max_completion_tokens`, + ) + } const structured = extractStructuredToolInput(res) if (structured === undefined) { throw new Error( @@ -439,6 +446,22 @@ export class BedrockConverseTextAdapter< } } + // Same truncation check as `structuredOutput()`: report the token limit + // before the empty-content and parse errors (issue #1426). + if (finishReason === 'length') { + const message = `${this.name}.structuredOutputStream: the response was cut off because the maximum token limit was reached (stopReason=max_tokens); raise modelOptions.max_completion_tokens` + yield { + type: EventType.RUN_ERROR, + runId, + model: chatOptions.model, + timestamp: Date.now(), + message, + code: 'max_tokens', + error: { message, code: 'max_tokens' }, + } + return + } + if (accumulatedRaw.length === 0) { yield { type: EventType.RUN_ERROR, diff --git a/packages/ai-bedrock/tests/converse/adapter.test.ts b/packages/ai-bedrock/tests/converse/adapter.test.ts index 2fa8ccd20c..12d7ef19c4 100644 --- a/packages/ai-bedrock/tests/converse/adapter.test.ts +++ b/packages/ai-bedrock/tests/converse/adapter.test.ts @@ -278,6 +278,61 @@ describe('BedrockConverseTextAdapter', () => { ).toBe('stop') }) + it('reports a max_tokens stop as truncation in structuredOutput', async () => { + const a = new StubAdapter({ apiKey: 'k' }, 'us.amazon.nova-pro-v1:0') + a.nonStreamOutput = { + output: { + message: { + role: 'assistant', + content: [ + { + toolUse: { toolUseId: 's', name: 'structured_output', input: {} }, + }, + ], + }, + }, + stopReason: 'max_tokens', + } as unknown as ConverseCommandOutput + await expect( + a.structuredOutput({ + chatOptions: textOptions({ + messages: [{ role: 'user', content: 'go' }], + }), + outputSchema: { type: 'object', properties: { n: { type: 'number' } } }, + }), + ).rejects.toThrow(/maximum token limit was reached/) + }) + + it('reports a max_tokens stop as truncation in structuredOutputStream', async () => { + const a = new StubAdapter({ apiKey: 'k' }, 'us.amazon.nova-pro-v1:0') + a.streamEvents = [ + { messageStart: { role: 'assistant' } }, + { + contentBlockStart: { + start: { toolUse: { toolUseId: 's', name: 'structured_output' } }, + contentBlockIndex: 0, + }, + }, + { + contentBlockDelta: { + delta: { toolUse: { input: '{"n":' } }, + contentBlockIndex: 0, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { messageStop: { stopReason: 'max_tokens' } }, + ] + const events: Array = [] + for await (const c of a.structuredOutputStream({ + chatOptions: textOptions({ messages: [{ role: 'user', content: 'go' }] }), + outputSchema: { type: 'object', properties: { n: { type: 'number' } } }, + })) { + events.push(c) + } + const runError = events.find((e) => e.type === EventType.RUN_ERROR) + expect((runError as { code?: string } | undefined)?.code).toBe('max_tokens') + }) + it('folds trailing metadata usage into the structuredOutputStream RUN_FINISHED (#1278)', async () => { const a = new StubAdapter({ apiKey: 'k' }, 'us.amazon.nova-pro-v1:0') a.streamEvents = [ diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index 13828b6f10..28418ce6a1 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -441,6 +441,14 @@ export class GeminiTextAdapter< }, }) + // A response cut off at the output cap is a truncated JSON document; + // report it before the parse error (issue #1426). + if (result.candidates?.[0]?.finishReason === FinishReason.MAX_TOKENS) { + throw new Error( + 'gemini.structuredOutput: the response was cut off because the maximum token limit was reached (finishReason=MAX_TOKENS); raise modelOptions.maxOutputTokens', + ) + } + // Extract text content from the response const rawText = this.extractTextFromResponse(result) diff --git a/packages/ai-gemini/tests/gemini-adapter.test.ts b/packages/ai-gemini/tests/gemini-adapter.test.ts index 3cebcec628..83e7c103bd 100644 --- a/packages/ai-gemini/tests/gemini-adapter.test.ts +++ b/packages/ai-gemini/tests/gemini-adapter.test.ts @@ -1403,6 +1403,28 @@ describe('GeminiAdapter through AI', () => { expect(structuredCompleteEvent(events)).toBeDefined() }) + it('reports a MAX_TOKENS stop as truncation in structuredOutput', async () => { + mocks.generateContentSpy.mockResolvedValue({ + candidates: [ + { + content: { parts: [{ text: '{"city":' }] }, + finishReason: 'MAX_TOKENS', + }, + ], + }) + + await expect( + createTextAdapter().structuredOutput({ + chatOptions: { + model: 'gemini-2.5-pro', + messages: [{ role: 'user', content: 'Return a city' }], + logger: testLogger, + }, + outputSchema: cityJsonSchema, + }), + ).rejects.toThrow(/maximum token limit was reached/) + }) + it('emits empty-response when the structured stream has no text', async () => { mocks.generateContentStreamSpy.mockResolvedValue( createStream([ diff --git a/packages/ai-mistral/src/adapters/text.ts b/packages/ai-mistral/src/adapters/text.ts index d9dbb70eb4..6cb94d4975 100644 --- a/packages/ai-mistral/src/adapters/text.ts +++ b/packages/ai-mistral/src/adapters/text.ts @@ -268,6 +268,14 @@ export class MistralTextAdapter< }, }) + // A response cut off at the output cap is a truncated JSON document; + // report it before the parse error (issue #1426). + if (response.choices[0]?.finishReason === 'length') { + throw new Error( + 'mistral.structuredOutput: the response was cut off because the maximum token limit was reached (finish_reason=length); raise modelOptions.max_tokens', + ) + } + const rawText = response.choices[0]?.message?.content const textContent = typeof rawText === 'string' ? rawText : '' diff --git a/packages/ai-mistral/tests/mistral-adapter.test.ts b/packages/ai-mistral/tests/mistral-adapter.test.ts index 0027fffe2c..a4d63e1654 100644 --- a/packages/ai-mistral/tests/mistral-adapter.test.ts +++ b/packages/ai-mistral/tests/mistral-adapter.test.ts @@ -234,6 +234,28 @@ describe('Mistral adapters', () => { ) }) + it('reports a length stop as truncation in structured output', async () => { + mockComplete.mockReset().mockResolvedValue({ + choices: [ + { message: { content: '{"value":' }, finishReason: 'length' }, + ], + }) + const adapter = createMistralText('mistral-large-latest', 'test-api-key') + + await expect( + adapter.structuredOutput({ + chatOptions: chatOpts({ + model: 'mistral-large-latest', + messages: [{ role: 'user', content: 'Return structured output' }], + }), + outputSchema: { + type: 'object', + properties: { value: { type: 'string' } }, + }, + }), + ).rejects.toThrow(/maximum token limit was reached/) + }) + it('throws if MISTRAL_API_KEY is not set when using mistralText', () => { vi.stubEnv('MISTRAL_API_KEY', '') diff --git a/packages/ai-ollama/src/adapters/text.ts b/packages/ai-ollama/src/adapters/text.ts index 48a9c6a35f..7a458785dd 100644 --- a/packages/ai-ollama/src/adapters/text.ts +++ b/packages/ai-ollama/src/adapters/text.ts @@ -170,6 +170,14 @@ export class OllamaTextAdapter extends BaseTextAdapter< format: outputSchema, }) + // A response cut off at the output cap is a truncated JSON document; + // report it before the parse error (issue #1426). + if (response.done_reason === 'length') { + throw new Error( + 'ollama.structuredOutput: the response was cut off because the maximum token limit was reached (done_reason=length); raise modelOptions.options.num_predict', + ) + } + const rawText = response.message.content // Parse the JSON response diff --git a/packages/ai-ollama/tests/text-adapter.test.ts b/packages/ai-ollama/tests/text-adapter.test.ts index 11a450e8ee..c12f2401aa 100644 --- a/packages/ai-ollama/tests/text-adapter.test.ts +++ b/packages/ai-ollama/tests/text-adapter.test.ts @@ -367,6 +367,23 @@ describe('OllamaTextAdapter.structuredOutput', () => { ).rejects.toThrow(/Failed to parse structured output/) }) + it('reports a length stop as truncation, not a parse failure', async () => { + chatMock.mockResolvedValueOnce({ + message: { role: 'assistant', content: '{"result":' }, + done_reason: 'length', + }) + const adapter = createOllamaChat('llama3.2') + await expect( + adapter.structuredOutput({ + chatOptions: { + logger: testLogger, + messages: [{ role: 'user', content: 'q' }], + }, + outputSchema: { type: 'object', properties: {} }, + } as any), + ).rejects.toThrow(/maximum token limit was reached/) + }) + it('surfaces upstream errors as structured-output errors', async () => { chatMock.mockRejectedValueOnce(new Error('network down')) const adapter = createOllamaChat('llama3.2') From adbd60cd61fede3f24de65df023fb90d6539646b Mon Sep 17 00:00:00 2001 From: kartsan03 Date: Tue, 29 Sep 2026 19:53:33 +0000 Subject: [PATCH 2/3] test(e2e): cover structured-output truncation for Anthropic, Mistral and Ollama --- .../api.structured-output-length-wire.ts | 122 +++++++++++++++++- .../structured-output-length-wire.spec.ts | 3 + 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/testing/e2e/src/routes/api.structured-output-length-wire.ts b/testing/e2e/src/routes/api.structured-output-length-wire.ts index a512ae1e2a..9fd73d50f5 100644 --- a/testing/e2e/src/routes/api.structured-output-length-wire.ts +++ b/testing/e2e/src/routes/api.structured-output-length-wire.ts @@ -1,8 +1,13 @@ +import { createServer } from 'node:http' import { createFileRoute } from '@tanstack/react-router' import { chat, createChatOptions } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { createMistralText } from '@tanstack/ai-mistral' +import { createOllamaChat } from '@tanstack/ai-ollama' import { openaiCompatible } from '@tanstack/ai-openai/compatible' import { createOpenRouterText } from '@tanstack/ai-openrouter' import { HTTPClient } from '@openrouter/sdk' +import type { AddressInfo } from 'node:net' const DUMMY_KEY = 'sk-e2e-test-dummy-key' @@ -40,11 +45,36 @@ async function errorOf(run: () => Promise): Promise { } } +/** Answers every request with `body`, for clients that take no custom `fetch`. */ +async function withServer( + body: unknown, + run: (baseURL: string) => Promise, +): Promise { + const server = createServer((req, res) => { + req.resume().on('end', () => { + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify(body)) + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + return await run( + `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + ) + } finally { + server.close() + server.closeAllConnections() + } +} + /** * `chat({ outputSchema })` on a response that stopped at the output cap - * (`finish_reason: "length"`). `chat()` takes the adapter's streaming - * structured-output path. A scripted `fetch` returns a truncated JSON - * document and, for a reasoning model that spent the budget, empty content. + * (`finish_reason: "length"`). For OpenAI-compatible and OpenRouter, + * `chat()` takes the adapter's streaming structured-output path. A scripted + * `fetch` returns a truncated JSON document and, for a reasoning model that + * spent the budget, empty content. Anthropic, Mistral and Ollama have no + * streaming structured output, so `chat()` calls their `structuredOutput()`; + * Mistral and Ollama take no custom `fetch` and get a local server instead. * The companion spec checks that each adapter reports the token limit * instead of a JSON parse or empty-content error. No aimock fixture is used. */ @@ -86,6 +116,92 @@ export const Route = createFileRoute('/api/structured-output-length-wire')({ }), ) } + + // A pre-4.5 model takes the forced-tool `structuredOutput()` path + // (4.5+ use the native combined path). The forced tool call stopped + // at max_tokens, with partial input. + const anthropic = createAnthropicChat('claude-opus-4-1', DUMMY_KEY, { + fetch: async () => + Response.json({ + id: 'msg_length', + type: 'message', + role: 'assistant', + model: 'claude-opus-4-1', + content: [ + { + type: 'tool_use', + id: 'toolu_length', + name: 'structured_output', + input: {}, + }, + ], + stop_reason: 'max_tokens', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 5 }, + }), + }) + errors['anthropic-truncated'] = await errorOf(() => + chat({ + ...createChatOptions({ adapter: anthropic }), + messages: [{ role: 'user', content: 'Give me a title' }], + outputSchema, + }), + ) + + errors['mistral-truncated'] = await withServer( + { + id: 'cmpl-length', + object: 'chat.completion', + created: 1, + model: 'mistral-large-latest', + choices: [ + { + index: 0, + message: { role: 'assistant', content: '{"title":"cut o' }, + finish_reason: 'length', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, + }, + (baseURL) => + errorOf(() => + chat({ + ...createChatOptions({ + adapter: createMistralText( + 'mistral-large-latest', + DUMMY_KEY, + { baseURL }, + ), + }), + messages: [{ role: 'user', content: 'Give me a title' }], + outputSchema, + }), + ), + ) + + errors['ollama-truncated'] = await withServer( + { + model: 'mistral', + created_at: '2026-01-01T00:00:00Z', + message: { role: 'assistant', content: '{"title":"cut o' }, + done: true, + done_reason: 'length', + }, + (baseURL) => + errorOf(() => + chat({ + ...createChatOptions({ + adapter: createOllamaChat('mistral', { baseURL }), + }), + messages: [{ role: 'user', content: 'Give me a title' }], + outputSchema, + }), + ), + ) return Response.json({ errors }) }, }, diff --git a/testing/e2e/tests/structured-output-length-wire.spec.ts b/testing/e2e/tests/structured-output-length-wire.spec.ts index 29fb57ba3b..a4cbe00e5a 100644 --- a/testing/e2e/tests/structured-output-length-wire.spec.ts +++ b/testing/e2e/tests/structured-output-length-wire.spec.ts @@ -14,8 +14,11 @@ test.describe('structured output — finish_reason=length', () => { } expect(Object.keys(errors).sort()).toEqual([ + 'anthropic-truncated', 'compatible-empty', 'compatible-truncated', + 'mistral-truncated', + 'ollama-truncated', 'openrouter-empty', 'openrouter-truncated', ]) From eddc70c60dac158a61e22ac80ce6ae0e51e16739 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Wed, 7 Oct 2026 17:54:58 +0200 Subject: [PATCH 3/3] chore(changeset): use shorter sentences in the truncation changeset --- .changeset/structured-output-truncation-other-adapters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/structured-output-truncation-other-adapters.md b/.changeset/structured-output-truncation-other-adapters.md index a7d4b56d92..0e6fe41be8 100644 --- a/.changeset/structured-output-truncation-other-adapters.md +++ b/.changeset/structured-output-truncation-other-adapters.md @@ -6,4 +6,4 @@ '@tanstack/ai-bedrock': patch --- -Report a structured-output response cut off at the output token limit as truncation instead of a JSON parse or schema error, as `openai-base` and `ai-openrouter` already do (#1426). +Structured output now reports a truncation error when the response stops at the output token limit. Before, you got a JSON parse error, or the partial result came back as valid data. `openai-base` and `ai-openrouter` already do this (#1426).