diff --git a/admin/server/ai.go b/admin/server/ai.go index b49f19cc49d3..730b7e2beaeb 100644 --- a/admin/server/ai.go +++ b/admin/server/ai.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "github.com/google/jsonschema-go/jsonschema" @@ -65,24 +66,37 @@ func (s *Server) Complete(ctx context.Context, req *adminv1.CompleteRequest) (*a zap.Int("output_messages", len(res.Message.Content)), zap.Int("input_tokens", res.InputTokens), zap.Int("cached_input_tokens", res.CachedInputTokens), + zap.Int("cache_write_input_tokens", res.CacheWriteInputTokens), zap.Int("output_tokens", res.OutputTokens), observability.ZapCtx(ctx), ) + // Older runtimes don't support reasoning blocks, so remove them unless the client opted in. + responseMessage := res.Message + if !req.IncludeReasoning { + responseMessage = &aiv1.CompletionMessage{ + Role: res.Message.Role, + Data: res.Message.Data, + Content: slices.DeleteFunc(slices.Clone(res.Message.Content), func(block *aiv1.ContentBlock) bool { + return block.GetReasoning() != nil + }), + } + } + // Handle response backwards compatibility: if request used old format, // populate both data and content fields for old runtime compatibility - responseMessage := res.Message if needsBackwardsCompatibleResponse { - responseMessage = convertContentToData(res.Message) + responseMessage = convertContentToData(responseMessage) } // Any tool use response will be passed to the client (the runtime server) for execution. return &adminv1.CompleteResponse{ - Message: responseMessage, - Provider: res.Provider, - InputTokens: uint32(res.InputTokens), - OutputTokens: uint32(res.OutputTokens), - CachedInputTokens: uint32(res.CachedInputTokens), + Message: responseMessage, + Provider: res.Provider, + InputTokens: uint32(res.InputTokens), + OutputTokens: uint32(res.OutputTokens), + CachedInputTokens: uint32(res.CachedInputTokens), + CacheWriteInputTokens: uint32(res.CacheWriteInputTokens), }, nil } diff --git a/admin/server/ai_test.go b/admin/server/ai_test.go new file mode 100644 index 000000000000..62ea1ec4b4d4 --- /dev/null +++ b/admin/server/ai_test.go @@ -0,0 +1,51 @@ +package server + +import ( + "context" + "testing" + + "github.com/rilldata/rill/admin" + adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1" + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/rilldata/rill/runtime/drivers" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestCompleteReasoning(t *testing.T) { + // Older runtimes don't support reasoning blocks, so they're removed unless the client opted in. + s := &Server{ + logger: zap.NewNop(), + admin: &admin.Service{AI: reasoningAIService{}}, + } + for _, includeReasoning := range []bool{false, true} { + res, err := s.Complete(t.Context(), &adminv1.CompleteRequest{ + Messages: []*aiv1.CompletionMessage{{Role: "user", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_Text{Text: "hi"}}}}}, + IncludeReasoning: includeReasoning, + }) + require.NoError(t, err) + if includeReasoning { + require.Len(t, res.Message.Content, 2) + require.Equal(t, "sig", res.Message.Content[0].GetReasoning().GetSignature()) + } else { + require.Len(t, res.Message.Content, 1) + require.Equal(t, "hello", res.Message.Content[0].GetText()) + } + } +} + +// reasoningAIService is a drivers.AIService that responds with reasoning followed by text. +type reasoningAIService struct{} + +func (reasoningAIService) Complete(_ context.Context, _ *drivers.CompleteOptions) (*drivers.CompleteResult, error) { + return &drivers.CompleteResult{ + Message: &aiv1.CompletionMessage{ + Role: "assistant", + Content: []*aiv1.ContentBlock{ + {BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Signature: "sig"}}}, + {BlockType: &aiv1.ContentBlock_Text{Text: "hello"}}, + }, + }, + Provider: "claude", + }, nil +} diff --git a/cli/cmd/admin/start.go b/cli/cmd/admin/start.go index eae8a2666339..26dc83ac2eb2 100644 --- a/cli/cmd/admin/start.go +++ b/cli/cmd/admin/start.go @@ -96,6 +96,7 @@ type Config struct { AIDriver string `default:"" split_words:"true"` OpenAIAPIKey string `envconfig:"openai_api_key"` ClaudeAPIKey string `envconfig:"claude_api_key"` + ClaudePromptCaching bool `envconfig:"claude_prompt_caching"` GeminiAPIKey string `envconfig:"gemini_api_key"` ActivitySinkType string `default:"" split_words:"true"` ActivitySinkKafkaBrokers string `default:"" split_words:"true"` @@ -256,6 +257,7 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command { logger.Fatal("RILL_ADMIN_CLAUDE_API_KEY is required when AI driver is 'claude'") } aiConfig["api_key"] = conf.ClaudeAPIKey + aiConfig["prompt_caching"] = conf.ClaudePromptCaching case "gemini": if conf.GeminiAPIKey == "" { logger.Fatal("RILL_ADMIN_GEMINI_API_KEY is required when AI driver is 'gemini'") diff --git a/docs/docs/developers/build/connectors/services/claude.md b/docs/docs/developers/build/connectors/services/claude.md index 56efe338d7f4..44489f48ccbf 100644 --- a/docs/docs/developers/build/connectors/services/claude.md +++ b/docs/docs/developers/build/connectors/services/claude.md @@ -49,6 +49,17 @@ For details on managing credentials across environments, see [Configure Local Cr For additional configuration options (model, temperature, token limits, etc.), see the [Claude connector reference](/reference/project-files/connectors#claude). +### Prompt caching + +Set `prompt_caching: true` to cache the tools, system prompt, and conversation between requests. AI chat resends these on every step of answering a question, so caching typically lowers cost and latency: cache reads cost a fraction of the input price, while cache writes cost somewhat more than it (see [Anthropic's pricing](https://www.anthropic.com/pricing)). It is disabled by default. + +```yaml +type: connector +driver: claude +api_key: "{{ .env.CLAUDE_API_KEY }}" +prompt_caching: true +``` + ## Deploy to Rill Cloud Rill requires you to explicitly provide a Claude API key to use the Claude connector. See the [connector reference](/reference/project-files/connectors#claude) for details. diff --git a/docs/docs/developers/build/connectors/services/openai.md b/docs/docs/developers/build/connectors/services/openai.md index 64ddde07f630..da8ea6710027 100644 --- a/docs/docs/developers/build/connectors/services/openai.md +++ b/docs/docs/developers/build/connectors/services/openai.md @@ -48,6 +48,20 @@ For details on managing credentials across environments, see [Configure Local Cr For additional configuration options (model, base URL, API type, etc.), see the [OpenAI connector reference](/reference/project-files/connectors#openai). +:::note Reasoning models + +Rill sends a `temperature` of `0.1` unless you set `reasoning_effort`. OpenAI's reasoning models reject `temperature` unless reasoning effort is `none`, so if you use a model that reasons by default, set `reasoning_effort` (e.g., `medium`) in the connector: + +```yaml +type: connector +driver: openai +api_key: "{{ .env.openai_api_key }}" +model: gpt-5.6 +reasoning_effort: medium +``` + +::: + ## Deploy to Rill Cloud Rill requires you to explicitly provide an OpenAI API key to use the OpenAI connector. See the [connector reference](/reference/project-files/connectors#openai) for details. diff --git a/docs/docs/reference/project-files/connectors.md b/docs/docs/reference/project-files/connectors.md index acc83e21c7a9..556ffe9e0848 100644 --- a/docs/docs/reference/project-files/connectors.md +++ b/docs/docs/reference/project-files/connectors.md @@ -738,7 +738,7 @@ _[number]_ - Maximum number of tokens to generate in the completion (default: 81 ### `reasoning_effort` -_[string]_ - Constrains effort on reasoning for reasoning models (e.g., 'low', 'medium', 'high') +_[string]_ - Constrains effort on reasoning for reasoning models (e.g., 'low', 'medium', 'high'). Set this when using a model that reasons by default, since Rill otherwise sends a temperature of 0.1, which reasoning models reject. ### `base_url` @@ -785,12 +785,16 @@ _[number]_ - Maximum number of tokens in the response (e.g., 8192) ### `temperature` -_[number]_ - Sampling temperature to use (e.g., 0.0) +_[number]_ - Sampling temperature to use (e.g., 0.0). Defaults to the model's default. Leave unset for models that don't support it, such as Claude Opus 4.7 and later. ### `base_url` _[string]_ - The base URL for the Claude API +### `prompt_caching` + +_[boolean]_ - Cache the tools, system prompt, and conversation between requests (default false). Cache reads cost a fraction of the input price, which typically lowers cost for AI chat, but cache writes cost more than the input price. + ```yaml # Example: Claude connector configuration type: connector @@ -827,7 +831,7 @@ _[number]_ - Maximum number of tokens in the response (e.g., 8192) ### `temperature` -_[number]_ - Sampling temperature to use (0.0-2.0) +_[number]_ - Sampling temperature to use (0.0-2.0). Defaults to the model's default, which Google recommends keeping for Gemini 3 models. ### `top_p` diff --git a/proto/gen/rill/admin/v1/admin.swagger.yaml b/proto/gen/rill/admin/v1/admin.swagger.yaml index a1a833987181..d5d528eeb0f0 100644 --- a/proto/gen/rill/admin/v1/admin.swagger.yaml +++ b/proto/gen/rill/admin/v1/admin.swagger.yaml @@ -5287,6 +5287,11 @@ definitions: outputJsonSchema: type: string title: Optional output JSON schema + includeReasoning: + type: boolean + description: |- + Whether the response may include reasoning blocks. + Older clients don't support them, so they are removed from the response unless this is set. v1CompleteResponse: type: object properties: @@ -5305,6 +5310,10 @@ definitions: type: integer format: int64 description: Number of cache-read (discounted) input tokens; a subset of input_tokens. + cacheWriteInputTokens: + type: integer + format: int64 + description: Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. provider: type: string description: The LLM provider that served the completion (e.g. "claude", "openai", "gemini"). @@ -5343,6 +5352,8 @@ definitions: $ref: '#/definitions/v1ToolCall' toolResult: $ref: '#/definitions/v1ToolResult' + reasoning: + $ref: '#/definitions/v1Reasoning' title: Content block within a message v1CreateAlertResponse: type: object @@ -6792,6 +6803,21 @@ definitions: type: string tokensPerSeat: type: string + v1Reasoning: + type: object + properties: + text: + type: string + description: Readable reasoning. Often empty, since providers may omit it or return a summary. + signature: + type: string + description: Opaque provider signature that lets the provider verify the reasoning. + redactedData: + type: string + description: Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + description: |- + The model's reasoning before its text or tool calls. + Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. v1RecordEventsRequest: type: object properties: diff --git a/proto/gen/rill/admin/v1/ai.pb.go b/proto/gen/rill/admin/v1/ai.pb.go index e5ed7ee2e806..e923c3c73a86 100644 --- a/proto/gen/rill/admin/v1/ai.pb.go +++ b/proto/gen/rill/admin/v1/ai.pb.go @@ -33,6 +33,9 @@ type CompleteRequest struct { Tools []*v1.Tool `protobuf:"bytes,2,rep,name=tools,proto3" json:"tools,omitempty"` // Optional output JSON schema OutputJsonSchema string `protobuf:"bytes,3,opt,name=output_json_schema,json=outputJsonSchema,proto3" json:"output_json_schema,omitempty"` + // Whether the response may include reasoning blocks. + // Older clients don't support them, so they are removed from the response unless this is set. + IncludeReasoning bool `protobuf:"varint,4,opt,name=include_reasoning,json=includeReasoning,proto3" json:"include_reasoning,omitempty"` } func (x *CompleteRequest) Reset() { @@ -88,6 +91,13 @@ func (x *CompleteRequest) GetOutputJsonSchema() string { return "" } +func (x *CompleteRequest) GetIncludeReasoning() bool { + if x != nil { + return x.IncludeReasoning + } + return false +} + type CompleteResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -101,6 +111,8 @@ type CompleteResponse struct { OutputTokens uint32 `protobuf:"varint,3,opt,name=output_tokens,json=outputTokens,proto3" json:"output_tokens,omitempty"` // Number of cache-read (discounted) input tokens; a subset of input_tokens. CachedInputTokens uint32 `protobuf:"varint,4,opt,name=cached_input_tokens,json=cachedInputTokens,proto3" json:"cached_input_tokens,omitempty"` + // Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. + CacheWriteInputTokens uint32 `protobuf:"varint,6,opt,name=cache_write_input_tokens,json=cacheWriteInputTokens,proto3" json:"cache_write_input_tokens,omitempty"` // The LLM provider that served the completion (e.g. "claude", "openai", "gemini"). Provider string `protobuf:"bytes,5,opt,name=provider,proto3" json:"provider,omitempty"` } @@ -165,6 +177,13 @@ func (x *CompleteResponse) GetCachedInputTokens() uint32 { return 0 } +func (x *CompleteResponse) GetCacheWriteInputTokens() uint32 { + if x != nil { + return x.CacheWriteInputTokens + } + return 0 +} + func (x *CompleteResponse) GetProvider() string { if x != nil { return x.Provider @@ -180,7 +199,7 @@ var file_rill_admin_v1_ai_proto_rawDesc = []byte{ 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x13, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x61, 0x69, 0x2f, 0x76, - 0x31, 0x2f, 0x61, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa2, 0x01, 0x0a, 0x0f, 0x43, + 0x31, 0x2f, 0x61, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x39, 0x0a, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, @@ -190,40 +209,47 @@ var file_rill_admin_v1_ai_proto_rawDesc = []byte{ 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, - 0xdf, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, - 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x11, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x32, 0x74, 0x0a, 0x09, 0x41, 0x49, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x67, - 0x0a, 0x08, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x72, 0x69, 0x6c, - 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, - 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x72, 0x69, 0x6c, - 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, - 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x14, 0x3a, 0x01, 0x2a, 0x22, 0x0f, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x69, 0x2f, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x42, 0xac, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, - 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x07, 0x41, - 0x69, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, - 0x6c, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, - 0x6c, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x64, 0x6d, 0x69, 0x6e, - 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x41, 0x58, 0xaa, 0x02, 0x0d, 0x52, 0x69, 0x6c, 0x6c, 0x2e, - 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0d, 0x52, 0x69, 0x6c, 0x6c, 0x5c, - 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x19, 0x52, 0x69, 0x6c, 0x6c, 0x5c, - 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x3a, 0x3a, 0x41, 0x64, 0x6d, - 0x69, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, + 0x2b, 0x0a, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, + 0x6e, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x22, 0x98, 0x02, 0x0a, + 0x10, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x37, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x23, 0x0a, + 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x11, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x63, 0x61, 0x63, 0x68, 0x65, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x32, 0x74, 0x0a, 0x09, 0x41, 0x49, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x67, 0x0a, 0x08, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, + 0x12, 0x1e, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1f, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x1a, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x14, 0x3a, 0x01, 0x2a, 0x22, 0x0f, 0x2f, 0x76, + 0x31, 0x2f, 0x61, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x42, 0xac, 0x01, + 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x2e, 0x76, 0x31, 0x42, 0x07, 0x41, 0x69, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x38, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, + 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, + 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2f, 0x76, 0x31, + 0x3b, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x41, 0x58, 0xaa, 0x02, + 0x0d, 0x52, 0x69, 0x6c, 0x6c, 0x2e, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, + 0x0d, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, + 0x19, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, + 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0f, 0x52, 0x69, 0x6c, + 0x6c, 0x3a, 0x3a, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/gen/rill/admin/v1/ai.pb.validate.go b/proto/gen/rill/admin/v1/ai.pb.validate.go index e7518dead177..b0c33eee996b 100644 --- a/proto/gen/rill/admin/v1/ai.pb.validate.go +++ b/proto/gen/rill/admin/v1/ai.pb.validate.go @@ -127,6 +127,8 @@ func (m *CompleteRequest) validate(all bool) error { // no validation rules for OutputJsonSchema + // no validation rules for IncludeReasoning + if len(errors) > 0 { return CompleteRequestMultiError(errors) } @@ -262,6 +264,8 @@ func (m *CompleteResponse) validate(all bool) error { // no validation rules for CachedInputTokens + // no validation rules for CacheWriteInputTokens + // no validation rules for Provider if len(errors) > 0 { diff --git a/proto/gen/rill/admin/v1/openapi.yaml b/proto/gen/rill/admin/v1/openapi.yaml index 289bcc496dee..64312901471a 100644 --- a/proto/gen/rill/admin/v1/openapi.yaml +++ b/proto/gen/rill/admin/v1/openapi.yaml @@ -382,6 +382,11 @@ components: type: object v1CompleteRequest: properties: + includeReasoning: + description: |- + Whether the response may include reasoning blocks. + Older clients don't support them, so they are removed from the response unless this is set. + type: boolean messages: description: Input message(s) for the AI to complete. items: @@ -398,6 +403,10 @@ components: type: object v1CompleteResponse: properties: + cacheWriteInputTokens: + description: Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. + format: int64 + type: integer cachedInputTokens: description: Number of cache-read (discounted) input tokens; a subset of input_tokens. format: int64 @@ -442,6 +451,8 @@ components: type: object v1ContentBlock: properties: + reasoning: + $ref: '#/components/schemas/v1Reasoning' text: type: string toolCall: @@ -1860,6 +1871,21 @@ components: tokensPerSeat: type: string type: object + v1Reasoning: + description: |- + The model's reasoning before its text or tool calls. + Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. + properties: + redactedData: + description: Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + type: string + signature: + description: Opaque provider signature that lets the provider verify the reasoning. + type: string + text: + description: Readable reasoning. Often empty, since providers may omit it or return a summary. + type: string + type: object v1RecordEventsRequest: properties: events: diff --git a/proto/gen/rill/admin/v1/public.openapi.yaml b/proto/gen/rill/admin/v1/public.openapi.yaml index 97668d3aea33..aea71c6022e2 100644 --- a/proto/gen/rill/admin/v1/public.openapi.yaml +++ b/proto/gen/rill/admin/v1/public.openapi.yaml @@ -382,6 +382,11 @@ components: type: object v1CompleteRequest: properties: + includeReasoning: + description: |- + Whether the response may include reasoning blocks. + Older clients don't support them, so they are removed from the response unless this is set. + type: boolean messages: description: Input message(s) for the AI to complete. items: @@ -398,6 +403,10 @@ components: type: object v1CompleteResponse: properties: + cacheWriteInputTokens: + description: Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. + format: int64 + type: integer cachedInputTokens: description: Number of cache-read (discounted) input tokens; a subset of input_tokens. format: int64 @@ -442,6 +451,8 @@ components: type: object v1ContentBlock: properties: + reasoning: + $ref: '#/components/schemas/v1Reasoning' text: type: string toolCall: @@ -1860,6 +1871,21 @@ components: tokensPerSeat: type: string type: object + v1Reasoning: + description: |- + The model's reasoning before its text or tool calls. + Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. + properties: + redactedData: + description: Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + type: string + signature: + description: Opaque provider signature that lets the provider verify the reasoning. + type: string + text: + description: Readable reasoning. Often empty, since providers may omit it or return a summary. + type: string + type: object v1RecordEventsRequest: properties: events: diff --git a/proto/gen/rill/ai/v1/ai.pb.go b/proto/gen/rill/ai/v1/ai.pb.go index 2edfb18b3344..1c7c4635c0da 100644 --- a/proto/gen/rill/ai/v1/ai.pb.go +++ b/proto/gen/rill/ai/v1/ai.pb.go @@ -243,6 +243,74 @@ func (x *ToolResult) GetIsError() bool { return false } +// The model's reasoning before its text or tool calls. +// Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. +type Reasoning struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Readable reasoning. Often empty, since providers may omit it or return a summary. + Text string `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"` + // Opaque provider signature that lets the provider verify the reasoning. + Signature string `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` + // Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + RedactedData string `protobuf:"bytes,3,opt,name=redacted_data,json=redactedData,proto3" json:"redacted_data,omitempty"` +} + +func (x *Reasoning) Reset() { + *x = Reasoning{} + if protoimpl.UnsafeEnabled { + mi := &file_rill_ai_v1_ai_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Reasoning) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Reasoning) ProtoMessage() {} + +func (x *Reasoning) ProtoReflect() protoreflect.Message { + mi := &file_rill_ai_v1_ai_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Reasoning.ProtoReflect.Descriptor instead. +func (*Reasoning) Descriptor() ([]byte, []int) { + return file_rill_ai_v1_ai_proto_rawDescGZIP(), []int{3} +} + +func (x *Reasoning) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +func (x *Reasoning) GetSignature() string { + if x != nil { + return x.Signature + } + return "" +} + +func (x *Reasoning) GetRedactedData() string { + if x != nil { + return x.RedactedData + } + return "" +} + // Content block within a message type ContentBlock struct { state protoimpl.MessageState @@ -254,13 +322,14 @@ type ContentBlock struct { // *ContentBlock_Text // *ContentBlock_ToolCall // *ContentBlock_ToolResult + // *ContentBlock_Reasoning BlockType isContentBlock_BlockType `protobuf_oneof:"block_type"` } func (x *ContentBlock) Reset() { *x = ContentBlock{} if protoimpl.UnsafeEnabled { - mi := &file_rill_ai_v1_ai_proto_msgTypes[3] + mi := &file_rill_ai_v1_ai_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -273,7 +342,7 @@ func (x *ContentBlock) String() string { func (*ContentBlock) ProtoMessage() {} func (x *ContentBlock) ProtoReflect() protoreflect.Message { - mi := &file_rill_ai_v1_ai_proto_msgTypes[3] + mi := &file_rill_ai_v1_ai_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -286,7 +355,7 @@ func (x *ContentBlock) ProtoReflect() protoreflect.Message { // Deprecated: Use ContentBlock.ProtoReflect.Descriptor instead. func (*ContentBlock) Descriptor() ([]byte, []int) { - return file_rill_ai_v1_ai_proto_rawDescGZIP(), []int{3} + return file_rill_ai_v1_ai_proto_rawDescGZIP(), []int{4} } func (m *ContentBlock) GetBlockType() isContentBlock_BlockType { @@ -317,6 +386,13 @@ func (x *ContentBlock) GetToolResult() *ToolResult { return nil } +func (x *ContentBlock) GetReasoning() *Reasoning { + if x, ok := x.GetBlockType().(*ContentBlock_Reasoning); ok { + return x.Reasoning + } + return nil +} + type isContentBlock_BlockType interface { isContentBlock_BlockType() } @@ -333,12 +409,18 @@ type ContentBlock_ToolResult struct { ToolResult *ToolResult `protobuf:"bytes,3,opt,name=tool_result,json=toolResult,proto3,oneof"` } +type ContentBlock_Reasoning struct { + Reasoning *Reasoning `protobuf:"bytes,4,opt,name=reasoning,proto3,oneof"` +} + func (*ContentBlock_Text) isContentBlock_BlockType() {} func (*ContentBlock_ToolCall) isContentBlock_BlockType() {} func (*ContentBlock_ToolResult) isContentBlock_BlockType() {} +func (*ContentBlock_Reasoning) isContentBlock_BlockType() {} + // Completion message for AI API calls type CompletionMessage struct { state protoimpl.MessageState @@ -353,7 +435,7 @@ type CompletionMessage struct { func (x *CompletionMessage) Reset() { *x = CompletionMessage{} if protoimpl.UnsafeEnabled { - mi := &file_rill_ai_v1_ai_proto_msgTypes[4] + mi := &file_rill_ai_v1_ai_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -366,7 +448,7 @@ func (x *CompletionMessage) String() string { func (*CompletionMessage) ProtoMessage() {} func (x *CompletionMessage) ProtoReflect() protoreflect.Message { - mi := &file_rill_ai_v1_ai_proto_msgTypes[4] + mi := &file_rill_ai_v1_ai_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -379,7 +461,7 @@ func (x *CompletionMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use CompletionMessage.ProtoReflect.Descriptor instead. func (*CompletionMessage) Descriptor() ([]byte, []int) { - return file_rill_ai_v1_ai_proto_rawDescGZIP(), []int{4} + return file_rill_ai_v1_ai_proto_rawDescGZIP(), []int{5} } func (x *CompletionMessage) GetRole() string { @@ -434,34 +516,44 @@ var file_rill_ai_v1_ai_proto_rawDesc = []byte{ 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x69, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa2, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x14, 0x0a, 0x04, 0x74, 0x65, 0x78, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, - 0x33, 0x0a, 0x09, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x54, 0x6f, 0x6f, 0x6c, 0x43, 0x61, 0x6c, 0x6c, 0x48, 0x00, 0x52, 0x08, 0x74, 0x6f, 0x6f, 0x6c, - 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x39, 0x0a, 0x0b, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x69, 0x6c, 0x6c, - 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x42, - 0x0c, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, - 0x11, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x72, 0x6f, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x32, 0x0a, 0x07, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x97, - 0x01, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, - 0x31, 0x42, 0x07, 0x41, 0x69, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x32, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, - 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, - 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x61, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x69, 0x76, 0x31, - 0xa2, 0x02, 0x03, 0x52, 0x41, 0x58, 0xaa, 0x02, 0x0a, 0x52, 0x69, 0x6c, 0x6c, 0x2e, 0x41, 0x69, - 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x41, 0x69, 0x5c, 0x56, 0x31, - 0xe2, 0x02, 0x16, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x41, 0x69, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, - 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0c, 0x52, 0x69, 0x6c, 0x6c, - 0x3a, 0x3a, 0x41, 0x69, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x07, 0x69, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x62, 0x0a, 0x09, 0x52, 0x65, 0x61, 0x73, + 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x64, 0x61, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x72, 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x22, 0xd9, 0x01, 0x0a, + 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x14, 0x0a, + 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x74, + 0x65, 0x78, 0x74, 0x12, 0x33, 0x0a, 0x09, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x63, 0x61, 0x6c, 0x6c, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x43, 0x61, 0x6c, 0x6c, 0x48, 0x00, 0x52, 0x08, + 0x74, 0x6f, 0x6f, 0x6c, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x39, 0x0a, 0x0b, 0x74, 0x6f, 0x6f, 0x6c, + 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x12, 0x35, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x69, 0x6e, 0x67, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, + 0x09, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x0a, 0x0a, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x11, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x72, 0x6f, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6c, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x32, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, + 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x97, 0x01, 0x0a, 0x0e, 0x63, 0x6f, + 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x61, 0x69, 0x2e, 0x76, 0x31, 0x42, 0x07, 0x41, 0x69, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, + 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, + 0x2f, 0x61, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x69, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x41, + 0x58, 0xaa, 0x02, 0x0a, 0x52, 0x69, 0x6c, 0x6c, 0x2e, 0x41, 0x69, 0x2e, 0x56, 0x31, 0xca, 0x02, + 0x0a, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x41, 0x69, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x52, 0x69, + 0x6c, 0x6c, 0x5c, 0x41, 0x69, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0c, 0x52, 0x69, 0x6c, 0x6c, 0x3a, 0x3a, 0x41, 0x69, 0x3a, + 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -476,26 +568,28 @@ func file_rill_ai_v1_ai_proto_rawDescGZIP() []byte { return file_rill_ai_v1_ai_proto_rawDescData } -var file_rill_ai_v1_ai_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_rill_ai_v1_ai_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_rill_ai_v1_ai_proto_goTypes = []any{ (*Tool)(nil), // 0: rill.ai.v1.Tool (*ToolCall)(nil), // 1: rill.ai.v1.ToolCall (*ToolResult)(nil), // 2: rill.ai.v1.ToolResult - (*ContentBlock)(nil), // 3: rill.ai.v1.ContentBlock - (*CompletionMessage)(nil), // 4: rill.ai.v1.CompletionMessage - (*structpb.Struct)(nil), // 5: google.protobuf.Struct + (*Reasoning)(nil), // 3: rill.ai.v1.Reasoning + (*ContentBlock)(nil), // 4: rill.ai.v1.ContentBlock + (*CompletionMessage)(nil), // 5: rill.ai.v1.CompletionMessage + (*structpb.Struct)(nil), // 6: google.protobuf.Struct } var file_rill_ai_v1_ai_proto_depIdxs = []int32{ - 5, // 0: rill.ai.v1.Tool.meta:type_name -> google.protobuf.Struct - 5, // 1: rill.ai.v1.ToolCall.input:type_name -> google.protobuf.Struct + 6, // 0: rill.ai.v1.Tool.meta:type_name -> google.protobuf.Struct + 6, // 1: rill.ai.v1.ToolCall.input:type_name -> google.protobuf.Struct 1, // 2: rill.ai.v1.ContentBlock.tool_call:type_name -> rill.ai.v1.ToolCall 2, // 3: rill.ai.v1.ContentBlock.tool_result:type_name -> rill.ai.v1.ToolResult - 3, // 4: rill.ai.v1.CompletionMessage.content:type_name -> rill.ai.v1.ContentBlock - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 3, // 4: rill.ai.v1.ContentBlock.reasoning:type_name -> rill.ai.v1.Reasoning + 4, // 5: rill.ai.v1.CompletionMessage.content:type_name -> rill.ai.v1.ContentBlock + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_rill_ai_v1_ai_proto_init() } @@ -541,7 +635,7 @@ func file_rill_ai_v1_ai_proto_init() { } } file_rill_ai_v1_ai_proto_msgTypes[3].Exporter = func(v any, i int) any { - switch v := v.(*ContentBlock); i { + switch v := v.(*Reasoning); i { case 0: return &v.state case 1: @@ -553,6 +647,18 @@ func file_rill_ai_v1_ai_proto_init() { } } file_rill_ai_v1_ai_proto_msgTypes[4].Exporter = func(v any, i int) any { + switch v := v.(*ContentBlock); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rill_ai_v1_ai_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*CompletionMessage); i { case 0: return &v.state @@ -565,10 +671,11 @@ func file_rill_ai_v1_ai_proto_init() { } } } - file_rill_ai_v1_ai_proto_msgTypes[3].OneofWrappers = []any{ + file_rill_ai_v1_ai_proto_msgTypes[4].OneofWrappers = []any{ (*ContentBlock_Text)(nil), (*ContentBlock_ToolCall)(nil), (*ContentBlock_ToolResult)(nil), + (*ContentBlock_Reasoning)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -576,7 +683,7 @@ func file_rill_ai_v1_ai_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_rill_ai_v1_ai_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, diff --git a/proto/gen/rill/ai/v1/ai.pb.validate.go b/proto/gen/rill/ai/v1/ai.pb.validate.go index 0c44fa212661..ed9e08b6314e 100644 --- a/proto/gen/rill/ai/v1/ai.pb.validate.go +++ b/proto/gen/rill/ai/v1/ai.pb.validate.go @@ -409,6 +409,111 @@ var _ interface { ErrorName() string } = ToolResultValidationError{} +// Validate checks the field values on Reasoning with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Reasoning) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Reasoning with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in ReasoningMultiError, or nil +// if none found. +func (m *Reasoning) ValidateAll() error { + return m.validate(true) +} + +func (m *Reasoning) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Text + + // no validation rules for Signature + + // no validation rules for RedactedData + + if len(errors) > 0 { + return ReasoningMultiError(errors) + } + + return nil +} + +// ReasoningMultiError is an error wrapping multiple validation errors returned +// by Reasoning.ValidateAll() if the designated constraints aren't met. +type ReasoningMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m ReasoningMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m ReasoningMultiError) AllErrors() []error { return m } + +// ReasoningValidationError is the validation error returned by +// Reasoning.Validate if the designated constraints aren't met. +type ReasoningValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e ReasoningValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e ReasoningValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e ReasoningValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e ReasoningValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e ReasoningValidationError) ErrorName() string { return "ReasoningValidationError" } + +// Error satisfies the builtin error interface +func (e ReasoningValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sReasoning.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = ReasoningValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = ReasoningValidationError{} + // Validate checks the field values on ContentBlock with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. @@ -526,6 +631,47 @@ func (m *ContentBlock) validate(all bool) error { } } + case *ContentBlock_Reasoning: + if v == nil { + err := ContentBlockValidationError{ + field: "BlockType", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetReasoning()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ContentBlockValidationError{ + field: "Reasoning", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ContentBlockValidationError{ + field: "Reasoning", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetReasoning()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ContentBlockValidationError{ + field: "Reasoning", + reason: "embedded message failed validation", + cause: err, + } + } + } + default: _ = v // ensures v is used } diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index ada7a1ea87d1..99a5b0a18732 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -5575,6 +5575,8 @@ definitions: $ref: '#/definitions/v1ToolCall' toolResult: $ref: '#/definitions/v1ToolResult' + reasoning: + $ref: '#/definitions/v1Reasoning' title: Content block within a message v1Conversation: type: object @@ -8187,6 +8189,21 @@ definitions: type: array items: type: object + v1Reasoning: + type: object + properties: + text: + type: string + description: Readable reasoning. Often empty, since providers may omit it or return a summary. + signature: + type: string + description: Opaque provider signature that lets the provider verify the reasoning. + redactedData: + type: string + description: Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + description: |- + The model's reasoning before its text or tool calls. + Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. v1ReconcileStatus: type: string enum: diff --git a/proto/rill/admin/v1/ai.proto b/proto/rill/admin/v1/ai.proto index 6284613a0a08..1fa6ad0aa941 100644 --- a/proto/rill/admin/v1/ai.proto +++ b/proto/rill/admin/v1/ai.proto @@ -23,6 +23,9 @@ message CompleteRequest { repeated rill.ai.v1.Tool tools = 2; // Optional output JSON schema string output_json_schema = 3; + // Whether the response may include reasoning blocks. + // Older clients don't support them, so they are removed from the response unless this is set. + bool include_reasoning = 4; } message CompleteResponse { @@ -34,6 +37,8 @@ message CompleteResponse { uint32 output_tokens = 3; // Number of cache-read (discounted) input tokens; a subset of input_tokens. uint32 cached_input_tokens = 4; + // Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. + uint32 cache_write_input_tokens = 6; // The LLM provider that served the completion (e.g. "claude", "openai", "gemini"). string provider = 5; } diff --git a/proto/rill/ai/v1/ai.proto b/proto/rill/ai/v1/ai.proto index dce89592dafb..fd767916228e 100644 --- a/proto/rill/ai/v1/ai.proto +++ b/proto/rill/ai/v1/ai.proto @@ -33,12 +33,24 @@ message ToolResult { bool is_error = 3; } +// The model's reasoning before its text or tool calls. +// Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. +message Reasoning { + // Readable reasoning. Often empty, since providers may omit it or return a summary. + string text = 1; + // Opaque provider signature that lets the provider verify the reasoning. + string signature = 2; + // Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + string redacted_data = 3; +} + // Content block within a message message ContentBlock { oneof block_type { string text = 1; ToolCall tool_call = 2; ToolResult tool_result = 3; + Reasoning reasoning = 4; } } diff --git a/runtime/ai/ai.go b/runtime/ai/ai.go index cc8e7e1ae1ad..cfdc8c71ecf9 100644 --- a/runtime/ai/ai.go +++ b/runtime/ai/ai.go @@ -1312,7 +1312,7 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp // TODO: For durable execution, add messages from current scope. // Telemetry - var iterations, truncations, inputTokens, outputTokens, cachedInputTokens int + var iterations, truncations, inputTokens, outputTokens, cachedInputTokens, cacheWriteInputTokens int var provider string s.logger.Debug("completion started", zap.Int("initial_messages", len(messages)), @@ -1347,13 +1347,15 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp attribute.Int("input_tokens", inputTokens), attribute.Int("output_tokens", outputTokens), attribute.Int("cached_input_tokens", cachedInputTokens), + attribute.Int("cache_write_input_tokens", cacheWriteInputTokens), attribute.String("provider", provider), ) // Emit billable token metrics. Tagged with the request source, the LLM provider, and whether the completion used // the Rill-managed AI connector, so the billable scope is decided downstream in SQL (emitted for every completion, // unlike tool_call). Values are reported as the provider returns them; cached_input_tokens is emitted separately - // so billing can price cached input at the cheaper rate (its relationship to input_tokens is provider-specific). + // so billing can price cached input at the cheaper rate (its relationship to input_tokens is provider-specific), + // and cache_write_input_tokens so billing can price cache writes at their premium rate. source := attribute.String("source", string(runtime.RequestSourceFromContext(ctx))) managedAI := attribute.Bool("managed_ai", s.managedAI) providerAttr := attribute.String("provider", provider) @@ -1366,6 +1368,9 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp if cachedInputTokens > 0 { s.activity.RecordMetric(ctx, "cached_input_tokens", float64(cachedInputTokens), source, managedAI, providerAttr) } + if cacheWriteInputTokens > 0 { + s.activity.RecordMetric(ctx, "cache_write_input_tokens", float64(cacheWriteInputTokens), source, managedAI, providerAttr) + } }() // Complete and execute tool calls in a loop. @@ -1408,6 +1413,7 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp inputTokens += res.InputTokens outputTokens += res.OutputTokens cachedInputTokens += res.CachedInputTokens + cacheWriteInputTokens += res.CacheWriteInputTokens if res.Provider != "" { provider = res.Provider } @@ -1446,6 +1452,13 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp // TODO: How to do durable execution here? for _, block := range res.Message.Content { switch block := block.BlockType.(type) { + case *aiv1.ContentBlock_Reasoning: + // Some providers require the reasoning to be passed back within the tool call loop, so keep it for the next iterations. + // It isn't added to the session, since it's only needed until the loop completes. + messages = append(messages, &aiv1.CompletionMessage{ + Role: string(RoleAssistant), + Content: []*aiv1.ContentBlock{{BlockType: block}}, + }) case *aiv1.ContentBlock_Text: msg := s.AddMessage(&AddMessageOptions{ Role: RoleAssistant, @@ -1517,6 +1530,8 @@ func (s *Session) Complete(ctx context.Context, name string, out any, opts *Comp } else if outputSchema != nil { outVal = json.RawMessage(block.Text) } + case *aiv1.ContentBlock_Reasoning: + // Not needed after the loop completes. default: return nil, fmt.Errorf("unexpected result block type: %T", block) } @@ -1766,6 +1781,14 @@ func maybeTruncateMessages(messages []*aiv1.CompletionMessage) []*aiv1.Completio return false }) + // Remove reasoning blocks, since providers that require them (e.g. Claude) reject them after the history they were produced from has changed. + // Removing all of them is allowed; removing only some of them is not. + result = slices.DeleteFunc(result, func(msg *aiv1.CompletionMessage) bool { + return slices.ContainsFunc(msg.Content, func(block *aiv1.ContentBlock) bool { + return block.GetReasoning() != nil + }) + }) + return result } diff --git a/runtime/ai/reasoning_test.go b/runtime/ai/reasoning_test.go new file mode 100644 index 000000000000..4099c9ec489f --- /dev/null +++ b/runtime/ai/reasoning_test.go @@ -0,0 +1,66 @@ +package ai_test + +import ( + "slices" + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/rilldata/rill/runtime/ai" + "github.com/rilldata/rill/runtime/drivers" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestReasoning(t *testing.T) { + script := &scriptedAIService{turns: []turnFunc{ + reasoningTurn("sig-1", &aiv1.ContentBlock{BlockType: &aiv1.ContentBlock_ToolCall{ToolCall: &aiv1.ToolCall{ + Id: "call-1", + Name: ai.ListMetricsViewsName, + Input: &structpb.Struct{}, + }}}), + // The final result may also contain reasoning. + reasoningTurn("sig-2", &aiv1.ContentBlock{BlockType: &aiv1.ContentBlock_Text{Text: "first answer"}}), + textTurn("second answer"), + }} + s := newSkillReferencesSession(t, script) + + _, err := s.CallTool(t.Context(), ai.RoleUser, ai.AnalystAgentName, nil, &ai.AnalystAgentArgs{Prompt: "first question"}) + require.NoError(t, err) + + // The reasoning is passed back on the next iteration of the tool call loop, before the tool call it led to. + require.Len(t, script.inputs, 2) + blocks := slices.Concat(contents(script.inputs[1])...) + i := slices.IndexFunc(blocks, func(b *aiv1.ContentBlock) bool { return b.GetReasoning() != nil }) + require.GreaterOrEqual(t, i, 0) + require.Equal(t, "sig-1", blocks[i].GetReasoning().Signature) + require.Equal(t, ai.ListMetricsViewsName, blocks[i+1].GetToolCall().GetName()) + + // The reasoning isn't kept after the tool call loop completes. + _, err = s.CallTool(t.Context(), ai.RoleUser, ai.AnalystAgentName, nil, &ai.AnalystAgentArgs{Prompt: "second question"}) + require.NoError(t, err) + require.Len(t, script.inputs, 3) + for _, block := range slices.Concat(contents(script.inputs[2])...) { + require.Nil(t, block.GetReasoning()) + } +} + +// reasoningTurn makes the model return its reasoning followed by the given block. +func reasoningTurn(signature string, block *aiv1.ContentBlock) turnFunc { + return func(_ *drivers.CompleteOptions) *aiv1.CompletionMessage { + return &aiv1.CompletionMessage{ + Role: "assistant", + Content: []*aiv1.ContentBlock{ + {BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Signature: signature}}}, + block, + }, + } + } +} + +func contents(msgs []*aiv1.CompletionMessage) [][]*aiv1.ContentBlock { + res := make([][]*aiv1.ContentBlock, len(msgs)) + for i, msg := range msgs { + res[i] = msg.Content + } + return res +} diff --git a/runtime/ai/truncate_internal_test.go b/runtime/ai/truncate_internal_test.go new file mode 100644 index 000000000000..8db509d2cfa0 --- /dev/null +++ b/runtime/ai/truncate_internal_test.go @@ -0,0 +1,43 @@ +package ai + +import ( + "fmt" + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/stretchr/testify/require" +) + +func TestMaybeTruncateMessagesReasoning(t *testing.T) { + newMessages := func(n int) []*aiv1.CompletionMessage { + var msgs []*aiv1.CompletionMessage + for i := range n { + block := &aiv1.ContentBlock{BlockType: &aiv1.ContentBlock_Text{Text: fmt.Sprintf("message %d", i)}} + if i%2 == 1 { + block = &aiv1.ContentBlock{BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Signature: fmt.Sprintf("sig-%d", i)}}} + } + msgs = append(msgs, &aiv1.CompletionMessage{Role: "assistant", Content: []*aiv1.ContentBlock{block}}) + } + return msgs + } + countReasoning := func(msgs []*aiv1.CompletionMessage) int { + var n int + for _, msg := range msgs { + for _, block := range msg.Content { + if block.GetReasoning() != nil { + n++ + } + } + } + return n + } + + // Without truncation, reasoning is kept. + msgs := newMessages(50) + require.Equal(t, msgs, maybeTruncateMessages(msgs)) + + // With truncation, the history changes, so all reasoning is removed. + truncated := maybeTruncateMessages(newMessages(150)) + require.Less(t, len(truncated), 150) + require.Zero(t, countReasoning(truncated)) +} diff --git a/runtime/drivers/admin/ai.go b/runtime/drivers/admin/ai.go index a587e278f82d..cd89d00c0d22 100644 --- a/runtime/drivers/admin/ai.go +++ b/runtime/drivers/admin/ai.go @@ -22,16 +22,18 @@ func (h *Handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (* Messages: opts.Messages, Tools: opts.Tools, OutputJsonSchema: outputJSONSchema, + IncludeReasoning: true, }) if err != nil { return nil, err } return &drivers.CompleteResult{ - Message: res.Message, - Provider: res.GetProvider(), - InputTokens: int(res.InputTokens), - OutputTokens: int(res.OutputTokens), - CachedInputTokens: int(res.GetCachedInputTokens()), + Message: res.Message, + Provider: res.GetProvider(), + InputTokens: int(res.InputTokens), + OutputTokens: int(res.OutputTokens), + CachedInputTokens: int(res.GetCachedInputTokens()), + CacheWriteInputTokens: int(res.GetCacheWriteInputTokens()), }, nil } diff --git a/runtime/drivers/ai.go b/runtime/drivers/ai.go index d595386cef3a..f81e7ed42f93 100644 --- a/runtime/drivers/ai.go +++ b/runtime/drivers/ai.go @@ -27,5 +27,8 @@ type CompleteResult struct { // is a subset of InputTokens. InputTokens int CachedInputTokens int - OutputTokens int + // CacheWriteInputTokens are input tokens written to the provider's prompt cache, which some providers (e.g. Claude) bill at a premium. + // Like for Claude's CachedInputTokens, they are disjoint from InputTokens. + CacheWriteInputTokens int + OutputTokens int } diff --git a/runtime/drivers/claude/claude.go b/runtime/drivers/claude/claude.go index f96414c6cbc9..7c93a572a4a3 100644 --- a/runtime/drivers/claude/claude.go +++ b/runtime/drivers/claude/claude.go @@ -17,8 +17,6 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -const defaultTemperature = 0.1 - func init() { drivers.Register("claude", driver{}) drivers.RegisterAsConnector("claude", driver{}) @@ -58,8 +56,7 @@ var spec = drivers.Spec{ Type: drivers.NumberPropertyType, Required: false, DisplayName: "Temperature", - Description: "Sampling temperature to use.", - Default: "0.1", + Description: "Sampling temperature to use. Defaults to the model's default. Leave unset for models that don't support it, such as Claude Opus 4.7 and later.", }, { Key: "base_url", @@ -69,6 +66,14 @@ var spec = drivers.Spec{ Description: "Custom base URL for the Claude API.", Placeholder: "", }, + { + Key: "prompt_caching", + Type: drivers.BooleanPropertyType, + Required: false, + DisplayName: "Prompt Caching", + Description: "Cache the tools, system prompt, and conversation between requests. Cache reads cost a fraction of the input price, but cache writes cost more than it.", + Default: "false", + }, }, ImplementsAI: true, } @@ -120,11 +125,12 @@ func (d driver) TertiarySourceConnectors(ctx context.Context, srcProps map[strin } type configProperties struct { - APIKey string `mapstructure:"api_key"` - Model string `mapstructure:"model"` - MaxTokens int `mapstructure:"max_tokens"` - Temperature *float64 `mapstructure:"temperature"` - BaseURL string `mapstructure:"base_url"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + MaxTokens int `mapstructure:"max_tokens"` + Temperature *float64 `mapstructure:"temperature"` + BaseURL string `mapstructure:"base_url"` + PromptCaching bool `mapstructure:"prompt_caching"` } func (c *configProperties) getModel() string { @@ -141,13 +147,6 @@ func (c *configProperties) getMaxTokens() int { return 8192 // Default max tokens } -func (c *configProperties) getTemperature() float64 { - if c.Temperature != nil { - return *c.Temperature - } - return defaultTemperature -} - type handle struct { client anthropic.Client config *configProperties @@ -259,7 +258,7 @@ func (h *handle) Ping(ctx context.Context) error { // Complete implements drivers.AIService. func (h *handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (*drivers.CompleteResult, error) { - system, msgs, err := convertMessages(opts.Messages) + system, msgs, err := convertMessages(opts.Messages, len(opts.Tools) > 0) if err != nil { return nil, fmt.Errorf("failed to convert messages: %w", err) } @@ -270,11 +269,14 @@ func (h *handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (* } params := anthropic.BetaMessageNewParams{ - Model: anthropic.Model(h.config.getModel()), - MaxTokens: int64(h.config.getMaxTokens()), - Temperature: anthropic.Float(h.config.getTemperature()), - Messages: msgs, - System: system, + Model: anthropic.Model(h.config.getModel()), + MaxTokens: int64(h.config.getMaxTokens()), + Messages: msgs, + System: system, + } + // Only set temperature if configured, since current models reject it (or reject non-default values). + if h.config.Temperature != nil { + params.Temperature = anthropic.Float(*h.config.Temperature) } if len(betaTools) > 0 { @@ -297,6 +299,26 @@ func (h *handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (* params.OutputFormat = anthropic.BetaJSONSchemaOutputFormat(schemaMap) } + // The tools, system prompt, and conversation are resent on every step of a tool loop. + // If enabled, mark the end of each as a cache breakpoint, so later steps read them from the prompt cache. + if h.config.PromptCaching { + if n := len(params.Tools); n > 0 { + if cc := params.Tools[n-1].GetCacheControl(); cc != nil { + *cc = anthropic.NewBetaCacheControlEphemeralParam() + } + } + if n := len(params.System); n > 0 { + params.System[n-1].CacheControl = anthropic.NewBetaCacheControlEphemeralParam() + } + if n := len(params.Messages); n > 0 { + if content := params.Messages[n-1].Content; len(content) > 0 { + if cc := content[len(content)-1].GetCacheControl(); cc != nil { + *cc = anthropic.NewBetaCacheControlEphemeralParam() + } + } + } + } + res, err := h.client.Beta.Messages.New(ctx, params) if err != nil { return nil, err @@ -309,23 +331,28 @@ func (h *handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (* // Claude reports cache-read and cache-creation tokens separately from InputTokens. return &drivers.CompleteResult{ - Message: resMsgs, - Provider: "claude", - InputTokens: int(res.Usage.InputTokens), - CachedInputTokens: int(res.Usage.CacheReadInputTokens), - OutputTokens: int(res.Usage.OutputTokens), + Message: resMsgs, + Provider: "claude", + InputTokens: int(res.Usage.InputTokens), + CachedInputTokens: int(res.Usage.CacheReadInputTokens), + CacheWriteInputTokens: int(res.Usage.CacheCreationInputTokens), + OutputTokens: int(res.Usage.OutputTokens), }, nil } // convertMessages converts Rill messages to Claude beta message format. // It returns system blocks separately because Claude's API treats them differently. -func convertMessages(msgs []*aiv1.CompletionMessage) ([]anthropic.BetaTextBlockParam, []anthropic.BetaMessageParam, error) { +// +// Reasoning blocks are passed back as thinking blocks, which Claude requires within a tool call loop. +// But a thinking block is bound to the tools offered when it was produced, +// so if no tools are offered (e.g. on the final iteration of a tool call loop), all reasoning blocks are removed. +func convertMessages(msgs []*aiv1.CompletionMessage, toolsOffered bool) ([]anthropic.BetaTextBlockParam, []anthropic.BetaMessageParam, error) { var system []anthropic.BetaTextBlockParam var other []anthropic.BetaMessageParam for _, msg := range msgs { if msg.Role != "system" { - converted, err := convertMessage(msg) + converted, err := convertMessage(msg, toolsOffered) if err != nil { return nil, nil, err } @@ -336,6 +363,9 @@ func convertMessages(msgs []*aiv1.CompletionMessage) ([]anthropic.BetaTextBlockP for _, block := range msg.Content { switch block := block.BlockType.(type) { case *aiv1.ContentBlock_Text: + if block.Text == "" { + continue + } system = append(system, anthropic.BetaTextBlockParam{Text: block.Text}) default: return nil, nil, fmt.Errorf("unsupported system message block type: %T", block) @@ -348,7 +378,7 @@ func convertMessages(msgs []*aiv1.CompletionMessage) ([]anthropic.BetaTextBlockP // convertMessage converts a single Rill message to Claude beta messages. // Tool results become separate user messages per Claude's API requirements. -func convertMessage(msg *aiv1.CompletionMessage) ([]anthropic.BetaMessageParam, error) { +func convertMessage(msg *aiv1.CompletionMessage, toolsOffered bool) ([]anthropic.BetaMessageParam, error) { var result []anthropic.BetaMessageParam role := anthropic.BetaMessageParamRoleUser @@ -359,12 +389,28 @@ func convertMessage(msg *aiv1.CompletionMessage) ([]anthropic.BetaMessageParam, for _, block := range msg.Content { switch b := block.BlockType.(type) { case *aiv1.ContentBlock_Text: + // Skip empty text blocks, which the API rejects (e.g. from conversations saved when thinking blocks were turned into text). + if b.Text == "" { + continue + } result = append(result, anthropic.BetaMessageParam{ Role: role, Content: []anthropic.BetaContentBlockParamUnion{ anthropic.NewBetaTextBlock(b.Text), }, }) + case *aiv1.ContentBlock_Reasoning: + if !toolsOffered { + continue + } + thinking := anthropic.NewBetaThinkingBlock(b.Reasoning.Signature, b.Reasoning.Text) + if b.Reasoning.RedactedData != "" { + thinking = anthropic.NewBetaRedactedThinkingBlock(b.Reasoning.RedactedData) + } + result = append(result, anthropic.BetaMessageParam{ + Role: anthropic.BetaMessageParamRoleAssistant, // NOTE: Hard-coded as assistant + Content: []anthropic.BetaContentBlockParamUnion{thinking}, + }) case *aiv1.ContentBlock_ToolCall: result = append(result, anthropic.BetaMessageParam{ Role: anthropic.BetaMessageParamRoleAssistant, // NOTE: Hard-coded as assistant @@ -434,6 +480,22 @@ func convertTool(tool *aiv1.Tool) (anthropic.BetaToolUnionParam, error) { if err := json.Unmarshal([]byte(tool.InputSchema), &inputSchema); err != nil { return anthropic.BetaToolUnionParam{}, fmt.Errorf("failed to parse schema for tool %q: %w", tool.Name, err) } + // The SDK type only has fields for "type", "properties", and "required", and unmarshaling doesn't populate ExtraFields. + // So we copy the remaining keywords (such as "$defs", which "$ref"s point to) to ExtraFields to send them as-is. + var raw map[string]any + if err := json.Unmarshal([]byte(tool.InputSchema), &raw); err != nil { + return anthropic.BetaToolUnionParam{}, fmt.Errorf("failed to parse schema for tool %q: %w", tool.Name, err) + } + for k, v := range raw { + switch k { + case "type", "properties", "required": + default: + if inputSchema.ExtraFields == nil { + inputSchema.ExtraFields = make(map[string]any) + } + inputSchema.ExtraFields[k] = v + } + } } result := anthropic.BetaToolUnionParamOfTool(inputSchema, tool.Name) @@ -451,12 +513,22 @@ func convertResponseMessage(msg *anthropic.BetaMessage) (*aiv1.CompletionMessage block := msg.Content[idx] // Note: Separate line to avoid lint error about copying large structs switch block.Type { case "text": + // Skip empty text blocks, since the API rejects them when they're sent back in a later request. + if block.Text == "" { + continue + } blocks = append(blocks, &aiv1.ContentBlock{ BlockType: &aiv1.ContentBlock_Text{Text: block.Text}, }) case "thinking": + // Current models return thinking blocks by default, with empty text unless display is "summarized". + // They're kept with their signature, since they must be passed back unmodified within a tool call loop. + blocks = append(blocks, &aiv1.ContentBlock{ + BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Text: block.Thinking, Signature: block.Signature}}, + }) + case "redacted_thinking": blocks = append(blocks, &aiv1.ContentBlock{ - BlockType: &aiv1.ContentBlock_Text{Text: block.Thinking}, + BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{RedactedData: block.Data}}, }) case "tool_use": cb, err := convertResponseToolUse(block) diff --git a/runtime/drivers/claude/claude_test.go b/runtime/drivers/claude/claude_test.go new file mode 100644 index 000000000000..d7ca1cc765f3 --- /dev/null +++ b/runtime/drivers/claude/claude_test.go @@ -0,0 +1,216 @@ +package claude + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/metricsview" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestComplete(t *testing.T) { + cases := []struct { + name string + config map[string]any + wantTemperature any + }{ + { + // Current models reject temperature (or non-default values), so it's only sent if configured. + name: "default", + config: map[string]any{}, + wantTemperature: nil, + }, + { + name: "temperature configured", + config: map[string]any{"temperature": 0.5}, + wantTemperature: 0.5, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Current models return thinking blocks by default, with empty text unless display is "summarized". + body, url := newFakeClaude(t, `{ + "id": "msg_1", "type": "message", "role": "assistant", "model": "test-model", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "text", "text": "Revenue is 100."} + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 3} + }`) + + cfg := map[string]any{"api_key": "test-key", "base_url": url} + for k, v := range tc.config { + cfg[k] = v + } + h, err := driver{}.Open(t.Context(), "", "default", cfg, nil, nil, zap.NewNop()) + require.NoError(t, err) + ai, ok := h.AsAI("default") + require.True(t, ok) + + res, err := ai.Complete(t.Context(), &drivers.CompleteOptions{ + Messages: []*aiv1.CompletionMessage{ + {Role: "user", Content: []*aiv1.ContentBlock{textBlock("What is revenue?")}}, + // An empty text block, e.g. from a thinking block in a conversation saved before the fix. + {Role: "assistant", Content: []*aiv1.ContentBlock{textBlock("")}}, + }, + Tools: []*aiv1.Tool{{Name: "query_metrics_view", InputSchema: metricsview.QueryJSONSchema}}, + }) + require.NoError(t, err) + + // Temperature is only sent if configured. + require.Equal(t, tc.wantTemperature, (*body)["temperature"]) + + // The empty text block isn't sent. + msgs := (*body)["messages"].([]any) + require.Len(t, msgs, 1) + + // The tool schema keeps $defs, which the $ref parameters point to. + schema := (*body)["tools"].([]any)[0].(map[string]any)["input_schema"].(map[string]any) + require.Contains(t, schema["$defs"], "TimeRange") + timeRange := schema["properties"].(map[string]any)["time_range"].(map[string]any) + require.Equal(t, "#/$defs/TimeRange", timeRange["$ref"]) + + // The thinking block is returned as reasoning (with its signature), not as a text block. + require.Len(t, res.Message.Content, 2) + require.Equal(t, "sig", res.Message.Content[0].GetReasoning().GetSignature()) + require.Equal(t, "Revenue is 100.", res.Message.Content[1].GetText()) + }) + } +} + +func TestPromptCaching(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) { + body, url := newFakeClaude(t, `{ + "id": "msg_1", "type": "message", "role": "assistant", "model": "test-model", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 50} + }`) + + h, err := driver{}.Open(t.Context(), "", "default", map[string]any{"api_key": "test-key", "base_url": url, "prompt_caching": enabled}, nil, nil, zap.NewNop()) + require.NoError(t, err) + ai, _ := h.AsAI("default") + res, err := ai.Complete(t.Context(), &drivers.CompleteOptions{ + Messages: []*aiv1.CompletionMessage{ + {Role: "system", Content: []*aiv1.ContentBlock{textBlock("You are an analyst.")}}, + {Role: "user", Content: []*aiv1.ContentBlock{textBlock("What is revenue?")}}, + }, + Tools: []*aiv1.Tool{{Name: "first"}, {Name: "last"}}, + }) + require.NoError(t, err) + + // Cache reads and writes are reported regardless of whether caching is enabled. + require.Equal(t, 100, res.CachedInputTokens) + require.Equal(t, 50, res.CacheWriteInputTokens) + + // If enabled, the last tool, system block, and message block are marked as cache breakpoints. + tools := (*body)["tools"].([]any) + system := (*body)["system"].([]any) + msgs := (*body)["messages"].([]any) + lastMsgContent := msgs[len(msgs)-1].(map[string]any)["content"].([]any) + marked := []any{ + tools[len(tools)-1], + system[len(system)-1], + lastMsgContent[len(lastMsgContent)-1], + } + for _, block := range marked { + if enabled { + require.Equal(t, map[string]any{"type": "ephemeral"}, block.(map[string]any)["cache_control"]) + } else { + require.NotContains(t, block, "cache_control") + } + } + require.NotContains(t, tools[0], "cache_control") + }) + } +} + +func TestReasoning(t *testing.T) { + // Claude requires thinking blocks to be passed back within a tool call loop, + // but a thinking block is bound to the tools offered when it was produced, so they're removed when no tools are offered. + for _, toolsOffered := range []bool{true, false} { + t.Run(fmt.Sprintf("tools=%v", toolsOffered), func(t *testing.T) { + body, url := newFakeClaude(t, `{ + "id": "msg_1", "type": "message", "role": "assistant", "model": "test-model", + "content": [ + {"type": "thinking", "thinking": "Thinking about revenue.", "signature": "sig-2"}, + {"type": "redacted_thinking", "data": "encrypted"}, + {"type": "text", "text": "Revenue is 100."} + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 3} + }`) + h, err := driver{}.Open(t.Context(), "", "default", map[string]any{"api_key": "test-key", "base_url": url}, nil, nil, zap.NewNop()) + require.NoError(t, err) + ai, _ := h.AsAI("default") + + input, err := structpb.NewStruct(map[string]any{}) + require.NoError(t, err) + opts := &drivers.CompleteOptions{ + Messages: []*aiv1.CompletionMessage{ + {Role: "user", Content: []*aiv1.ContentBlock{textBlock("What is revenue?")}}, + {Role: "assistant", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Signature: "sig-1"}}}}}, + {Role: "assistant", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_ToolCall{ToolCall: &aiv1.ToolCall{Id: "call-1", Name: "query", Input: input}}}}}, + {Role: "user", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_ToolResult{ToolResult: &aiv1.ToolResult{Id: "call-1", Content: "100"}}}}}, + }, + } + if toolsOffered { + opts.Tools = []*aiv1.Tool{{Name: "query"}} + } + res, err := ai.Complete(t.Context(), opts) + require.NoError(t, err) + + // Thinking and redacted thinking blocks are returned as reasoning. + require.Len(t, res.Message.Content, 3) + require.Equal(t, "Thinking about revenue.", res.Message.Content[0].GetReasoning().GetText()) + require.Equal(t, "sig-2", res.Message.Content[0].GetReasoning().GetSignature()) + require.Equal(t, "encrypted", res.Message.Content[1].GetReasoning().GetRedactedData()) + + // The reasoning in the history is passed back as a thinking block before its tool call, unless no tools are offered. + var types []string + for _, msg := range (*body)["messages"].([]any) { + for _, block := range msg.(map[string]any)["content"].([]any) { + block := block.(map[string]any) + types = append(types, block["type"].(string)) + if block["type"] == "thinking" { + require.Equal(t, "sig-1", block["signature"]) + } + } + } + if toolsOffered { + require.Equal(t, []string{"text", "thinking", "tool_use", "tool_result"}, types) + } else { + require.Equal(t, []string{"text", "tool_use", "tool_result"}, types) + } + }) + } +} + +// newFakeClaude starts a fake Claude API server that returns the given response. +// It returns a pointer to the decoded body of the last request, and the server's URL to use as the base URL. +func newFakeClaude(t *testing.T, response string) (*map[string]any, string) { + body := &map[string]any{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, body)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(srv.Close) + return body, srv.URL +} + +func textBlock(s string) *aiv1.ContentBlock { + return &aiv1.ContentBlock{BlockType: &aiv1.ContentBlock_Text{Text: s}} +} diff --git a/runtime/drivers/gemini/convert_test.go b/runtime/drivers/gemini/convert_test.go new file mode 100644 index 000000000000..9831a670178b --- /dev/null +++ b/runtime/drivers/gemini/convert_test.go @@ -0,0 +1,23 @@ +package gemini + +import ( + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/stretchr/testify/require" +) + +func TestConvertMessagesSkipsReasoning(t *testing.T) { + // Reasoning blocks from other providers are skipped instead of failing the request. + _, contents, err := convertMessages([]*aiv1.CompletionMessage{ + {Role: "user", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_Text{Text: "hi"}}}}, + {Role: "assistant", Content: []*aiv1.ContentBlock{ + {BlockType: &aiv1.ContentBlock_Reasoning{Reasoning: &aiv1.Reasoning{Signature: "sig"}}}, + {BlockType: &aiv1.ContentBlock_Text{Text: "hello"}}, + }}, + }) + require.NoError(t, err) + require.Len(t, contents, 2) + require.Len(t, contents[1].Parts, 1) + require.Equal(t, "hello", contents[1].Parts[0].Text) +} diff --git a/runtime/drivers/gemini/gemini.go b/runtime/drivers/gemini/gemini.go index 3b7023de69d7..4796955896b0 100644 --- a/runtime/drivers/gemini/gemini.go +++ b/runtime/drivers/gemini/gemini.go @@ -18,8 +18,6 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -const defaultTemperature = 0.1 - func init() { drivers.Register("gemini", driver{}) drivers.RegisterAsConnector("gemini", driver{}) @@ -73,7 +71,7 @@ var spec = drivers.Spec{ Type: drivers.NumberPropertyType, Required: false, DisplayName: "Temperature", - Description: "Sampling temperature to use (0.0-2.0).", + Description: "Sampling temperature to use (0.0-2.0). Defaults to the model's default, which Google recommends keeping for Gemini 3 models.", }, { Key: "top_p", @@ -155,13 +153,6 @@ func (c *configProperties) getModel() string { return "gemini-3-flash-preview" } -func (c *configProperties) getTemperature() float64 { - if c.Temperature != nil { - return *c.Temperature - } - return defaultTemperature -} - func (c *configProperties) getThinkingConfig() *genai.ThinkingConfig { res := &genai.ThinkingConfig{ IncludeThoughts: c.IncludeThoughts, @@ -295,7 +286,10 @@ func (h *handle) Complete(ctx context.Context, opts *drivers.CompleteOptions) (* MaxOutputTokens: int32(h.config.MaxOutputTokens), SystemInstruction: systemInstructions, ThinkingConfig: h.config.getThinkingConfig(), - Temperature: genai.Ptr(float32(h.config.getTemperature())), + } + // Only set temperature if configured, since Google warns that lowering it on Gemini 3 models can cause looping and degraded performance. + if h.config.Temperature != nil { + genConfig.Temperature = genai.Ptr(float32(*h.config.Temperature)) } if h.config.TopP > 0 { genConfig.TopP = genai.Ptr(float32(h.config.TopP)) @@ -504,6 +498,9 @@ func convertMessage(msg *aiv1.CompletionMessage, callIDToName map[string]string) } parts = append(parts, part) + case *aiv1.ContentBlock_Reasoning: + // Skip reasoning from other providers, which Gemini can't use. + default: return nil, fmt.Errorf("unsupported message block type: %T", block.BlockType) } diff --git a/runtime/drivers/gemini/temperature_test.go b/runtime/drivers/gemini/temperature_test.go new file mode 100644 index 000000000000..5a57591a23b3 --- /dev/null +++ b/runtime/drivers/gemini/temperature_test.go @@ -0,0 +1,70 @@ +package gemini + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/rilldata/rill/runtime/drivers" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestTemperature(t *testing.T) { + cases := []struct { + name string + config map[string]any + want any + }{ + { + // Google warns that lowering temperature on Gemini 3 models can cause looping, so it's only sent if configured. + name: "default", + config: map[string]any{}, + want: nil, + }, + { + name: "configured", + config: map[string]any{"temperature": 0.5}, + want: 0.5, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &body)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "candidates": [{"content": {"role": "model", "parts": [{"text": "hi"}]}}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1} + }`)) + })) + defer srv.Close() + t.Setenv("GOOGLE_GEMINI_BASE_URL", srv.URL) + + cfg := map[string]any{"api_key": "test-key"} + for k, v := range tc.config { + cfg[k] = v + } + h, err := driver{}.Open(t.Context(), "", "default", cfg, nil, nil, zap.NewNop()) + require.NoError(t, err) + ai, _ := h.AsAI("default") + _, err = ai.Complete(t.Context(), &drivers.CompleteOptions{ + Messages: []*aiv1.CompletionMessage{{Role: "user", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_Text{Text: "hi"}}}}}, + }) + require.NoError(t, err) + + genConfig, _ := body["generationConfig"].(map[string]any) + if tc.want == nil { + require.NotContains(t, genConfig, "temperature") + } else { + require.Equal(t, tc.want, genConfig["temperature"]) + } + }) + } +} diff --git a/runtime/drivers/openai/temperature_test.go b/runtime/drivers/openai/temperature_test.go new file mode 100644 index 000000000000..f0533059b9c3 --- /dev/null +++ b/runtime/drivers/openai/temperature_test.go @@ -0,0 +1,69 @@ +package openai + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + aiv1 "github.com/rilldata/rill/proto/gen/rill/ai/v1" + "github.com/rilldata/rill/runtime/drivers" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestTemperature(t *testing.T) { + cases := []struct { + name string + config map[string]any + want any + }{ + { + name: "default", + config: map[string]any{}, + want: 0.1, + }, + { + // Reasoning models reject temperature unless reasoning effort is "none", so it's omitted when reasoning effort is set. + name: "reasoning effort", + config: map[string]any{"reasoning_effort": "medium"}, + want: nil, + }, + { + name: "configured", + config: map[string]any{"temperature": 0.5}, + want: 0.5, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &body)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + }`)) + })) + defer srv.Close() + + cfg := map[string]any{"api_key": "test-key", "base_url": srv.URL} + for k, v := range tc.config { + cfg[k] = v + } + h, err := driver{}.Open(t.Context(), "", "default", cfg, nil, nil, zap.NewNop()) + require.NoError(t, err) + ai, _ := h.AsAI("default") + _, err = ai.Complete(t.Context(), &drivers.CompleteOptions{ + Messages: []*aiv1.CompletionMessage{{Role: "user", Content: []*aiv1.ContentBlock{{BlockType: &aiv1.ContentBlock_Text{Text: "hi"}}}}}, + }) + require.NoError(t, err) + require.Equal(t, tc.want, body["temperature"]) + }) + } +} diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 5655d1dc30f0..ef970cd84615 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -703,7 +703,7 @@ definitions: description: "Maximum number of tokens to generate in the completion (default: 8192)" reasoning_effort: type: string - description: "Constrains effort on reasoning for reasoning models (e.g., 'low', 'medium', 'high')" + description: "Constrains effort on reasoning for reasoning models (e.g., 'low', 'medium', 'high'). Set this when using a model that reasons by default, since Rill otherwise sends a temperature of 0.1, which reasoning models reject." base_url: type: string description: The base URL for the OpenAI API (e.g., 'https://api.openai.com/v1') @@ -745,10 +745,13 @@ definitions: description: Maximum number of tokens in the response (e.g., 8192) temperature: type: number - description: Sampling temperature to use (e.g., 0.0) + description: Sampling temperature to use (e.g., 0.0). Defaults to the model's default. Leave unset for models that don't support it, such as Claude Opus 4.7 and later. base_url: type: string description: The base URL for the Claude API + prompt_caching: + type: boolean + description: Cache the tools, system prompt, and conversation between requests (default false). Cache reads cost a fraction of the input price, which typically lowers cost for AI chat, but cache writes cost more than the input price. examples: - # Example: Claude connector configuration type: connector @@ -781,7 +784,7 @@ definitions: description: Maximum number of tokens in the response (e.g., 8192) temperature: type: number - description: Sampling temperature to use (0.0-2.0) + description: Sampling temperature to use (0.0-2.0). Defaults to the model's default, which Google recommends keeping for Gemini 3 models. top_p: type: number description: Nucleus sampling parameter diff --git a/runtime/server/generate_canvas_dashboard.go b/runtime/server/generate_canvas_dashboard.go index bdef57ce5c74..d9e8b1c0cf1e 100644 --- a/runtime/server/generate_canvas_dashboard.go +++ b/runtime/server/generate_canvas_dashboard.go @@ -186,6 +186,8 @@ func (s *Server) generateCanvasDashboardYAMLWithAI(ctx context.Context, instance if text := blockType.Text; text != "" { responseText += text } + case *aiv1.ContentBlock_Reasoning: + // Skip the model's reasoning; only its response is needed. default: return "", fmt.Errorf("unexpected content block type in AI response: %T", blockType) } diff --git a/runtime/server/generate_metrics_view.go b/runtime/server/generate_metrics_view.go index 8f7888e1e00c..784125b34451 100644 --- a/runtime/server/generate_metrics_view.go +++ b/runtime/server/generate_metrics_view.go @@ -252,6 +252,8 @@ func (s *Server) generateMetricsViewYAMLWithAI(ctx context.Context, instanceID, if text := blockType.Text; text != "" { responseText += text } + case *aiv1.ContentBlock_Reasoning: + // Skip the model's reasoning; only its response is needed. default: // For metrics view generation, we only expect text responses return nil, fmt.Errorf("unexpected content block type in AI response: %T", blockType) diff --git a/web-admin/src/client/gen/index.schemas.ts b/web-admin/src/client/gen/index.schemas.ts index 6bf819e8d260..56380fcde078 100644 --- a/web-admin/src/client/gen/index.schemas.ts +++ b/web-admin/src/client/gen/index.schemas.ts @@ -305,6 +305,9 @@ export interface V1CompleteRequest { messages?: V1CompletionMessage[]; tools?: V1Tool[]; outputJsonSchema?: string; + /** Whether the response may include reasoning blocks. +Older clients don't support them, so they are removed from the response unless this is set. */ + includeReasoning?: boolean; } export interface V1CompleteResponse { @@ -315,6 +318,8 @@ export interface V1CompleteResponse { outputTokens?: number; /** Number of cache-read (discounted) input tokens; a subset of input_tokens. */ cachedInputTokens?: number; + /** Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. */ + cacheWriteInputTokens?: number; /** The LLM provider that served the completion (e.g. "claude", "openai", "gemini"). */ provider?: string; } @@ -338,6 +343,7 @@ export interface V1ContentBlock { text?: string; toolCall?: V1ToolCall; toolResult?: V1ToolResult; + reasoning?: V1Reasoning; } export interface V1CreateAlertResponse { @@ -1299,6 +1305,19 @@ export interface V1Quotas { tokensPerSeat?: string; } +/** + * The model's reasoning before its text or tool calls. +Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. + */ +export interface V1Reasoning { + /** Readable reasoning. Often empty, since providers may omit it or return a summary. */ + text?: string; + /** Opaque provider signature that lets the provider verify the reasoning. */ + signature?: string; + /** Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. */ + redactedData?: string; +} + export type V1RecordEventsRequestEventsItem = { [key: string]: unknown }; export interface V1RecordEventsRequest { diff --git a/web-common/src/proto/gen/rill/admin/v1/ai_pb.ts b/web-common/src/proto/gen/rill/admin/v1/ai_pb.ts index 4f4d4c7d2be6..06e5f6fcf3f4 100644 --- a/web-common/src/proto/gen/rill/admin/v1/ai_pb.ts +++ b/web-common/src/proto/gen/rill/admin/v1/ai_pb.ts @@ -32,6 +32,14 @@ export class CompleteRequest extends Message { */ outputJsonSchema = ""; + /** + * Whether the response may include reasoning blocks. + * Older clients don't support them, so they are removed from the response unless this is set. + * + * @generated from field: bool include_reasoning = 4; + */ + includeReasoning = false; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -43,6 +51,7 @@ export class CompleteRequest extends Message { { no: 1, name: "messages", kind: "message", T: CompletionMessage, repeated: true }, { no: 2, name: "tools", kind: "message", T: Tool, repeated: true }, { no: 3, name: "output_json_schema", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 4, name: "include_reasoning", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): CompleteRequest { @@ -94,6 +103,13 @@ export class CompleteResponse extends Message { */ cachedInputTokens = 0; + /** + * Number of input tokens written to the prompt cache, which some providers (e.g. Claude) bill at a premium. + * + * @generated from field: uint32 cache_write_input_tokens = 6; + */ + cacheWriteInputTokens = 0; + /** * The LLM provider that served the completion (e.g. "claude", "openai", "gemini"). * @@ -113,6 +129,7 @@ export class CompleteResponse extends Message { { no: 2, name: "input_tokens", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 3, name: "output_tokens", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 4, name: "cached_input_tokens", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, + { no: 6, name: "cache_write_input_tokens", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 5, name: "provider", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); diff --git a/web-common/src/proto/gen/rill/ai/v1/ai_pb.ts b/web-common/src/proto/gen/rill/ai/v1/ai_pb.ts index 32b93c4535f6..cc15e66fe719 100644 --- a/web-common/src/proto/gen/rill/ai/v1/ai_pb.ts +++ b/web-common/src/proto/gen/rill/ai/v1/ai_pb.ts @@ -189,6 +189,64 @@ export class ToolResult extends Message { } } +/** + * The model's reasoning before its text or tool calls. + * Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. + * + * @generated from message rill.ai.v1.Reasoning + */ +export class Reasoning extends Message { + /** + * Readable reasoning. Often empty, since providers may omit it or return a summary. + * + * @generated from field: string text = 1; + */ + text = ""; + + /** + * Opaque provider signature that lets the provider verify the reasoning. + * + * @generated from field: string signature = 2; + */ + signature = ""; + + /** + * Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. + * + * @generated from field: string redacted_data = 3; + */ + redactedData = ""; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "rill.ai.v1.Reasoning"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "text", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "signature", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "redacted_data", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): Reasoning { + return new Reasoning().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): Reasoning { + return new Reasoning().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): Reasoning { + return new Reasoning().fromJsonString(jsonString, options); + } + + static equals(a: Reasoning | PlainMessage | undefined, b: Reasoning | PlainMessage | undefined): boolean { + return proto3.util.equals(Reasoning, a, b); + } +} + /** * Content block within a message * @@ -216,6 +274,12 @@ export class ContentBlock extends Message { */ value: ToolResult; case: "toolResult"; + } | { + /** + * @generated from field: rill.ai.v1.Reasoning reasoning = 4; + */ + value: Reasoning; + case: "reasoning"; } | { case: undefined; value?: undefined } = { case: undefined }; constructor(data?: PartialMessage) { @@ -229,6 +293,7 @@ export class ContentBlock extends Message { { no: 1, name: "text", kind: "scalar", T: 9 /* ScalarType.STRING */, oneof: "block_type" }, { no: 2, name: "tool_call", kind: "message", T: ToolCall, oneof: "block_type" }, { no: 3, name: "tool_result", kind: "message", T: ToolResult, oneof: "block_type" }, + { no: 4, name: "reasoning", kind: "message", T: Reasoning, oneof: "block_type" }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ContentBlock { diff --git a/web-common/src/runtime-client/gen/index.schemas.ts b/web-common/src/runtime-client/gen/index.schemas.ts index f69f59a22f6a..0c2114c2e498 100644 --- a/web-common/src/runtime-client/gen/index.schemas.ts +++ b/web-common/src/runtime-client/gen/index.schemas.ts @@ -838,6 +838,7 @@ export interface V1ContentBlock { text?: string; toolCall?: V1ToolCall; toolResult?: V1ToolResult; + reasoning?: V1Reasoning; } export interface V1Conversation { @@ -2345,6 +2346,19 @@ export const V1ReconcileStatus = { RECONCILE_STATUS_RUNNING: "RECONCILE_STATUS_RUNNING", } as const; +/** + * The model's reasoning before its text or tool calls. +Some providers (e.g. Claude) require it to be passed back unmodified within a tool call loop. + */ +export interface V1Reasoning { + /** Readable reasoning. Often empty, since providers may omit it or return a summary. */ + text?: string; + /** Opaque provider signature that lets the provider verify the reasoning. */ + signature?: string; + /** Opaque encrypted reasoning, set instead of text and signature when the provider redacts the reasoning. */ + redactedData?: string; +} + export interface V1RefreshModelTrigger { /** The model to refresh. */ model?: string;