diff --git a/providertests/common_test.go b/providertests/common_test.go index 0f0316d52e52e4597954c960e4e0c46c0c3b0adc..4430a194416a52105c9f62a99743076700107df2 100644 --- a/providertests/common_test.go +++ b/providertests/common_test.go @@ -226,3 +226,60 @@ func testMultiTool(t *testing.T, pair builderPair) { checkResult(t, result) }) } + +func testThinking(t *testing.T, pairs []builderPair, thinkChecks func(*testing.T, *ai.AgentResult)) { + for _, pair := range pairs { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + require.NoError(t, err, "failed to build language model") + + type WeatherInput struct { + Location string `json:"location" description:"the city"` + } + + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "What's the weather in Florence, Italy?", + ProviderOptions: pair.providerOptions, + // ProviderOptions: ai.ProviderOptions{ + // "anthropic": &anthropic.ProviderOptions{ + // Thinking: &anthropic.ThinkingProviderOption{ + // BudgetTokens: 10_000, + // }, + // }, + // "google": &google.ProviderOptions{ + // ThinkingConfig: &google.ThinkingConfig{ + // ThinkingBudget: ai.IntOption(100), + // IncludeThoughts: ai.BoolOption(true), + // }, + // }, + // "openai": &openai.ProviderOptions{ + // ReasoningEffort: openai.ReasoningEffortOption(openai.ReasoningEffortMedium), + // }, + // }, + }) + require.NoError(t, err, "failed to generate") + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + + thinkChecks(t, result) + }) + } +} diff --git a/providertests/google_test.go b/providertests/google_test.go index 41d78f0fd4955b07d3285811459239ab26d0b2b7..d2eba21e62346d0e9693258bf853476e1888e48e 100644 --- a/providertests/google_test.go +++ b/providertests/google_test.go @@ -8,6 +8,7 @@ import ( "github.com/charmbracelet/fantasy/ai" "github.com/charmbracelet/fantasy/google" + "github.com/stretchr/testify/require" "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" ) @@ -16,6 +17,33 @@ func TestGoogleCommon(t *testing.T) { {"gemini-2.5-flash", builderGoogleGemini25Flash, nil}, {"gemini-2.5-pro", builderGoogleGemini25Pro, nil}, }) + opts := ai.ProviderOptions{ + google.Name: &google.ProviderOptions{ + ThinkingConfig: &google.ThinkingConfig{ + ThinkingBudget: ai.IntOption(100), + IncludeThoughts: ai.BoolOption(true), + }, + }, + } + testThinking(t, []builderPair{ + {"gemini-2.5-flash", builderGoogleGemini25Flash, opts}, + {"gemini-2.5-pro", builderGoogleGemini25Pro, opts}, + }, testGoogleThinking) +} + +func testGoogleThinking(t *testing.T, result *ai.AgentResult) { + reasoningContentCount := 0 + // Test if we got the signature + for _, step := range result.Steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + } + } + } + } + require.Greater(t, reasoningContentCount, 0) } func builderGoogleGemini25Flash(r *recorder.Recorder) (ai.LanguageModel, error) { diff --git a/providertests/testdata/TestGoogleCommon/gemini-2.5-flash.yaml b/providertests/testdata/TestGoogleCommon/gemini-2.5-flash.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cad11a8eb94671c02ada96759d017d1ba6fdad4e --- /dev/null +++ b/providertests/testdata/TestGoogleCommon/gemini-2.5-flash.yaml @@ -0,0 +1,130 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: generativelanguage.googleapis.com + body: | + {"contents":[{"parts":[{"text":"What's the weather in Florence, Italy?"}],"role":"user"}],"generationConfig":{"thinkingConfig":{"includeThoughts":true,"thinkingBudget":128}},"systemInstruction":{"parts":[{"text":"You are a helpful assistant"}],"role":"user"},"toolConfig":{"functionCallingConfig":{"mode":"AUTO"}},"tools":[{"functionDeclarations":[{"description":"Get weather information for a location","name":"weather","parameters":{"properties":{"location":{"description":"the city","type":"STRING"}},"required":["location"],"type":"OBJECT"}}]}]} + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "**Okay, let's break this down.**\n\nRight, I've got a user who needs the weather in Florence, Italy. Simple enough. Since I have the `weather` tool at my disposal, that's definitely the route to go. The key here is the `location` parameter within that tool. I need to specify \"Florence, Italy\" as the value for the `location` parameter. Therefore, the most direct and efficient approach is to call `default_api.weather(location=\"Florence, Italy\")`. That should get us exactly what we need.\n", + "thought": true + }, + { + "functionCall": { + "name": "weather", + "args": { + "location": "Florence, Italy" + } + }, + "thoughtSignature": "CrQCAdHtim/6CS/mj6pws3rQtKhb1HdlEqRkjQ85xgZKrCFA2v3+MTLIf1foIHl0I7YqRgRSUZQClztnsrrD7avdIcWMRKxC03GaSaPD5Z0ydXt6p4hufk/mmLTAARDWyjW2pVkT+Lk59+TwdUH1f7GsSmtXZJrwN70wnDTawFfuKSJhdaZ9r2ZPe6H9Yv+BfA+IRZ/r2no1aLtaFGSuQJvzqko+vADveXtEQRwHeUOE1ElKT76yHTP71XC6PakJfNIj0vnd+EMJh4iESDNAbWIoPNixXmNpINgTfZjeI12dgT2JWCH6IKmoAIehZSKXr5/SBY/jyKWYRSZwMzy5YCbf19//q+qby1M0eGgj4jqo176oD9LBM/PBEFI3oAgEwet1wT+Ssja1VJ40VwkRmErS1jdMdGQ=" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 54, + "candidatesTokenCount": 15, + "totalTokenCount": 134, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 54 + } + ], + "thoughtsTokenCount": 65 + }, + "modelVersion": "gemini-2.5-flash", + "responseId": "38LTaK2VKdjcnsEP84uh0AM" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 1.941863208s + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 776 + host: generativelanguage.googleapis.com + body: | + {"contents":[{"parts":[{"text":"What's the weather in Florence, Italy?"}],"role":"user"},{"parts":[{"functionCall":{"args":{"location":"Florence, Italy"},"id":"weather","name":"weather"}}],"role":"model"},{"parts":[{"functionResponse":{"id":"weather","name":"weather","response":{"result":"40 C"}}}],"role":"user"}],"generationConfig":{"thinkingConfig":{"includeThoughts":true,"thinkingBudget":128}},"systemInstruction":{"parts":[{"text":"You are a helpful assistant"}],"role":"user"},"toolConfig":{"functionCallingConfig":{"mode":"AUTO"}},"tools":[{"functionDeclarations":[{"description":"Get weather information for a location","name":"weather","parameters":{"properties":{"location":{"description":"the city","type":"STRING"}},"required":["location"],"type":"OBJECT"}}]}]} + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The weather in Florence, Italy is 40 degrees Celsius." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 84, + "candidatesTokenCount": 13, + "totalTokenCount": 97, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 84 + } + ] + }, + "modelVersion": "gemini-2.5-flash", + "responseId": "4MLTaJrLBJ7hnsEPwuCKqA0" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 407.813167ms diff --git a/providertests/testdata/TestGoogleCommon/gemini-2.5-pro.yaml b/providertests/testdata/TestGoogleCommon/gemini-2.5-pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a43ce9ce1383e44d21dfa6d1344876616e1aa67 --- /dev/null +++ b/providertests/testdata/TestGoogleCommon/gemini-2.5-pro.yaml @@ -0,0 +1,130 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: generativelanguage.googleapis.com + body: | + {"contents":[{"parts":[{"text":"What's the weather in Florence, Italy?"}],"role":"user"}],"generationConfig":{"thinkingConfig":{"includeThoughts":true,"thinkingBudget":128}},"systemInstruction":{"parts":[{"text":"You are a helpful assistant"}],"role":"user"},"toolConfig":{"functionCallingConfig":{"mode":"AUTO"}},"tools":[{"functionDeclarations":[{"description":"Get weather information for a location","name":"weather","parameters":{"properties":{"location":{"description":"the city","type":"STRING"}},"required":["location"],"type":"OBJECT"}}]}]} + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "**Okay, let's break this down.**\n\nI see the user is asking about the weather in Florence, Italy. That's a clear signal to use the `weather.get_weather` function. It's a straightforward request, so I need to translate \"Florence, Italy\" into the correct parameters for that function. The `get_weather` function needs a `location` parameter, and the user has helpfully given me exactly what I need: \"Florence, Italy\". Perfect. Now I just need to feed that information into the function.\n", + "thought": true + }, + { + "functionCall": { + "name": "weather", + "args": { + "location": "Florence, Italy" + } + }, + "thoughtSignature": "CpUCAdHtim+FkIKuYF/pMBD+SuDOrCu9MGrL4toE9OsH93oInrdaqZgJaEGI3FmSd1fyuYdx9mttRXxUhA9cjrC3NqIMUVYG7nXS6QcPOSi8qwO4MqYNWnllSpo/0jDdHem7id9bVJW7Y2SeKXO4kPK72QSonogI9ZDWmzRMhdoGg6FJ/5I1cx8Wnpc80kdXKmgWZNPOjJbt7IUqRucH+bRoWCTnUMuBjyAs4Ut3f55MYGmyn6bb8PRc+5/ZxN8OCFxcffc+TubEIr3VGBOZXxEyTLlTGK8RnCumpdph/a40UCSf8OFnN0FgxrPEa2cNUnGRvNTxMcSCnoz02k7cG+/7C/1S8ICEx6b/fDZoCzhEW0Z9oh/l9w==" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 54, + "candidatesTokenCount": 15, + "totalTokenCount": 133, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 54 + } + ], + "thoughtsTokenCount": 64 + }, + "modelVersion": "gemini-2.5-pro", + "responseId": "5MLTaIDkCJ6FkdUP-Py0kA0" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 4.03471325s + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 776 + host: generativelanguage.googleapis.com + body: | + {"contents":[{"parts":[{"text":"What's the weather in Florence, Italy?"}],"role":"user"},{"parts":[{"functionCall":{"args":{"location":"Florence, Italy"},"id":"weather","name":"weather"}}],"role":"model"},{"parts":[{"functionResponse":{"id":"weather","name":"weather","response":{"result":"40 C"}}}],"role":"user"}],"generationConfig":{"thinkingConfig":{"includeThoughts":true,"thinkingBudget":128}},"systemInstruction":{"parts":[{"text":"You are a helpful assistant"}],"role":"user"},"toolConfig":{"functionCallingConfig":{"mode":"AUTO"}},"tools":[{"functionDeclarations":[{"description":"Get weather information for a location","name":"weather","parameters":{"properties":{"location":{"description":"the city","type":"STRING"}},"required":["location"],"type":"OBJECT"}}]}]} + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "It is 40 C in Florence, Italy. \n" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 84, + "candidatesTokenCount": 11, + "totalTokenCount": 95, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 84 + } + ] + }, + "modelVersion": "gemini-2.5-pro", + "responseId": "5sLTaNHWA6j7nsEPm8f_0A0" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 1.913587958s