diff --git a/CRUSH.md b/CRUSH.md new file mode 100644 index 0000000000000000000000000000000000000000..ad5079839ef80b9e3b1bd752198d6bd23b2d95cf --- /dev/null +++ b/CRUSH.md @@ -0,0 +1,29 @@ +# CRUSH.md - Fantasy AI SDK + +## Build/Test/Lint Commands +- **Build**: `go build ./...` +- **Test all**: `task test` or `go test ./... -count=1` +- **Test single**: `go test -run TestName ./package -v` +- **Test with args**: `task test -- -v -run TestName` +- **Lint**: `task lint` or `golangci-lint run` +- **Format**: `task fmt` or `gofmt -s -w .` +- **Modernize**: `task modernize` or `modernize -fix ./...` + +## Code Style Guidelines +- **Package naming**: lowercase, single word (ai, openai, anthropic, google) +- **Imports**: standard library first, then third-party, then local packages +- **Error handling**: Use custom error types with structured fields, wrap with context +- **Types**: Use type aliases for function signatures (`type Option = func(*options)`) +- **Naming**: CamelCase for exported, camelCase for unexported +- **Constants**: Use const blocks with descriptive names (ProviderName, DefaultURL) +- **Structs**: Embed anonymous structs for composition (APICallError embeds *AIError) +- **Functions**: Return error as last parameter, use context.Context as first param +- **Testing**: Use testify/assert, table-driven tests, recorder pattern for HTTP mocking +- **Comments**: Godoc format for exported functions, explain complex logic inline +- **JSON**: Use struct tags for marshaling, handle empty values gracefully + +## Project Structure +- `/ai` - Core AI abstractions and agent logic +- `/openai`, `/anthropic`, `/google` - Provider implementations +- `/providertests` - Cross-provider integration tests with VCR recordings +- `/examples` - Usage examples for different patterns \ No newline at end of file diff --git a/ai/tool_test.go b/ai/tool_test.go index 2d11d899122596b1250b5a09ecccc4d15326cc50..8539c38f1e9f1c38b529c9db62988c6fd005f144 100644 --- a/ai/tool_test.go +++ b/ai/tool_test.go @@ -4,8 +4,9 @@ import ( "context" "fmt" "reflect" - "strings" "testing" + + "github.com/stretchr/testify/require" ) // Example of a simple typed tool using the function approach @@ -28,12 +29,9 @@ func TestTypedToolFuncExample(t *testing.T) { // Check the tool info info := tool.Info() - if info.Name != "calculator" { - t.Errorf("Expected tool name 'calculator', got %s", info.Name) - } - if len(info.Required) != 1 || info.Required[0] != "expression" { - t.Errorf("Expected required field 'expression', got %v", info.Required) - } + require.Equal(t, "calculator", info.Name) + require.Len(t, info.Required, 1) + require.Equal(t, "expression", info.Required[0]) // Test execution call := ToolCall{ @@ -43,15 +41,9 @@ func TestTypedToolFuncExample(t *testing.T) { } result, err := tool.Run(context.Background(), call) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if result.Content != "4" { - t.Errorf("Expected result '4', got %s", result.Content) - } - if result.IsError { - t.Errorf("Expected successful result, got error") - } + require.NoError(t, err) + require.Equal(t, "4", result.Content) + require.False(t, result.IsError) } func TestEnumToolExample(t *testing.T) { @@ -76,13 +68,10 @@ func TestEnumToolExample(t *testing.T) { // Check that the schema includes enum values info := tool.Info() unitsParam, ok := info.Parameters["units"].(map[string]any) - if !ok { - t.Fatal("Expected units parameter to exist") - } + require.True(t, ok, "Expected units parameter to exist") enumValues, ok := unitsParam["enum"].([]any) - if !ok || len(enumValues) != 2 { - t.Errorf("Expected 2 enum values, got %v", enumValues) - } + require.True(t, ok) + require.Len(t, enumValues, 2) // Test execution with enum value call := ToolCall{ @@ -92,15 +81,9 @@ func TestEnumToolExample(t *testing.T) { } result, err := tool.Run(context.Background(), call) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !strings.Contains(result.Content, "San Francisco") { - t.Errorf("Expected result to contain 'San Francisco', got %s", result.Content) - } - if !strings.Contains(result.Content, "72°F") { - t.Errorf("Expected result to contain '72°F', got %s", result.Content) - } + require.NoError(t, err) + require.Contains(t, result.Content, "San Francisco") + require.Contains(t, result.Content, "72°F") } func TestEnumSupport(t *testing.T) { @@ -113,30 +96,20 @@ func TestEnumSupport(t *testing.T) { schema := generateSchema(reflect.TypeOf(WeatherInput{})) - if schema.Type != "object" { - t.Errorf("Expected schema type 'object', got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) // Check units field has enum values unitsSchema := schema.Properties["units"] - if unitsSchema == nil { - t.Fatal("Expected units property to exist") - } - if len(unitsSchema.Enum) != 3 { - t.Errorf("Expected 3 enum values for units, got %d", len(unitsSchema.Enum)) - } + require.NotNil(t, unitsSchema, "Expected units property to exist") + require.Len(t, unitsSchema.Enum, 3) expectedUnits := []string{"celsius", "fahrenheit", "kelvin"} for i, expected := range expectedUnits { - if unitsSchema.Enum[i] != expected { - t.Errorf("Expected enum value %s, got %v", expected, unitsSchema.Enum[i]) - } + require.Equal(t, expected, unitsSchema.Enum[i]) } // Check required fields (format should not be required due to omitempty) expectedRequired := []string{"location", "units"} - if len(schema.Required) != len(expectedRequired) { - t.Errorf("Expected %d required fields, got %d", len(expectedRequired), len(schema.Required)) - } + require.Len(t, schema.Required, len(expectedRequired)) } func TestSchemaToParameters(t *testing.T) { @@ -170,43 +143,24 @@ func TestSchemaToParameters(t *testing.T) { // Check name parameter nameParam, ok := params["name"].(map[string]any) - if !ok { - t.Fatal("Expected name parameter to exist") - } - if nameParam["type"] != "string" { - t.Errorf("Expected name type 'string', got %v", nameParam["type"]) - } - if nameParam["description"] != "The name field" { - t.Errorf("Expected name description 'The name field', got %v", nameParam["description"]) - } + require.True(t, ok, "Expected name parameter to exist") + require.Equal(t, "string", nameParam["type"]) + require.Equal(t, "The name field", nameParam["description"]) // Check age parameter with min/max ageParam, ok := params["age"].(map[string]any) - if !ok { - t.Fatal("Expected age parameter to exist") - } - if ageParam["type"] != "integer" { - t.Errorf("Expected age type 'integer', got %v", ageParam["type"]) - } - if ageParam["minimum"] != 0.0 { - t.Errorf("Expected age minimum 0, got %v", ageParam["minimum"]) - } - if ageParam["maximum"] != 120.0 { - t.Errorf("Expected age maximum 120, got %v", ageParam["maximum"]) - } + require.True(t, ok, "Expected age parameter to exist") + require.Equal(t, "integer", ageParam["type"]) + require.Equal(t, 0.0, ageParam["minimum"]) + require.Equal(t, 120.0, ageParam["maximum"]) // Check priority parameter with enum priorityParam, ok := params["priority"].(map[string]any) - if !ok { - t.Fatal("Expected priority parameter to exist") - } - if priorityParam["type"] != "string" { - t.Errorf("Expected priority type 'string', got %v", priorityParam["type"]) - } + require.True(t, ok, "Expected priority parameter to exist") + require.Equal(t, "string", priorityParam["type"]) enumValues, ok := priorityParam["enum"].([]any) - if !ok || len(enumValues) != 3 { - t.Errorf("Expected 3 enum values, got %v", enumValues) - } + require.True(t, ok) + require.Len(t, enumValues, 3) } func TestGenerateSchemaBasicTypes(t *testing.T) { @@ -258,9 +212,7 @@ func TestGenerateSchemaBasicTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected.Type { - t.Errorf("Expected type %s, got %s", tt.expected.Type, schema.Type) - } + require.Equal(t, tt.expected.Type, schema.Type) }) } } @@ -303,15 +255,9 @@ func TestGenerateSchemaArrayTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected.Type { - t.Errorf("Expected type %s, got %s", tt.expected.Type, schema.Type) - } - if schema.Items == nil { - t.Fatal("Expected items schema to exist") - } - if schema.Items.Type != tt.expected.Items.Type { - t.Errorf("Expected items type %s, got %s", tt.expected.Items.Type, schema.Items.Type) - } + require.Equal(t, tt.expected.Type, schema.Type) + require.NotNil(t, schema.Items, "Expected items schema to exist") + require.Equal(t, tt.expected.Items.Type, schema.Items.Type) }) } } @@ -345,9 +291,7 @@ func TestGenerateSchemaMapTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected { - t.Errorf("Expected type %s, got %s", tt.expected, schema.Type) - } + require.Equal(t, tt.expected, schema.Type) }) } } @@ -384,60 +328,36 @@ func TestGenerateSchemaStructTypes(t *testing.T) { name: "simple struct", input: SimpleStruct{}, validate: func(t *testing.T, schema Schema) { - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } - if len(schema.Properties) != 2 { - t.Errorf("Expected 2 properties, got %d", len(schema.Properties)) - } - if schema.Properties["name"] == nil { - t.Error("Expected name property to exist") - } - if schema.Properties["name"].Description != "The name field" { - t.Errorf("Expected description 'The name field', got %s", schema.Properties["name"].Description) - } - if len(schema.Required) != 2 { - t.Errorf("Expected 2 required fields, got %d", len(schema.Required)) - } + require.Equal(t, "object", schema.Type) + require.Len(t, schema.Properties, 2) + require.NotNil(t, schema.Properties["name"], "Expected name property to exist") + require.Equal(t, "The name field", schema.Properties["name"].Description) + require.Len(t, schema.Required, 2) }, }, { name: "struct with omitempty", input: StructWithOmitEmpty{}, validate: func(t *testing.T, schema Schema) { - if len(schema.Required) != 1 { - t.Errorf("Expected 1 required field, got %d", len(schema.Required)) - } - if schema.Required[0] != "required" { - t.Errorf("Expected required field 'required', got %s", schema.Required[0]) - } + require.Len(t, schema.Required, 1) + require.Equal(t, "required", schema.Required[0]) }, }, { name: "struct with json ignore", input: StructWithJSONIgnore{}, validate: func(t *testing.T, schema Schema) { - if len(schema.Properties) != 1 { - t.Errorf("Expected 1 property, got %d", len(schema.Properties)) - } - if schema.Properties["visible"] == nil { - t.Error("Expected visible property to exist") - } - if schema.Properties["hidden"] != nil { - t.Error("Expected hidden property to not exist") - } + require.Len(t, schema.Properties, 1) + require.NotNil(t, schema.Properties["visible"], "Expected visible property to exist") + require.Nil(t, schema.Properties["hidden"], "Expected hidden property to not exist") }, }, { name: "struct without json tags", input: StructWithoutJSONTags{}, validate: func(t *testing.T, schema Schema) { - if schema.Properties["first_name"] == nil { - t.Error("Expected first_name property to exist") - } - if schema.Properties["last_name"] == nil { - t.Error("Expected last_name property to exist") - } + require.NotNil(t, schema.Properties["first_name"], "Expected first_name property to exist") + require.NotNil(t, schema.Properties["last_name"], "Expected last_name property to exist") }, }, } @@ -461,23 +381,13 @@ func TestGenerateSchemaPointerTypes(t *testing.T) { schema := generateSchema(reflect.TypeOf(StructWithPointers{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["name"] == nil { - t.Fatal("Expected name property to exist") - } - if schema.Properties["name"].Type != "string" { - t.Errorf("Expected name type string, got %s", schema.Properties["name"].Type) - } + require.NotNil(t, schema.Properties["name"], "Expected name property to exist") + require.Equal(t, "string", schema.Properties["name"].Type) - if schema.Properties["age"] == nil { - t.Fatal("Expected age property to exist") - } - if schema.Properties["age"].Type != "integer" { - t.Errorf("Expected age type integer, got %s", schema.Properties["age"].Type) - } + require.NotNil(t, schema.Properties["age"], "Expected age property to exist") + require.Equal(t, "integer", schema.Properties["age"].Type) } func TestGenerateSchemaNestedStructs(t *testing.T) { @@ -495,25 +405,15 @@ func TestGenerateSchemaNestedStructs(t *testing.T) { schema := generateSchema(reflect.TypeOf(Person{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["address"] == nil { - t.Fatal("Expected address property to exist") - } + require.NotNil(t, schema.Properties["address"], "Expected address property to exist") addressSchema := schema.Properties["address"] - if addressSchema.Type != "object" { - t.Errorf("Expected address type object, got %s", addressSchema.Type) - } + require.Equal(t, "object", addressSchema.Type) - if addressSchema.Properties["street"] == nil { - t.Error("Expected street property in address to exist") - } - if addressSchema.Properties["city"] == nil { - t.Error("Expected city property in address to exist") - } + require.NotNil(t, addressSchema.Properties["street"], "Expected street property in address to exist") + require.NotNil(t, addressSchema.Properties["city"], "Expected city property in address to exist") } func TestGenerateSchemaRecursiveStructs(t *testing.T) { @@ -526,23 +426,15 @@ func TestGenerateSchemaRecursiveStructs(t *testing.T) { schema := generateSchema(reflect.TypeOf(Node{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["value"] == nil { - t.Error("Expected value property to exist") - } + require.NotNil(t, schema.Properties["value"], "Expected value property to exist") - if schema.Properties["next"] == nil { - t.Error("Expected next property to exist") - } + require.NotNil(t, schema.Properties["next"], "Expected next property to exist") // The recursive reference should be handled gracefully nextSchema := schema.Properties["next"] - if nextSchema.Type != "object" { - t.Errorf("Expected next type object, got %s", nextSchema.Type) - } + require.Equal(t, "object", nextSchema.Type) } func TestGenerateSchemaWithEnumTags(t *testing.T) { @@ -558,33 +450,21 @@ func TestGenerateSchemaWithEnumTags(t *testing.T) { // Check level field levelSchema := schema.Properties["level"] - if levelSchema == nil { - t.Fatal("Expected level property to exist") - } - if len(levelSchema.Enum) != 4 { - t.Errorf("Expected 4 enum values for level, got %d", len(levelSchema.Enum)) - } + require.NotNil(t, levelSchema, "Expected level property to exist") + require.Len(t, levelSchema.Enum, 4) expectedLevels := []string{"debug", "info", "warn", "error"} for i, expected := range expectedLevels { - if levelSchema.Enum[i] != expected { - t.Errorf("Expected enum value %s, got %v", expected, levelSchema.Enum[i]) - } + require.Equal(t, expected, levelSchema.Enum[i]) } // Check format field formatSchema := schema.Properties["format"] - if formatSchema == nil { - t.Fatal("Expected format property to exist") - } - if len(formatSchema.Enum) != 2 { - t.Errorf("Expected 2 enum values for format, got %d", len(formatSchema.Enum)) - } + require.NotNil(t, formatSchema, "Expected format property to exist") + require.Len(t, formatSchema.Enum, 2) // Check required fields (optional should not be required due to omitempty) expectedRequired := []string{"level", "format"} - if len(schema.Required) != len(expectedRequired) { - t.Errorf("Expected %d required fields, got %d", len(expectedRequired), len(schema.Required)) - } + require.Len(t, schema.Required, len(expectedRequired)) } func TestGenerateSchemaComplexTypes(t *testing.T) { @@ -601,45 +481,25 @@ func TestGenerateSchemaComplexTypes(t *testing.T) { // Check string slice stringSliceSchema := schema.Properties["string_slice"] - if stringSliceSchema == nil { - t.Fatal("Expected string_slice property to exist") - } - if stringSliceSchema.Type != "array" { - t.Errorf("Expected string_slice type array, got %s", stringSliceSchema.Type) - } - if stringSliceSchema.Items.Type != "string" { - t.Errorf("Expected string_slice items type string, got %s", stringSliceSchema.Items.Type) - } + require.NotNil(t, stringSliceSchema, "Expected string_slice property to exist") + require.Equal(t, "array", stringSliceSchema.Type) + require.Equal(t, "string", stringSliceSchema.Items.Type) // Check int map intMapSchema := schema.Properties["int_map"] - if intMapSchema == nil { - t.Fatal("Expected int_map property to exist") - } - if intMapSchema.Type != "object" { - t.Errorf("Expected int_map type object, got %s", intMapSchema.Type) - } + require.NotNil(t, intMapSchema, "Expected int_map property to exist") + require.Equal(t, "object", intMapSchema.Type) // Check nested slice nestedSliceSchema := schema.Properties["nested_slice"] - if nestedSliceSchema == nil { - t.Fatal("Expected nested_slice property to exist") - } - if nestedSliceSchema.Type != "array" { - t.Errorf("Expected nested_slice type array, got %s", nestedSliceSchema.Type) - } - if nestedSliceSchema.Items.Type != "object" { - t.Errorf("Expected nested_slice items type object, got %s", nestedSliceSchema.Items.Type) - } + require.NotNil(t, nestedSliceSchema, "Expected nested_slice property to exist") + require.Equal(t, "array", nestedSliceSchema.Type) + require.Equal(t, "object", nestedSliceSchema.Items.Type) // Check interface interfaceSchema := schema.Properties["interface"] - if interfaceSchema == nil { - t.Fatal("Expected interface property to exist") - } - if interfaceSchema.Type != "object" { - t.Errorf("Expected interface type object, got %s", interfaceSchema.Type) - } + require.NotNil(t, interfaceSchema, "Expected interface property to exist") + require.Equal(t, "object", interfaceSchema.Type) } func TestToSnakeCase(t *testing.T) { @@ -664,9 +524,7 @@ func TestToSnakeCase(t *testing.T) { t.Run(tt.input, func(t *testing.T) { t.Parallel() result := toSnakeCase(tt.input) - if result != tt.expected { - t.Errorf("toSnakeCase(%s) = %s, expected %s", tt.input, result, tt.expected) - } + require.Equal(t, tt.expected, result, "toSnakeCase(%s)", tt.input) }) } } @@ -740,21 +598,14 @@ func TestSchemaToParametersEdgeCases(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() result := schemaToParameters(tt.schema) - if len(result) != len(tt.expected) { - t.Errorf("Expected %d parameters, got %d", len(tt.expected), len(result)) - } + require.Len(t, result, len(tt.expected)) for key, expectedValue := range tt.expected { - if result[key] == nil { - t.Errorf("Expected parameter %s to exist", key) - continue - } + require.NotNil(t, result[key], "Expected parameter %s to exist", key) // Deep comparison would be complex, so we'll check key properties resultParam := result[key].(map[string]any) expectedParam := expectedValue.(map[string]any) for propKey, propValue := range expectedParam { - if resultParam[propKey] != propValue { - t.Errorf("Expected %s.%s = %v, got %v", key, propKey, propValue, resultParam[propKey]) - } + require.Equal(t, propValue, resultParam[propKey], "Expected %s.%s", key, propKey) } } }) diff --git a/anthropic/anthropic.go b/anthropic/anthropic.go index 7a6eaf02bdb8d48bdad5800bbcdc70d1216f6a41..8598db459f8f742a19e604d695e9ef1cec310cd1 100644 --- a/anthropic/anthropic.go +++ b/anthropic/anthropic.go @@ -18,8 +18,8 @@ import ( ) const ( - ProviderName = "anthropic" - DefaultURL = "https://api.anthropic.com" + Name = "anthropic" + DefaultURL = "https://api.anthropic.com" ) type options struct { @@ -45,7 +45,7 @@ func New(opts ...Option) ai.Provider { } providerOptions.baseURL = cmp.Or(providerOptions.baseURL, DefaultURL) - providerOptions.name = cmp.Or(providerOptions.name, ProviderName) + providerOptions.name = cmp.Or(providerOptions.name, Name) return &provider{options: providerOptions} } @@ -97,7 +97,7 @@ func (a *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { } return languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.messages", a.options.name), + provider: a.options.name, options: a.options, client: anthropic.NewClient(anthropicClientOptions...), }, nil diff --git a/anthropic/provider_options.go b/anthropic/provider_options.go index 5ccfaf6a5e6ec9c8ac29ef7ca66e0c209a505355..e582edf41cc111f32a5bb3b0a48c05b731c0b904 100644 --- a/anthropic/provider_options.go +++ b/anthropic/provider_options.go @@ -2,8 +2,6 @@ package anthropic import "github.com/charmbracelet/fantasy/ai" -const Name = "anthropic" - type ProviderOptions struct { SendReasoning *bool `json:"send_reasoning"` Thinking *ThinkingProviderOption `json:"thinking"` diff --git a/cspell.json b/cspell.json index 4436732c9d075240149890a296a289b16940ac79..dad6dc4e9a6dc9e6e42c09d5e920ffae10360f3e 100644 --- a/cspell.json +++ b/cspell.json @@ -1,9 +1 @@ -{ - "language": "en", - "version": "0.2", - "flagWords": [], - "words": [ - "mapstructure", - "mapstructure" - ] -} +{"language":"en","words":["mapstructure","mapstructure","charmbracelet","providertests","joho","godotenv","stretchr"],"version":"0.2","flagWords":[]} \ No newline at end of file diff --git a/google/google.go b/google/google.go index 78325100ea698f00fe7ba6a55c75c6a718bfa6c9..f82ca46bb1902b4f70bc6e83b3e0df343e9696af 100644 --- a/google/google.go +++ b/google/google.go @@ -17,6 +17,8 @@ import ( "google.golang.org/genai" ) +const Name = "google" + type provider struct { options options } @@ -38,7 +40,7 @@ func New(opts ...Option) ai.Provider { o(&options) } - options.name = cmp.Or(options.name, "google") + options.name = cmp.Or(options.name, Name) return &provider{ options: options, @@ -101,7 +103,7 @@ func (g *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { } return &languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.generative-ai", g.options.name), + provider: g.options.name, providerOptions: g.options, client: client, }, nil @@ -120,16 +122,26 @@ func (a languageModel) prepareParams(call ai.Call) (*genai.GenerateContentConfig systemInstructions, content, warnings := toGooglePrompt(call.Prompt) - if providerOptions.ThinkingConfig != nil && - providerOptions.ThinkingConfig.IncludeThoughts != nil && - *providerOptions.ThinkingConfig.IncludeThoughts && - strings.HasPrefix(a.provider, "google.vertex.") { - warnings = append(warnings, ai.CallWarning{ - Type: ai.CallWarningTypeOther, - Message: "The 'includeThoughts' option is only supported with the Google Vertex provider " + - "and might not be supported or could behave unexpectedly with the current Google provider " + - fmt.Sprintf("(%s)", a.provider), - }) + if providerOptions.ThinkingConfig != nil { + if providerOptions.ThinkingConfig.IncludeThoughts != nil && + *providerOptions.ThinkingConfig.IncludeThoughts && + strings.HasPrefix(a.provider, "google.vertex.") { + warnings = append(warnings, ai.CallWarning{ + Type: ai.CallWarningTypeOther, + Message: "The 'includeThoughts' option is only supported with the Google Vertex provider " + + "and might not be supported or could behave unexpectedly with the current Google provider " + + fmt.Sprintf("(%s)", a.provider), + }) + } + + if providerOptions.ThinkingConfig.ThinkingBudget != nil && + *providerOptions.ThinkingConfig.ThinkingBudget < 128 { + warnings = append(warnings, ai.CallWarning{ + Type: ai.CallWarningTypeOther, + Message: "The 'thinking_budget' option can not be under 128 and will be set to 128 by default", + }) + providerOptions.ThinkingConfig.ThinkingBudget = ai.IntOption(128) + } } isGemmaModel := strings.HasPrefix(strings.ToLower(a.modelID), "gemma-") diff --git a/google/provider_options.go b/google/provider_options.go index 703c277a81fe60549da8c04cd0bf5f9805a7c197..945bff84740fe9b3bab2a0031442d8ae352265dd 100644 --- a/google/provider_options.go +++ b/google/provider_options.go @@ -1,7 +1,5 @@ package google -const Name = "google" - type ThinkingConfig struct { ThinkingBudget *int64 `json:"thinking_budget"` IncludeThoughts *bool `json:"include_thoughts"` diff --git a/openai/openai.go b/openai/openai.go index 4451aba206297ab53885ca4f40c503a79dc67e70..e7d51767af75ff38e5fe1ad810588ec3ca1fe0f3 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -21,8 +21,8 @@ import ( ) const ( - ProviderName = "openai" - DefaultURL = "https://api.openai.com/v1" + Name = "openai" + DefaultURL = "https://api.openai.com/v1" ) type provider struct { @@ -50,7 +50,7 @@ func New(opts ...Option) ai.Provider { } providerOptions.baseURL = cmp.Or(providerOptions.baseURL, DefaultURL) - providerOptions.name = cmp.Or(providerOptions.name, ProviderName) + providerOptions.name = cmp.Or(providerOptions.name, Name) if providerOptions.organization != "" { providerOptions.headers["OpenAi-Organization"] = providerOptions.organization @@ -124,7 +124,7 @@ func (o *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { return languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.chat", o.options.name), + provider: o.options.name, options: o.options, client: openai.NewClient(openaiClientOptions...), }, nil diff --git a/openai/provider_options.go b/openai/provider_options.go index af3d86fdf45eb9e8d2256602816a327b2b582ba9..617604883cfe71a1e850e470a3966afed872ae81 100644 --- a/openai/provider_options.go +++ b/openai/provider_options.go @@ -5,8 +5,6 @@ import ( "github.com/openai/openai-go/v2" ) -const Name = "openai" - type ReasoningEffort string const ( diff --git a/providertests/builders_test.go b/providertests/builders_test.go index 6a6c7652dd6d27af686dab72bf1b2f2ac5c3c462..c7e5add7335e65617ee7ae92a77e96e8633f7714 100644 --- a/providertests/builders_test.go +++ b/providertests/builders_test.go @@ -27,6 +27,12 @@ var languageModelBuilders = []builderPair{ {"google-gemini-2.5-pro", builderGoogleGemini25Pro}, } +var thinkingLanguageModelBuilders = []builderPair{ + {"openai-gpt-5", builderOpenaiGpt5}, + {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4}, + {"google-gemini-2.5-pro", builderGoogleGemini25Pro}, +} + func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { provider := openai.New( openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), @@ -43,6 +49,14 @@ func builderOpenaiGpt4oMini(r *recorder.Recorder) (ai.LanguageModel, error) { return provider.LanguageModel("gpt-4o-mini") } +func builderOpenaiGpt5(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := openai.New( + openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), + openai.WithHTTPClient(&http.Client{Transport: r}), + ) + return provider.LanguageModel("gpt-5") +} + func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { provider := anthropic.New( anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), diff --git a/providertests/provider_test.go b/providertests/provider_test.go index 96ebf64daf991f314fa522b417a87b5814493c35..acc30e78d7b9e0823cfc45f4e95174b27482df7f 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -7,7 +7,11 @@ import ( "testing" "github.com/charmbracelet/fantasy/ai" + "github.com/charmbracelet/fantasy/anthropic" + "github.com/charmbracelet/fantasy/google" + "github.com/charmbracelet/fantasy/openai" _ "github.com/joho/godotenv/autoload" + "github.com/stretchr/testify/require" ) func TestSimple(t *testing.T) { @@ -16,9 +20,7 @@ func TestSimple(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") agent := ai.NewAgent( languageModel, @@ -27,16 +29,12 @@ func TestSimple(t *testing.T) { result, err := agent.Generate(t.Context(), ai.AgentCall{ Prompt: "Say hi in Portuguese", }) - if err != nil { - t.Fatalf("failed to generate: %v", err) - } + require.NoError(t, err, "failed to generate") option1 := "Oi" option2 := "Olá" got := result.Response.Content.Text() - if !strings.Contains(got, option1) && !strings.Contains(got, option2) { - t.Fatalf("unexpected response: got %q, want %q or %q", got, option1, option2) - } + require.True(t, strings.Contains(got, option1) || strings.Contains(got, option2), "unexpected response: got %q, want %q or %q", got, option1, option2) }) } } @@ -47,9 +45,7 @@ func TestTool(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") type WeatherInput struct { Location string `json:"location" description:"the city"` @@ -71,16 +67,124 @@ func TestTool(t *testing.T) { result, err := agent.Generate(t.Context(), ai.AgentCall{ Prompt: "What's the weather in Florence?", }) - if err != nil { - t.Fatalf("failed to generate: %v", err) + require.NoError(t, err, "failed to generate") + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + }) + } +} + +func TestThinking(t *testing.T) { + for _, pair := range thinkingLanguageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + require.NoError(t, err, "failed to build language model") + + type WeatherInput struct { + Location string `json:"location" description:"the city"` } + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "What's the weather in Florence, Italy?", + ProviderOptions: ai.ProviderOptions{ + "anthropic": &anthropic.ProviderOptions{ + Thinking: &anthropic.ThinkingProviderOption{ + BudgetTokens: 10_000, + }, + }, + "google": &google.ProviderOptions{ + ThinkingConfig: &google.ThinkingConfig{ + ThinkingBudget: ai.IntOption(100), + IncludeThoughts: ai.BoolOption(true), + }, + }, + "openai": &openai.ProviderOptions{ + ReasoningEffort: openai.ReasoningEffortOption(openai.ReasoningEffortMedium), + }, + }, + }) + require.NoError(t, err, "failed to generate") + want1 := "Florence" want2 := "40" got := result.Response.Content.Text() - if !strings.Contains(got, want1) || !strings.Contains(got, want2) { - t.Fatalf("unexpected response: got %q, want %q %q", got, want1, want2) + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + + testThinkingSteps(t, languageModel.Provider(), result.Steps) + }) + } +} + +func TestThinkingStreaming(t *testing.T) { + for _, pair := range thinkingLanguageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + require.NoError(t, err, "failed to build language model") + + type WeatherInput struct { + Location string `json:"location" description:"the city"` } + + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Stream(t.Context(), ai.AgentStreamCall{ + Prompt: "What's the weather in Florence, Italy?", + ProviderOptions: ai.ProviderOptions{ + "anthropic": &anthropic.ProviderOptions{ + Thinking: &anthropic.ThinkingProviderOption{ + BudgetTokens: 10_000, + }, + }, + "google": &google.ProviderOptions{ + ThinkingConfig: &google.ThinkingConfig{ + ThinkingBudget: ai.IntOption(100), + IncludeThoughts: ai.BoolOption(true), + }, + }, + "openai": &openai.ProviderOptions{ + ReasoningEffort: openai.ReasoningEffortOption(openai.ReasoningEffortMedium), + }, + }, + }) + require.NoError(t, err, "failed to generate") + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + + testThinkingSteps(t, languageModel.Provider(), result.Steps) }) } } @@ -91,9 +195,7 @@ func TestStream(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") agent := ai.NewAgent( languageModel, @@ -118,32 +220,20 @@ func TestStream(t *testing.T) { } result, err := agent.Stream(t.Context(), streamCall) - if err != nil { - t.Fatalf("failed to stream: %v", err) - } + require.NoError(t, err, "failed to stream") finalText := result.Response.Content.Text() - if finalText == "" { - t.Fatal("expected non-empty response") - } + require.NotEmpty(t, finalText, "expected non-empty response") - if !strings.Contains(strings.ToLower(finalText), "uno") || - !strings.Contains(strings.ToLower(finalText), "dos") || - !strings.Contains(strings.ToLower(finalText), "tres") { - t.Fatalf("unexpected response: %q", finalText) - } + require.True(t, strings.Contains(strings.ToLower(finalText), "uno") && + strings.Contains(strings.ToLower(finalText), "dos") && + strings.Contains(strings.ToLower(finalText), "tres"), "unexpected response: %q", finalText) - if textDeltaCount == 0 { - t.Fatal("expected at least one text delta callback") - } + require.Greater(t, textDeltaCount, 0, "expected at least one text delta callback") - if stepCount == 0 { - t.Fatal("expected at least one step finish callback") - } + require.Greater(t, stepCount, 0, "expected at least one step finish callback") - if collectedText.String() == "" { - t.Fatal("expected collected text from deltas to be non-empty") - } + require.NotEmpty(t, collectedText.String(), "expected collected text from deltas to be non-empty") }) } } @@ -154,9 +244,7 @@ func TestStreamWithTools(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") type CalculatorInput struct { A int `json:"a" description:"first number"` @@ -190,9 +278,7 @@ func TestStreamWithTools(t *testing.T) { }, OnToolCall: func(toolCall ai.ToolCallContent) error { toolCallCount++ - if toolCall.ToolName != "add" { - t.Errorf("unexpected tool name: %s", toolCall.ToolName) - } + require.Equal(t, "add", toolCall.ToolName, "unexpected tool name") return nil }, OnToolResult: func(result ai.ToolResultContent) error { @@ -202,22 +288,14 @@ func TestStreamWithTools(t *testing.T) { } result, err := agent.Stream(t.Context(), streamCall) - if err != nil { - t.Fatalf("failed to stream: %v", err) - } + require.NoError(t, err, "failed to stream") finalText := result.Response.Content.Text() - if !strings.Contains(finalText, "42") { - t.Fatalf("expected response to contain '42', got: %q", finalText) - } + require.Contains(t, finalText, "42", "expected response to contain '42', got: %q", finalText) - if toolCallCount == 0 { - t.Fatal("expected at least one tool call") - } + require.Greater(t, toolCallCount, 0, "expected at least one tool call") - if toolResultCount == 0 { - t.Fatal("expected at least one tool result") - } + require.Greater(t, toolResultCount, 0, "expected at least one tool result") }) } } diff --git a/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a554d7b3d6b8265095a14259552834c6cb1f719 --- /dev/null +++ b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01NmYsbcWZbtPpmV1aMa5WWT\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have a weather function available that takes a location parameter. The user has provided the location as \\\"Florence, Italy\\\" which is specific enough for the weather function.\\n\\nLet me call the weather function with \\\"Florence, Italy\\\" as the location parameter.\",\"signature\":\"EuwDCkYIBxgCKkChgOvL+rOlboiQFkEOC20rmj1/Xs3mTGfMFk5lIVU0H0drGyFYAl+5JU5PoWng2ZU7J9EpJrLUonCw9KBjS78oEgzAegs6pV953eMRkQAaDKJXIOEXcqfXFXnNayIwweUSskDSybgjCXZOKTQBm5xBlvThzhK75k4zycqwZpx3zeDZrdaV/+MIjgK1GAVqKtMCWp9QcmFNxVmwMGsORlN0zS3KY+3Xgd1D489b1lMG+FT8t1Xy2HxDBLlk9XY6HUQK7nN3HNXu/liglYnLT0weuYHsrzp8QgVrmgSWKLtX2pCI6SB8Df+9oQLzppw81d9+Vm3o7aJeI4nzwMxmZRekUu2j3LJiBFq5iQEAYnaGchWJ5B60mT5dk3UhnjTJYjVfaqgTHqybIwZ0ZrkAho4cybEwmQV7fCNsVIDom3v2XwDQF2TLeOGp/uFNElP4mpzQsB7k9x4asSb/kMsW8N34E5oWevGYyWDsX6c1NkTcJ+afmVN0df8i77bzFwtkrSz7/N85vX85rUxNxXCUfUiX5RkXq1ZHEL/y34ecpa9lP2CikFATgYKTfFQfc1x84LAC2aiBsDTKZFaZZocJcTHbO/PC1Ui+n4Ef8z33epy+AmGELkXG0CPgp6cqB08+AgoFlH78GAE=\"},{\"type\":\"tool_use\",\"id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"name\":\"weather\",\"input\":{\"location\":\"Florence, Italy\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":125,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.398491s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 1879 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"EuwDCkYIBxgCKkChgOvL+rOlboiQFkEOC20rmj1/Xs3mTGfMFk5lIVU0H0drGyFYAl+5JU5PoWng2ZU7J9EpJrLUonCw9KBjS78oEgzAegs6pV953eMRkQAaDKJXIOEXcqfXFXnNayIwweUSskDSybgjCXZOKTQBm5xBlvThzhK75k4zycqwZpx3zeDZrdaV/+MIjgK1GAVqKtMCWp9QcmFNxVmwMGsORlN0zS3KY+3Xgd1D489b1lMG+FT8t1Xy2HxDBLlk9XY6HUQK7nN3HNXu/liglYnLT0weuYHsrzp8QgVrmgSWKLtX2pCI6SB8Df+9oQLzppw81d9+Vm3o7aJeI4nzwMxmZRekUu2j3LJiBFq5iQEAYnaGchWJ5B60mT5dk3UhnjTJYjVfaqgTHqybIwZ0ZrkAho4cybEwmQV7fCNsVIDom3v2XwDQF2TLeOGp/uFNElP4mpzQsB7k9x4asSb/kMsW8N34E5oWevGYyWDsX6c1NkTcJ+afmVN0df8i77bzFwtkrSz7/N85vX85rUxNxXCUfUiX5RkXq1ZHEL/y34ecpa9lP2CikFATgYKTfFQfc1x84LAC2aiBsDTKZFaZZocJcTHbO/PC1Ui+n4Ef8z33epy+AmGELkXG0CPgp6cqB08+AgoFlH78GAE=\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have a weather function available that takes a location parameter. The user has provided the location as \\\"Florence, Italy\\\" which is specific enough for the weather function.\\n\\nLet me call the weather function with \\\"Florence, Italy\\\" as the location parameter.\",\"type\":\"thinking\"},{\"id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"input\":{\"location\":\"Florence, Italy\"},\"name\":\"weather\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"content\":[{\"text\":\"40 C\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01DyEBuCyDtP4cjnhtqdbqrh\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"The current weather in Florence, Italy is 40°C (104°F). That's quite hot! It seems like a very warm day in Florence.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":563,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":35,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.744995167s diff --git a/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b927c7edd35572aac4e9337693d5fd346ac52f73 --- /dev/null +++ b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml @@ -0,0 +1,59 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"**Getting the Weather in Florence**\\n\\nOkay, so I see the user wants the weather for \\\"Florence, Italy.\\\" My initial thought is to grab the `weather.get_weather` tool. Perfect, it has a `location` parameter, which is exactly what I need. The user has thoughtfully provided \\\"Florence, Italy,\\\" so that's a straightforward input for the `location` parameter. This is a quick and efficient request to handle.\\n\",\n \"thought\": true\n },\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence, Italy\"\n }\n },\n \"thoughtSignature\": \"CokDAdHtim93vKtHJYI78AitJBYITb44JuVhBViGlnpnj3bSPvRBDI3GF8joEA68HpEu4qw281IW11+lQD+rSyPmhuYibh1cABkgMBMnlzHWn1FyJ6Vxv14WNDQKchoHMJpJ7yvFsga1jI2ALYJ+beV+6jrJa2/yA5VAaEKFTtxisBZzxM7U2HkHsZrAhWZtVK+GBx2bYuXRRF5THFT3ilIzPOCb02cG8Ve4abqO23J/augLfoftvDn+QK+PKyj13MdD3w/f89xjLr7MH4WXn6eEWU9TENJGiMgOoEXvNyjf/ZiAoShowtBYkhXxG0IFGai+O6x42LryDImGoXCbhdNQ7/zOVwbLBzWBnLEuVVNN7KuJXhl+FrsYJj+WDhxmTxjE1MKatCG9mNFAl3BxSEqDkNtBDng2SrFLTpt80+PFR9QB0jdBoml7e/k0kNCBH+5ObGFaKAxQaSF/QHarAF9K4YmR/7OpryU/y+etlYGHYc/tGvXBZRZ8NUxbfjEBt0N99VoGUx2WXKJo\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 54,\n \"candidatesTokenCount\": 15,\n \"totalTokenCount\": 162,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 54\n }\n ],\n \"thoughtsTokenCount\": 93\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"bpHKaMGVJc6YkdUP3fStCA\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 4.193896625s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 776 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence, Italy\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"It's 40 C in Florence, Italy. \\n\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 84,\n \"candidatesTokenCount\": 12,\n \"totalTokenCount\": 96,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 84\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"cJHKaOO8KfrY7M8Pg8-psQo\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 2.006433542s diff --git a/providertests/testdata/TestThinking/openai-gpt-5.yaml b/providertests/testdata/TestThinking/openai-gpt-5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c2ed0cd472823363484cc0d3eeb66b2605f96f7 --- /dev/null +++ b/providertests/testdata/TestThinking/openai-gpt-5.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 458 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CGjxZHxMSr7N8cRMctGQZk1GoGaeb\",\n \"object\": \"chat.completion\",\n \"created\": 1758105953,\n \"model\": \"gpt-5-2025-08-07\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_altLYQTktFAyPzFgFXA8no5D\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence, Italy\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 145,\n \"completion_tokens\": 153,\n \"total_tokens\": 298,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 128,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": null\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 3.138198958s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 710 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_altLYQTktFAyPzFgFXA8no5D\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence, Italy\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_altLYQTktFAyPzFgFXA8no5D\",\"role\":\"tool\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CGjxczIS9JrgSAuyFofVReqav7T93\",\n \"object\": \"chat.completion\",\n \"created\": 1758105956,\n \"model\": \"gpt-5-2025-08-07\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It’s currently about 40°C (104°F) in Florence, Italy.\\n\\nWant the forecast, humidity, or wind details as well?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 176,\n \"completion_tokens\": 229,\n \"total_tokens\": 405,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 192,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": null\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 6.448143542s diff --git a/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml b/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb888a85b327c81ab5c95a798ecf61916605fcee --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 564 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01VPcbUdEoEcxp65A65U43Cq\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":6,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"The user is asking for\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" weather information for Florence, Italy. I have\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" access to a weather function that takes a location parameter. The\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" location is clearly specified as \\\"Florence, Italy\\\".\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" I have all the required parameters to make this function call.\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EqUDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgzskB/aBlt0gS1BtUkaDPJSL8B++51vZ5DHlSIw+iMxyK5JptEx7nmgVEe8qK1bXt78PF7K83woxhkziWjrJcj/kndIznLO+qELQpNNKowCh+qjLD0jAIucketOZRE4uiSegDiqlzkenv9exVlEeoFvjiN1zVdgVKpWeylvA3BZYIviwFqgUVGAXjSsWcG+RNvB6SQmNk0PA5R9NmCvckI+Q/6VA9hp2hrjIskceJIsSg3mAtRQ36Rml4ie5ttHDD8f8XevtDu0NS9ymBRf5NPfZHtBPl9AQ32v15XgGS2oYzjn+vd/S/F/hkYd2e9XRnc3hdDa/AmoWbVjS6xU46gT0haidg2LkL79QeX8/2s5FfhIMOizH0XOzlwEHgvBRVBTERGEiSlpUXziHnaUVWbPtTDe42SvwuO/b2npygBoxXkHJyRS0Dz9FCZVs5XM2CoKRN0giFX8szV0rhgB\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"name\":\"weather\",\"input\":{}} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"loc\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"ation\\\": \\\"Flo\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"rence, It\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"aly\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":110} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.289802542s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 1721 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"EqUDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgzskB/aBlt0gS1BtUkaDPJSL8B++51vZ5DHlSIw+iMxyK5JptEx7nmgVEe8qK1bXt78PF7K83woxhkziWjrJcj/kndIznLO+qELQpNNKowCh+qjLD0jAIucketOZRE4uiSegDiqlzkenv9exVlEeoFvjiN1zVdgVKpWeylvA3BZYIviwFqgUVGAXjSsWcG+RNvB6SQmNk0PA5R9NmCvckI+Q/6VA9hp2hrjIskceJIsSg3mAtRQ36Rml4ie5ttHDD8f8XevtDu0NS9ymBRf5NPfZHtBPl9AQ32v15XgGS2oYzjn+vd/S/F/hkYd2e9XRnc3hdDa/AmoWbVjS6xU46gT0haidg2LkL79QeX8/2s5FfhIMOizH0XOzlwEHgvBRVBTERGEiSlpUXziHnaUVWbPtTDe42SvwuO/b2npygBoxXkHJyRS0Dz9FCZVs5XM2CoKRN0giFX8szV0rhgB\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have access to a weather function that takes a location parameter. The location is clearly specified as \\\"Florence, Italy\\\". I have all the required parameters to make this function call.\",\"type\":\"thinking\"},{\"id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"input\":{\"location\":\"Florence, Italy\"},\"name\":\"weather\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"content\":[{\"text\":\"40 C\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01GNkVaVsDPcTec33HABjn5e\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":548,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"The\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" weather in Florence, Italy is currently 40°C (104°F),\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" which is quite hot! This is typical\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" summer weather for Florence during the warmer months. Make\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" sure to stay hydrated and seek shade if you're planning to be out\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"doors.\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":548,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":56} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.676578167s diff --git a/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml b/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2292fb205a4cd24ce50c6f4a64d866176831e891 --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + form: + alt: + - sse + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"**Determining Weather Parameters**\\n\\nI've determined that the user's request, \\\"weather in Florence, Italy,\\\" perfectly aligns with the `weather.get_weather` function. It seems the `location` parameter is the key, and I'll confidently set it to \\\"Florence, Italy\\\".\\n\\n\\n\",\"thought\": true}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 54,\"totalTokenCount\": 121,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 54}],\"thoughtsTokenCount\": 67},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"zZHKaL2DMrPTvdIP4aHWyAs\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"weather\",\"args\": {\"location\": \"Florence, Italy\"}},\"thoughtSignature\": \"CiIB0e2Kb+4o560+ejjFBllU8TlXrgq4/b/tb77clkkTfKwtCmcB0e2Kb1IrE0Kd0PSfakpKlsp4JG0j3GDD1xu8po3uW1bBSFXZqOXycsUfeT7p/narEgT6HIrPBNkwA1qVwPeAj4D/jUbhIY2Oj2lS8ysNnQeqytNstgmTw32h9vbrhhvOcaH6IH1dCocBAdHtim8NodjIJ2/KTN7Ujms6FtiMUYBQHquMPWyGvJlCrwy2HjucHBKXc4u9og3+2Sd1BKC06BSuzCMiAIF5WqLBhnpTQSBqAoJQmpa856a1FUdmusVWLunaMM1HOuLlZCGNp0KYekg5i6swmIVyAzYOn/4HrEant/3HN8U0q7cPpyfFulYZCnAB0e2Kbxs+yhyWd5uvIZBXkefbF2xrggIwwWmG5FoCMXkhf9fTrrT24QGyHtBDZtjO/+VFvchpcv3XJC11Mfx+T9NpkWFKEQ78DFVVejHgyGEDGkE41mS+da0ejxpAXnHWYsh8Nbz5w9fkGJESlaqK\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 54,\"candidatesTokenCount\": 14,\"totalTokenCount\": 135,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 54}],\"thoughtsTokenCount\": 67},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"zZHKaL2DMrPTvdIP4aHWyAs\"}\r\n\r\n" + headers: + Content-Type: + - text/event-stream + status: 200 OK + code: 200 + duration: 3.416265375s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 776 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence, Italy\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + form: + alt: + - sse + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 84,\"candidatesTokenCount\": 1,\"totalTokenCount\": 85,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 84}]},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"0pHKaOvQBIXrvdIP44SHgAQ\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" weather in Florence, Italy is 40 degrees Celsius. \\n\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 84,\"candidatesTokenCount\": 13,\"totalTokenCount\": 97,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 84}]},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"0pHKaOvQBIXrvdIP44SHgAQ\"}\r\n\r\n" + headers: + Content-Type: + - text/event-stream + status: 200 OK + code: 200 + duration: 2.557538458s diff --git a/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml b/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..335246cbc6102a679a0f902f5823cf0440bb0675 --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 512 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"type\":\"function\",\"function\":{\"name\":\"weather\",\"arguments\":\"\"}}],\"refusal\":null},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"0xwspG0FC\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"V1qmL7MMFxIDEJ3\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"location\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"AM6r08CpIx\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"LNb7muLzepVLd\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Flor\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un0Jtn9FQ9Dh7e\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"ence\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"PnIt3YJQ7Odcnm\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\",\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"J\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\" Italy\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rNjAaLKtPTCR\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ogKZ9Um1xvfM6Sm\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":null,\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[],\"usage\":{\"prompt_tokens\":145,\"completion_tokens\":153,\"total_tokens\":298,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":128,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"tstpIe\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 2.807384709s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 764 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence, Italy\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"role\":\"tool\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"3dXcrYmrOy\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"It\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"PqgQgdemta\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"’s\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"kCefUAWvH9\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" currently\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"R6\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" about\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JzHfhJ\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UnOIco1wqlK\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"40\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"gxVfChj5p1\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1Yfx063pRL\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"atYPSethq\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Florence\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Hm3\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ZoMzHd1dUin\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Italy\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eBSjfa\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UuHUTv0keAz\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"mHVZv6\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[],\"usage\":{\"prompt_tokens\":176,\"completion_tokens\":277,\"total_tokens\":453,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":256,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"MMvL8v\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 4.36330425s diff --git a/providertests/thinking_test.go b/providertests/thinking_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e4a9363be4937f3e3ed80468a3527a6c54016e13 --- /dev/null +++ b/providertests/thinking_test.go @@ -0,0 +1,70 @@ +package providertests + +import ( + "testing" + + "github.com/charmbracelet/fantasy/ai" + "github.com/charmbracelet/fantasy/anthropic" + "github.com/charmbracelet/fantasy/google" + "github.com/stretchr/testify/require" +) + +func testThinkingSteps(t *testing.T, providerName string, steps []ai.StepResult) { + switch providerName { + case anthropic.Name: + testAnthropicThinking(t, steps) + case google.Name: + testGoogleThinking(t, steps) + } +} + +func testGoogleThinking(t *testing.T, steps []ai.StepResult) { + reasoningContentCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + } + } + } + } + require.Greater(t, reasoningContentCount, 0) +} + +func testAnthropicThinking(t *testing.T, steps []ai.StepResult) { + reasoningContentCount := 0 + signaturesCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + reasoningContent, ok := ai.AsContentType[ai.ReasoningPart](content) + if !ok { + continue + } + if len(reasoningContent.ProviderOptions) == 0 { + continue + } + + anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name] + if !ok { + continue + } + if reasoningContent.Text != "" { + if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok { + require.NotEmpty(t, typed.Signature) + signaturesCount += 1 + } + } + } + } + } + } + require.Greater(t, reasoningContentCount, 0) + require.Greater(t, signaturesCount, 0) + require.Equal(t, reasoningContentCount, signaturesCount) +}