From c8711ead48d99318e619b2e0f82cdc8a75fed9e7 Mon Sep 17 00:00:00 2001 From: Kujtim Hoxha Date: Wed, 24 Sep 2025 12:15:29 +0200 Subject: [PATCH] test: add thinking to anthropic --- providertests/anthropic_test.go | 48 ++++++++++++++ .../TestAnthropicCommon/claude-sonnet-4.yaml | 63 +++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 providertests/testdata/TestAnthropicCommon/claude-sonnet-4.yaml diff --git a/providertests/anthropic_test.go b/providertests/anthropic_test.go index d238076831612c020f9283e8f388b2e27a93dd43..3345833c045f1b62c016b6132e03b3eadb3ddaf2 100644 --- a/providertests/anthropic_test.go +++ b/providertests/anthropic_test.go @@ -7,6 +7,7 @@ import ( "github.com/charmbracelet/fantasy/ai" "github.com/charmbracelet/fantasy/anthropic" + "github.com/stretchr/testify/require" "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" ) @@ -14,6 +15,53 @@ func TestAnthropicCommon(t *testing.T) { testCommon(t, []builderPair{ {"claude-sonnet-4", builderAnthropicClaudeSonnet4, nil}, }) + + opts := ai.ProviderOptions{ + anthropic.Name: &anthropic.ProviderOptions{ + Thinking: &anthropic.ThinkingProviderOption{ + BudgetTokens: 4000, + }, + }, + } + testThinking(t, []builderPair{ + {"claude-sonnet-4", builderAnthropicClaudeSonnet4, opts}, + }, testGoogleThinking) +} + +func testAnthropicThinking(t *testing.T, result *ai.AgentResult) { + reasoningContentCount := 0 + signaturesCount := 0 + // Test if we got the signature + for _, step := range result.Steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + reasoningContent, ok := ai.AsContentType[ai.ReasoningPart](content) + if !ok { + continue + } + if len(reasoningContent.ProviderOptions) == 0 { + continue + } + + anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name] + if !ok { + continue + } + if reasoningContent.Text != "" { + if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok { + require.NotEmpty(t, typed.Signature) + signaturesCount += 1 + } + } + } + } + } + } + require.Greater(t, reasoningContentCount, 0) + require.Greater(t, signaturesCount, 0) + require.Equal(t, reasoningContentCount, signaturesCount) } func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { diff --git a/providertests/testdata/TestAnthropicCommon/claude-sonnet-4.yaml b/providertests/testdata/TestAnthropicCommon/claude-sonnet-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..420822b25097121c733989c9b697c26f5908761a --- /dev/null +++ b/providertests/testdata/TestAnthropicCommon/claude-sonnet-4.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 548 + host: "" + body: '{"max_tokens":8096,"messages":[{"content":[{"text":"What''s the weather in Florence, Italy?","type":"text"}],"role":"user"}],"model":"claude-sonnet-4-20250514","system":[{"text":"You are a helpful assistant","type":"text"}],"thinking":{"budget_tokens":4000,"type":"enabled"},"tool_choice":{"disable_parallel_tool_use":false,"type":"auto"},"tools":[{"input_schema":{"properties":{"location":{"description":"the city","type":"string"}},"required":["location"],"type":"object"},"name":"weather","description":"Get weather information for a location"}]}' + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: '{"id":"msg_01UYqHuA4Tt7HtZBmSzncLKW","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[{"type":"thinking","thinking":"The user is asking for weather information for Florence, Italy. I have access to a weather function that takes a location parameter. The location is clearly specified as \"Florence, Italy\". I have all the required parameters to make this function call.","signature":"EqQDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgy36dF4ihE6WIc/asUaDPRSkmjhgGgrEt7hKiIwDUKY8xyfDCTKHRPA6Kw77cnWCWDlnSX3pXfowzysv7VILx33hTNRBKROj0AiDhbEKosCj+Lqk0PWtRPRv1GxAWsbXqXBPDVL9DrFTKJfqRL2xkISMYbWCGEysel73WMkBaUPGzEU39FXQjCSu7BxhIorpBozrdNyr430+yKyPAWBn9XiU0bq6KAYXwEaTlRWVbhnIPZal4TeCITOQ10DcGLhAObBbo+tMTG9F1G1ALWjxfFuVq3cPqXaquTqhrIBPxGPHX+GSA2665mIrGqYNv3J2e6dpCfr5ZBmS8DWdRIshsMboxoLx7yAIuErjiAIHnFb3GEBZJ2igLbCnBkEULFQ2j87fZhg3yDcXqbZ3ILr1iK32hEoREuX4vsyfUfKi8KfPjf6vYV2pdhWGiu8ASD5iViW1Xpl0fx0vuk9GAE="},{"type":"tool_use","id":"toolu_01EinYCf2KLtnmBLsLi8UHUd","name":"weather","input":{"location":"Florence, Italy"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":423,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":110,"service_tier":"standard"}}' + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 4.5487575s + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 1705 + host: "" + body: '{"max_tokens":8096,"messages":[{"content":[{"text":"What''s the weather in Florence, Italy?","type":"text"}],"role":"user"},{"content":[{"signature":"EqQDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgy36dF4ihE6WIc/asUaDPRSkmjhgGgrEt7hKiIwDUKY8xyfDCTKHRPA6Kw77cnWCWDlnSX3pXfowzysv7VILx33hTNRBKROj0AiDhbEKosCj+Lqk0PWtRPRv1GxAWsbXqXBPDVL9DrFTKJfqRL2xkISMYbWCGEysel73WMkBaUPGzEU39FXQjCSu7BxhIorpBozrdNyr430+yKyPAWBn9XiU0bq6KAYXwEaTlRWVbhnIPZal4TeCITOQ10DcGLhAObBbo+tMTG9F1G1ALWjxfFuVq3cPqXaquTqhrIBPxGPHX+GSA2665mIrGqYNv3J2e6dpCfr5ZBmS8DWdRIshsMboxoLx7yAIuErjiAIHnFb3GEBZJ2igLbCnBkEULFQ2j87fZhg3yDcXqbZ3ILr1iK32hEoREuX4vsyfUfKi8KfPjf6vYV2pdhWGiu8ASD5iViW1Xpl0fx0vuk9GAE=","thinking":"The user is asking for weather information for Florence, Italy. I have access to a weather function that takes a location parameter. The location is clearly specified as \"Florence, Italy\". I have all the required parameters to make this function call.","type":"thinking"},{"id":"toolu_01EinYCf2KLtnmBLsLi8UHUd","input":{"location":"Florence, Italy"},"name":"weather","type":"tool_use"}],"role":"assistant"},{"content":[{"tool_use_id":"toolu_01EinYCf2KLtnmBLsLi8UHUd","content":[{"text":"40 C","type":"text"}],"type":"tool_result"}],"role":"user"}],"model":"claude-sonnet-4-20250514","system":[{"text":"You are a helpful assistant","type":"text"}],"thinking":{"budget_tokens":4000,"type":"enabled"},"tool_choice":{"disable_parallel_tool_use":false,"type":"auto"},"tools":[{"input_schema":{"properties":{"location":{"description":"the city","type":"string"}},"required":["location"],"type":"object"},"name":"weather","description":"Get weather information for a location"}]}' + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: '{"id":"msg_01HVqNJfVm6BJirBoqgAdpkm","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[{"type":"text","text":"The current weather in Florence, Italy is 40°C (104°F). That''s quite hot! Make sure to stay hydrated and seek shade if you''re planning to be outdoors."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":548,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":43,"service_tier":"standard"}}' + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.249631625s