1package providertests
2
3import (
4 "context"
5 "net/http"
6 "os"
7 "testing"
8
9 "charm.land/fantasy"
10 "charm.land/fantasy/providers/anthropic"
11 "github.com/stretchr/testify/require"
12 "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
13)
14
15var anthropicTestModels = []testModel{
16 {"claude-sonnet-4", "claude-sonnet-4-20250514", true},
17}
18
19func TestAnthropicCommon(t *testing.T) {
20 var pairs []builderPair
21 for _, m := range anthropicTestModels {
22 pairs = append(pairs, builderPair{m.name, anthropicBuilder(m.model), nil, nil})
23 }
24 testCommon(t, pairs)
25}
26
27func addAnthropicCaching(ctx context.Context, options fantasy.PrepareStepFunctionOptions) (context.Context, fantasy.PrepareStepResult, error) {
28 prepared := fantasy.PrepareStepResult{}
29 prepared.Messages = options.Messages
30
31 for i := range prepared.Messages {
32 prepared.Messages[i].ProviderOptions = nil
33 }
34 providerOption := fantasy.ProviderOptions{
35 anthropic.Name: &anthropic.ProviderCacheControlOptions{
36 CacheControl: anthropic.CacheControl{Type: "ephemeral"},
37 },
38 }
39
40 lastSystemRoleInx := 0
41 systemMessageUpdated := false
42 for i, msg := range prepared.Messages {
43 // only add cache control to the last message
44 if msg.Role == fantasy.MessageRoleSystem {
45 lastSystemRoleInx = i
46 } else if !systemMessageUpdated {
47 prepared.Messages[lastSystemRoleInx].ProviderOptions = providerOption
48 systemMessageUpdated = true
49 }
50 // than add cache control to the last 2 messages
51 if i > len(prepared.Messages)-3 {
52 prepared.Messages[i].ProviderOptions = providerOption
53 }
54 }
55 return ctx, prepared, nil
56}
57
58func TestAnthropicCommonWithCacheControl(t *testing.T) {
59 var pairs []builderPair
60 for _, m := range anthropicTestModels {
61 pairs = append(pairs, builderPair{m.name, anthropicBuilder(m.model), nil, addAnthropicCaching})
62 }
63 testCommon(t, pairs)
64}
65
66func TestAnthropicThinking(t *testing.T) {
67 opts := fantasy.ProviderOptions{
68 anthropic.Name: &anthropic.ProviderOptions{
69 Thinking: &anthropic.ThinkingProviderOption{
70 BudgetTokens: 4000,
71 },
72 },
73 }
74 var pairs []builderPair
75 for _, m := range anthropicTestModels {
76 if !m.reasoning {
77 continue
78 }
79 pairs = append(pairs, builderPair{m.name, anthropicBuilder(m.model), opts, nil})
80 }
81 testThinking(t, pairs, testAnthropicThinking)
82}
83
84func TestAnthropicThinkingWithCacheControl(t *testing.T) {
85 opts := fantasy.ProviderOptions{
86 anthropic.Name: &anthropic.ProviderOptions{
87 Thinking: &anthropic.ThinkingProviderOption{
88 BudgetTokens: 4000,
89 },
90 },
91 }
92 var pairs []builderPair
93 for _, m := range anthropicTestModels {
94 if !m.reasoning {
95 continue
96 }
97 pairs = append(pairs, builderPair{m.name, anthropicBuilder(m.model), opts, addAnthropicCaching})
98 }
99 testThinking(t, pairs, testAnthropicThinking)
100}
101
102func testAnthropicThinking(t *testing.T, result *fantasy.AgentResult) {
103 reasoningContentCount := 0
104 signaturesCount := 0
105 // Test if we got the signature
106 for _, step := range result.Steps {
107 for _, msg := range step.Messages {
108 for _, content := range msg.Content {
109 if content.GetType() == fantasy.ContentTypeReasoning {
110 reasoningContentCount += 1
111 reasoningContent, ok := fantasy.AsContentType[fantasy.ReasoningPart](content)
112 if !ok {
113 continue
114 }
115 if len(reasoningContent.ProviderOptions) == 0 {
116 continue
117 }
118
119 anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name]
120 if !ok {
121 continue
122 }
123 if reasoningContent.Text != "" {
124 if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok {
125 require.NotEmpty(t, typed.Signature)
126 signaturesCount += 1
127 }
128 }
129 }
130 }
131 }
132 }
133 require.Greater(t, reasoningContentCount, 0)
134 require.Greater(t, signaturesCount, 0)
135 require.Equal(t, reasoningContentCount, signaturesCount)
136}
137
138func anthropicBuilder(model string) builderFunc {
139 return func(r *recorder.Recorder) (fantasy.LanguageModel, error) {
140 provider, err := anthropic.New(
141 anthropic.WithAPIKey(os.Getenv("FANTASY_ANTHROPIC_API_KEY")),
142 anthropic.WithHTTPClient(&http.Client{Transport: r}),
143 )
144 if err != nil {
145 return nil, err
146 }
147 return provider.LanguageModel(model)
148 }
149}