1package providertests
2
3import (
4 "net/http"
5 "os"
6 "testing"
7
8 "charm.land/fantasy"
9 "charm.land/fantasy/providers/anthropic"
10 "charm.land/fantasy/providers/openrouter"
11 "github.com/stretchr/testify/require"
12 "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
13)
14
15var openrouterTestModels = []testModel{
16 {"kimi-k2", "moonshotai/kimi-k2-0905", false},
17 {"grok-code-fast-1", "x-ai/grok-code-fast-1", true},
18 {"claude-sonnet-4", "anthropic/claude-sonnet-4", true},
19 {"gemini-2.5-flash", "google/gemini-2.5-flash", false},
20 {"deepseek-chat-v3.1-free", "deepseek/deepseek-chat-v3.1:free", false},
21 {"qwen3-235b-a22b-2507", "qwen/qwen3-235b-a22b-2507", false},
22 {"gpt-5", "openai/gpt-5", true},
23 {"glm-4.5", "z-ai/glm-4.5", false},
24 {"glm-4.6", "z-ai/glm-4.6", true},
25}
26
27func TestOpenRouterCommon(t *testing.T) {
28 var pairs []builderPair
29 for _, m := range openrouterTestModels {
30 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), nil, nil})
31 }
32 testCommon(t, pairs)
33}
34
35func TestOpenRouterCommonWithAnthropicCache(t *testing.T) {
36 testCommon(t, []builderPair{
37 {"claude-sonnet-4", openrouterBuilder("anthropic/claude-sonnet-4"), nil, addAnthropicCaching},
38 })
39}
40
41func TestOpenRouterThinking(t *testing.T) {
42 opts := fantasy.ProviderOptions{
43 openrouter.Name: &openrouter.ProviderOptions{
44 Reasoning: &openrouter.ReasoningOptions{
45 Effort: openrouter.ReasoningEffortOption(openrouter.ReasoningEffortMedium),
46 },
47 },
48 }
49
50 var pairs []builderPair
51 for _, m := range openrouterTestModels {
52 if !m.reasoning {
53 continue
54 }
55 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), opts, nil})
56 }
57 testThinking(t, pairs, testOpenrouterThinking)
58
59 // test anthropic signature
60 testThinking(t, []builderPair{
61 {"claude-sonnet-4-sig", openrouterBuilder("anthropic/claude-sonnet-4"), opts, nil},
62 }, testOpenrouterThinkingWithSignature)
63}
64
65func testOpenrouterThinkingWithSignature(t *testing.T, result *fantasy.AgentResult) {
66 reasoningContentCount := 0
67 signaturesCount := 0
68 // Test if we got the signature
69 for _, step := range result.Steps {
70 for _, msg := range step.Messages {
71 for _, content := range msg.Content {
72 if content.GetType() == fantasy.ContentTypeReasoning {
73 reasoningContentCount += 1
74 reasoningContent, ok := fantasy.AsContentType[fantasy.ReasoningPart](content)
75 if !ok {
76 continue
77 }
78 if len(reasoningContent.ProviderOptions) == 0 {
79 continue
80 }
81
82 anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name]
83 if !ok {
84 continue
85 }
86 if reasoningContent.Text != "" {
87 if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok {
88 require.NotEmpty(t, typed.Signature)
89 signaturesCount += 1
90 }
91 }
92 }
93 }
94 }
95 }
96 require.Greater(t, reasoningContentCount, 0)
97 require.Greater(t, signaturesCount, 0)
98 require.Equal(t, reasoningContentCount, signaturesCount)
99 // we also add the anthropic metadata so test that
100 testAnthropicThinking(t, result)
101}
102
103func testOpenrouterThinking(t *testing.T, result *fantasy.AgentResult) {
104 reasoningContentCount := 0
105 for _, step := range result.Steps {
106 for _, msg := range step.Messages {
107 for _, content := range msg.Content {
108 if content.GetType() == fantasy.ContentTypeReasoning {
109 reasoningContentCount += 1
110 }
111 }
112 }
113 }
114 require.Greater(t, reasoningContentCount, 0)
115}
116
117func openrouterBuilder(model string) builderFunc {
118 return func(r *recorder.Recorder) (fantasy.LanguageModel, error) {
119 provider, err := openrouter.New(
120 openrouter.WithAPIKey(os.Getenv("FANTASY_OPENROUTER_API_KEY")),
121 openrouter.WithHTTPClient(&http.Client{Transport: r}),
122 )
123 if err != nil {
124 return nil, err
125 }
126 return provider.LanguageModel(model)
127 }
128}