1package providertests
2
3import (
4 "net/http"
5 "os"
6 "testing"
7
8 "charm.land/fantasy"
9 "charm.land/fantasy/providers/anthropic"
10 "charm.land/fantasy/providers/openrouter"
11 "charm.land/x/vcr"
12 "github.com/stretchr/testify/require"
13)
14
15var openrouterTestModels = []testModel{
16 {"kimi-k2", "moonshotai/kimi-k2-0905", false},
17 {"grok-code-fast-1", "x-ai/grok-code-fast-1", true},
18 {"claude-sonnet-4", "anthropic/claude-sonnet-4", true},
19 {"gemini-2.5-flash", "google/gemini-2.5-flash", false},
20 {"deepseek-chat-v3.1-free", "deepseek/deepseek-chat-v3.1:free", false},
21 {"qwen3-235b-a22b-2507", "qwen/qwen3-235b-a22b-2507", false},
22 {"gpt-5", "openai/gpt-5", true},
23 {"gemini-3-pro-preview", "google/gemini-3-pro-preview", true},
24 {"glm-4.5", "z-ai/glm-4.5", false},
25 {"glm-4.6", "z-ai/glm-4.6", true},
26}
27
28func TestOpenRouterCommon(t *testing.T) {
29 var pairs []builderPair
30 for _, m := range openrouterTestModels {
31 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), nil, nil})
32 }
33 testCommon(t, pairs)
34}
35
36func TestOpenRouterCommonWithAnthropicCache(t *testing.T) {
37 testCommon(t, []builderPair{
38 {"claude-sonnet-4", openrouterBuilder("anthropic/claude-sonnet-4"), nil, addAnthropicCaching},
39 })
40}
41
42func TestOpenRouterThinking(t *testing.T) {
43 opts := fantasy.ProviderOptions{
44 openrouter.Name: &openrouter.ProviderOptions{
45 Reasoning: &openrouter.ReasoningOptions{
46 Effort: openrouter.ReasoningEffortOption(openrouter.ReasoningEffortMedium),
47 },
48 },
49 }
50
51 var pairs []builderPair
52 for _, m := range openrouterTestModels {
53 if !m.reasoning {
54 continue
55 }
56 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), opts, nil})
57 }
58 testThinking(t, pairs, testOpenrouterThinking)
59
60 // test anthropic signature
61 testThinking(t, []builderPair{
62 {"claude-sonnet-4-sig", openrouterBuilder("anthropic/claude-sonnet-4"), opts, nil},
63 }, testOpenrouterThinkingWithSignature)
64}
65
66func testOpenrouterThinkingWithSignature(t *testing.T, result *fantasy.AgentResult) {
67 reasoningContentCount := 0
68 signaturesCount := 0
69 // Test if we got the signature
70 for _, step := range result.Steps {
71 for _, msg := range step.Messages {
72 for _, content := range msg.Content {
73 if content.GetType() == fantasy.ContentTypeReasoning {
74 reasoningContentCount += 1
75 reasoningContent, ok := fantasy.AsContentType[fantasy.ReasoningPart](content)
76 if !ok {
77 continue
78 }
79 if len(reasoningContent.ProviderOptions) == 0 {
80 continue
81 }
82
83 anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name]
84 if !ok {
85 continue
86 }
87 if reasoningContent.Text != "" {
88 if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok {
89 require.NotEmpty(t, typed.Signature)
90 signaturesCount += 1
91 }
92 }
93 }
94 }
95 }
96 }
97 require.Greater(t, reasoningContentCount, 0)
98 require.Greater(t, signaturesCount, 0)
99 require.Equal(t, reasoningContentCount, signaturesCount)
100 // we also add the anthropic metadata so test that
101 testAnthropicThinking(t, result)
102}
103
104func testOpenrouterThinking(t *testing.T, result *fantasy.AgentResult) {
105 reasoningContentCount := 0
106 for _, step := range result.Steps {
107 for _, msg := range step.Messages {
108 for _, content := range msg.Content {
109 if content.GetType() == fantasy.ContentTypeReasoning {
110 reasoningContentCount += 1
111 }
112 }
113 }
114 }
115 require.Greater(t, reasoningContentCount, 0)
116}
117
118func openrouterBuilder(model string) builderFunc {
119 return func(t *testing.T, r *vcr.Recorder) (fantasy.LanguageModel, error) {
120 provider, err := openrouter.New(
121 openrouter.WithAPIKey(os.Getenv("FANTASY_OPENROUTER_API_KEY")),
122 openrouter.WithHTTPClient(&http.Client{Transport: r}),
123 )
124 if err != nil {
125 return nil, err
126 }
127 return provider.LanguageModel(t.Context(), model)
128 }
129}