1package providertests
2
3import (
4 "net/http"
5 "os"
6 "testing"
7
8 "charm.land/fantasy"
9 "charm.land/fantasy/providers/openrouter"
10 "github.com/stretchr/testify/require"
11 "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
12)
13
14var openrouterTestModels = []testModel{
15 {"kimi-k2", "moonshotai/kimi-k2-0905", false},
16 {"grok-code-fast-1", "x-ai/grok-code-fast-1", false},
17 {"claude-sonnet-4", "anthropic/claude-sonnet-4", true},
18 {"gemini-2.5-flash", "google/gemini-2.5-flash", false},
19 {"deepseek-chat-v3.1-free", "deepseek/deepseek-chat-v3.1:free", false},
20 {"qwen3-235b-a22b-2507", "qwen/qwen3-235b-a22b-2507", false},
21 {"gpt-5", "openai/gpt-5", true},
22 {"glm-4.5", "z-ai/glm-4.5", false},
23 {"glm-4.6", "z-ai/glm-4.6", true},
24}
25
26func TestOpenRouterCommon(t *testing.T) {
27 var pairs []builderPair
28 for _, m := range openrouterTestModels {
29 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), nil})
30 }
31 testCommon(t, pairs)
32}
33
34func TestOpenRouterThinking(t *testing.T) {
35 opts := fantasy.ProviderOptions{
36 openrouter.Name: &openrouter.ProviderOptions{
37 Reasoning: &openrouter.ReasoningOptions{
38 Effort: openrouter.ReasoningEffortOption(openrouter.ReasoningEffortMedium),
39 },
40 },
41 }
42
43 var pairs []builderPair
44 for _, m := range openrouterTestModels {
45 if !m.reasoning {
46 continue
47 }
48 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), opts})
49 }
50 testThinking(t, pairs, testOpenrouterThinking)
51
52 // test anthropic signature
53 testThinking(t, []builderPair{
54 {"claude-sonnet-4-sig", openrouterBuilder("anthropic/claude-sonnet-4"), opts},
55 }, testOpenrouterThinkingWithSignature)
56}
57
58func testOpenrouterThinkingWithSignature(t *testing.T, result *fantasy.AgentResult) {
59 reasoningContentCount := 0
60 signaturesCount := 0
61 // Test if we got the signature
62 for _, step := range result.Steps {
63 for _, msg := range step.Messages {
64 for _, content := range msg.Content {
65 if content.GetType() == fantasy.ContentTypeReasoning {
66 reasoningContentCount += 1
67 reasoningContent, ok := fantasy.AsContentType[fantasy.ReasoningPart](content)
68 if !ok {
69 continue
70 }
71 if len(reasoningContent.ProviderOptions) == 0 {
72 continue
73 }
74
75 anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[openrouter.Name]
76 if !ok {
77 continue
78 }
79 if reasoningContent.Text != "" {
80 if typed, ok := anthropicReasoningMetadata.(*openrouter.ReasoningMetadata); ok {
81 require.NotEmpty(t, typed.Signature)
82 signaturesCount += 1
83 }
84 }
85 }
86 }
87 }
88 }
89 require.Greater(t, reasoningContentCount, 0)
90 require.Greater(t, signaturesCount, 0)
91 require.Equal(t, reasoningContentCount, signaturesCount)
92 // we also add the anthropic metadata so test that
93 testAnthropicThinking(t, result)
94}
95
96func testOpenrouterThinking(t *testing.T, result *fantasy.AgentResult) {
97 reasoningContentCount := 0
98 for _, step := range result.Steps {
99 for _, msg := range step.Messages {
100 for _, content := range msg.Content {
101 if content.GetType() == fantasy.ContentTypeReasoning {
102 reasoningContentCount += 1
103 }
104 }
105 }
106 }
107 require.Greater(t, reasoningContentCount, 0)
108}
109
110func openrouterBuilder(model string) builderFunc {
111 return func(r *recorder.Recorder) (fantasy.LanguageModel, error) {
112 provider := openrouter.New(
113 openrouter.WithAPIKey(os.Getenv("FANTASY_OPENROUTER_API_KEY")),
114 openrouter.WithHTTPClient(&http.Client{Transport: r}),
115 )
116 return provider.LanguageModel(model)
117 }
118}