1package providertests
2
3import (
4 "net/http"
5 "os"
6 "testing"
7
8 "charm.land/fantasy"
9 "charm.land/fantasy/openrouter"
10 "github.com/stretchr/testify/require"
11 "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
12)
13
14var openrouterTestModels = []testModel{
15 {"kimi-k2", "moonshotai/kimi-k2-0905", false},
16 {"grok-code-fast-1", "x-ai/grok-code-fast-1", false},
17 {"claude-sonnet-4", "anthropic/claude-sonnet-4", true},
18 {"gemini-2.5-flash", "google/gemini-2.5-flash", false},
19 {"deepseek-chat-v3.1-free", "deepseek/deepseek-chat-v3.1:free", false},
20 {"qwen3-235b-a22b-2507", "qwen/qwen3-235b-a22b-2507", false},
21 {"gpt-5", "openai/gpt-5", true},
22 {"glm-4.5", "z-ai/glm-4.5", false},
23}
24
25func TestOpenRouterCommon(t *testing.T) {
26 var pairs []builderPair
27 for _, m := range openrouterTestModels {
28 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), nil})
29 }
30 testCommon(t, pairs)
31}
32
33func TestOpenRouterThinking(t *testing.T) {
34 opts := fantasy.ProviderOptions{
35 openrouter.Name: &openrouter.ProviderOptions{
36 Reasoning: &openrouter.ReasoningOptions{
37 Effort: openrouter.ReasoningEffortOption(openrouter.ReasoningEffortMedium),
38 },
39 },
40 }
41
42 var pairs []builderPair
43 for _, m := range openrouterTestModels {
44 if !m.reasoning {
45 continue
46 }
47 pairs = append(pairs, builderPair{m.name, openrouterBuilder(m.model), opts})
48 }
49 testThinking(t, pairs, testOpenrouterThinking)
50
51 // test anthropic signature
52 testThinking(t, []builderPair{
53 {"claude-sonnet-4-sig", openrouterBuilder("anthropic/claude-sonnet-4"), opts},
54 }, testOpenrouterThinkingWithSignature)
55}
56
57func testOpenrouterThinkingWithSignature(t *testing.T, result *fantasy.AgentResult) {
58 reasoningContentCount := 0
59 signaturesCount := 0
60 // Test if we got the signature
61 for _, step := range result.Steps {
62 for _, msg := range step.Messages {
63 for _, content := range msg.Content {
64 if content.GetType() == fantasy.ContentTypeReasoning {
65 reasoningContentCount += 1
66 reasoningContent, ok := fantasy.AsContentType[fantasy.ReasoningPart](content)
67 if !ok {
68 continue
69 }
70 if len(reasoningContent.ProviderOptions) == 0 {
71 continue
72 }
73
74 anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[openrouter.Name]
75 if !ok {
76 continue
77 }
78 if reasoningContent.Text != "" {
79 if typed, ok := anthropicReasoningMetadata.(*openrouter.ReasoningMetadata); ok {
80 require.NotEmpty(t, typed.Signature)
81 signaturesCount += 1
82 }
83 }
84 }
85 }
86 }
87 }
88 require.Greater(t, reasoningContentCount, 0)
89 require.Greater(t, signaturesCount, 0)
90 require.Equal(t, reasoningContentCount, signaturesCount)
91 // we also add the anthropic metadata so test that
92 testAnthropicThinking(t, result)
93}
94
95func testOpenrouterThinking(t *testing.T, result *fantasy.AgentResult) {
96 reasoningContentCount := 0
97 for _, step := range result.Steps {
98 for _, msg := range step.Messages {
99 for _, content := range msg.Content {
100 if content.GetType() == fantasy.ContentTypeReasoning {
101 reasoningContentCount += 1
102 }
103 }
104 }
105 }
106 require.Greater(t, reasoningContentCount, 0)
107}
108
109func openrouterBuilder(model string) builderFunc {
110 return func(r *recorder.Recorder) (fantasy.LanguageModel, error) {
111 provider := openrouter.New(
112 openrouter.WithAPIKey(os.Getenv("FANTASY_OPENROUTER_API_KEY")),
113 openrouter.WithHTTPClient(&http.Client{Transport: r}),
114 )
115 return provider.LanguageModel(model)
116 }
117}