.gitattributes 🔗
@@ -0,0 +1 @@
+providertests/testdata/**/*.yaml linguist-generated=true
test: setup go-vcr to start recording real providers interactions
Andrey Nering created
Change summary
.gitattributes | 1
go.mod | 3
go.sum | 6
providertests/.env.sample | 2
providertests/builders_test.go | 47 ++
providertests/provider_test.go | 39 ++
providertests/recorder_test.go | 74 ++++
providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml | 33 +
providertests/testdata/TestSimple/openai-gpt-4o.yaml | 33 +
9 files changed, 238 insertions(+)
Detailed changes
go.mod 🔗
@@ -7,12 +7,15 @@ require (
github.com/charmbracelet/x/json v0.2.0
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/google/uuid v1.6.0
+ github.com/joho/godotenv v1.5.1
github.com/openai/openai-go/v2 v2.3.0
github.com/stretchr/testify v1.11.1
+ gopkg.in/dnaeon/go-vcr.v4 v4.0.5
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/goccy/go-yaml v1.18.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
go.sum 🔗
@@ -6,8 +6,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
+github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/openai/openai-go/v2 v2.3.0 h1:y9U+V1tlHjvvb/5XIswuySqnG5EnKBFAbMxgBvTHXvg=
github.com/openai/openai-go/v2 v2.3.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -26,5 +30,7 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.5 h1:I0hpTIvD5rII+8LgYGrHMA2d4SQPoL6u7ZvJakWKsiA=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.5/go.mod h1:dRos81TkW9C1WJt6tTaE+uV2Lo8qJT3AG2b35+CB/nQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
providertests/.env.sample 🔗
@@ -0,0 +1,2 @@
+ANTHROPIC_API_KEY=
+OPENAI_API_KEY=
providertests/builders_test.go 🔗
@@ -0,0 +1,47 @@
+package providertests
+
+import (
+ "net/http"
+ "os"
+
+ "github.com/charmbracelet/ai/ai"
+ "github.com/charmbracelet/ai/anthropic"
+ "github.com/charmbracelet/ai/openai"
+ "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
+)
+
+type builderFunc func(r *recorder.Recorder) (ai.LanguageModel, error)
+
+type builderPair struct {
+ name string
+ builder builderFunc
+}
+
+var languageModelBuilders = []builderPair{
+ {"openai-gpt-4o", builderOpenaiGpt4o},
+ {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4},
+}
+
+func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) {
+ provider := openai.New(
+ openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")),
+ openai.WithHTTPClient(&http.Client{Transport: r}),
+ )
+ model, err := provider.LanguageModel("gpt-4o")
+ if err != nil {
+ return nil, err
+ }
+ return model, nil
+}
+
+func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) {
+ provider := anthropic.New(
+ anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")),
+ anthropic.WithHTTPClient(&http.Client{Transport: r}),
+ )
+ model, err := provider.LanguageModel("claude-sonnet-4-20250514")
+ if err != nil {
+ return nil, err
+ }
+ return model, nil
+}
providertests/provider_test.go 🔗
@@ -0,0 +1,39 @@
+package providertests
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/charmbracelet/ai/ai"
+ _ "github.com/joho/godotenv/autoload"
+)
+
+func TestSimple(t *testing.T) {
+ for _, pair := range languageModelBuilders {
+ t.Run(pair.name, func(t *testing.T) {
+ r := newRecorder(t)
+
+ languageModel, err := pair.builder(r)
+ if err != nil {
+ t.Fatalf("failed to build language model: %v", err)
+ }
+
+ agent := ai.NewAgent(
+ languageModel,
+ ai.WithSystemPrompt("You are a helpful assistant"),
+ )
+ result, err := agent.Generate(t.Context(), ai.AgentCall{
+ Prompt: "Say hi in Portuguese",
+ })
+ if err != nil {
+ t.Fatalf("failed to generate: %v", err)
+ }
+
+ want := "Olá"
+ got := result.Response.Content.Text()
+ if !strings.Contains(got, want) {
+ t.Fatalf("unexpected response: got %q, want %q", got, want)
+ }
+ })
+ }
+}
providertests/recorder_test.go 🔗
@@ -0,0 +1,74 @@
+package providertests
+
+import (
+ "bytes"
+ "io"
+ "net/http"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "gopkg.in/dnaeon/go-vcr.v4/pkg/cassette"
+ "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
+)
+
+func newRecorder(t *testing.T) *recorder.Recorder {
+ cassetteName := filepath.Join("testdata", t.Name())
+
+ r, err := recorder.New(
+ cassetteName,
+ recorder.WithMode(recorder.ModeRecordOnce),
+ recorder.WithMatcher(customMatcher(t)),
+ recorder.WithHook(hookRemoveHeaders, recorder.AfterCaptureHook),
+ )
+ if err != nil {
+ t.Fatalf("recorder: failed to create recorder: %v", err)
+ }
+
+ t.Cleanup(func() {
+ if err := r.Stop(); err != nil {
+ t.Errorf("recorder: failed to stop recorder: %v", err)
+ }
+ })
+
+ return r
+}
+
+func customMatcher(t *testing.T) recorder.MatcherFunc {
+ return func(r *http.Request, i cassette.Request) bool {
+ if r.Body == nil || r.Body == http.NoBody {
+ return cassette.DefaultMatcher(r, i)
+ }
+
+ var reqBody []byte
+ var err error
+ reqBody, err = io.ReadAll(r.Body)
+ if err != nil {
+ t.Fatalf("recorder: failed to read request body")
+ }
+ r.Body.Close()
+ r.Body = io.NopCloser(bytes.NewBuffer(reqBody))
+
+ return r.Method == i.Method && r.URL.String() == i.URL && string(reqBody) == i.Body
+ }
+}
+
+var headersToKeep = map[string]struct{}{
+ "accept": {},
+ "content-type": {},
+ "user-agent": {},
+}
+
+func hookRemoveHeaders(i *cassette.Interaction) error {
+ for k := range i.Request.Headers {
+ if _, ok := headersToKeep[strings.ToLower(k)]; !ok {
+ delete(i.Request.Headers, k)
+ }
+ }
+ for k := range i.Response.Headers {
+ if _, ok := headersToKeep[strings.ToLower(k)]; !ok {
+ delete(i.Response.Headers, k)
+ }
+ }
+ return nil
+}
providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml 🔗
@@ -0,0 +1,33 @@
+---
+version: 2
+interactions:
+- id: 0
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 205
+ host: ""
+ body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"Say hi in Portuguese\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}]}"
+ headers:
+ Accept:
+ - application/json
+ Content-Type:
+ - application/json
+ User-Agent:
+ - Anthropic/Go 1.10.0
+ url: https://api.anthropic.com/v1/messages
+ method: POST
+ response:
+ proto: HTTP/2.0
+ proto_major: 2
+ proto_minor: 0
+ content_length: -1
+ uncompressed: true
+ body: "{\"id\":\"msg_014AQFTJZeZ1KNGT5y9TSMSs\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"Oi! or Olá!\\n\\nBoth are common ways to say \\\"hi\\\" in Portuguese. \\\"Oi\\\" is more casual and commonly used in Brazilian Portuguese, while \\\"Olá\\\" is a bit more formal and used in both Brazilian and European Portuguese.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":16,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":60,\"service_tier\":\"standard\"}}"
+ headers:
+ Content-Type:
+ - application/json
+ status: 200 OK
+ code: 200
+ duration: 2.412166125s
providertests/testdata/TestSimple/openai-gpt-4o.yaml 🔗
@@ -0,0 +1,33 @@
+---
+version: 2
+interactions:
+- id: 0
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 138
+ host: ""
+ body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Say hi in Portuguese\",\"role\":\"user\"}],\"model\":\"gpt-4o\"}"
+ headers:
+ Accept:
+ - application/json
+ Content-Type:
+ - application/json
+ User-Agent:
+ - OpenAI/Go 2.3.0
+ url: https://api.openai.com/v1/chat/completions
+ method: POST
+ response:
+ proto: HTTP/2.0
+ proto_major: 2
+ proto_minor: 0
+ content_length: -1
+ uncompressed: true
+ body: "{\n \"id\": \"chatcmpl-CBolfurp0H2jFXSwJHVGbLWOYHhbM\",\n \"object\": \"chat.completion\",\n \"created\": 1756932795,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"Olá! Como posso ajudar você hoje?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 20,\n \"completion_tokens\": 8,\n \"total_tokens\": 28,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_f33640a400\"\n}\n"
+ headers:
+ Content-Type:
+ - application/json
+ status: 200 OK
+ code: 200
+ duration: 3.363218s