From 3340d64e228106ce7fde9073a606f0704519a34b Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Wed, 3 Sep 2025 18:00:00 -0300 Subject: [PATCH] test: setup go-vcr to start recording real providers interactions --- .gitattributes | 1 + go.mod | 3 + go.sum | 6 ++ providertests/.env.sample | 2 + providertests/builders_test.go | 47 ++++++++++++ providertests/provider_test.go | 39 ++++++++++ providertests/recorder_test.go | 74 +++++++++++++++++++ .../TestSimple/anthropic-claude-sonnet.yaml | 33 +++++++++ .../testdata/TestSimple/openai-gpt-4o.yaml | 33 +++++++++ 9 files changed, 238 insertions(+) create mode 100644 .gitattributes create mode 100644 providertests/.env.sample create mode 100644 providertests/builders_test.go create mode 100644 providertests/provider_test.go create mode 100644 providertests/recorder_test.go create mode 100644 providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestSimple/openai-gpt-4o.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..46d706f3c753c853c57f0af8fbd8ff5cfea89b2e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +providertests/testdata/**/*.yaml linguist-generated=true diff --git a/go.mod b/go.mod index 1353ace5d5f704fe61f57b3fb8e94870f04a898f..90f60c75743525ae807e0688f72eec15d47c6375 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,15 @@ require ( github.com/charmbracelet/x/json v0.2.0 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/google/uuid v1.6.0 + github.com/joho/godotenv v1.5.1 github.com/openai/openai-go/v2 v2.3.0 github.com/stretchr/testify v1.11.1 + gopkg.in/dnaeon/go-vcr.v4 v4.0.5 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index f0842950eff85bf44d2d221921d1696a5938aa79..93ed55706736f113cce0de10f313c77aaa17700d 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/openai/openai-go/v2 v2.3.0 h1:y9U+V1tlHjvvb/5XIswuySqnG5EnKBFAbMxgBvTHXvg= github.com/openai/openai-go/v2 v2.3.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -26,5 +30,7 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/dnaeon/go-vcr.v4 v4.0.5 h1:I0hpTIvD5rII+8LgYGrHMA2d4SQPoL6u7ZvJakWKsiA= +gopkg.in/dnaeon/go-vcr.v4 v4.0.5/go.mod h1:dRos81TkW9C1WJt6tTaE+uV2Lo8qJT3AG2b35+CB/nQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/providertests/.env.sample b/providertests/.env.sample new file mode 100644 index 0000000000000000000000000000000000000000..69b599454e29c6b7ddbb4e0207adcd1aeca921d4 --- /dev/null +++ b/providertests/.env.sample @@ -0,0 +1,2 @@ +ANTHROPIC_API_KEY= +OPENAI_API_KEY= diff --git a/providertests/builders_test.go b/providertests/builders_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3d5e080ce4b8f1d09a1c32c6ff055dff0c599ab4 --- /dev/null +++ b/providertests/builders_test.go @@ -0,0 +1,47 @@ +package providertests + +import ( + "net/http" + "os" + + "github.com/charmbracelet/ai/ai" + "github.com/charmbracelet/ai/anthropic" + "github.com/charmbracelet/ai/openai" + "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" +) + +type builderFunc func(r *recorder.Recorder) (ai.LanguageModel, error) + +type builderPair struct { + name string + builder builderFunc +} + +var languageModelBuilders = []builderPair{ + {"openai-gpt-4o", builderOpenaiGpt4o}, + {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4}, +} + +func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := openai.New( + openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), + openai.WithHTTPClient(&http.Client{Transport: r}), + ) + model, err := provider.LanguageModel("gpt-4o") + if err != nil { + return nil, err + } + return model, nil +} + +func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := anthropic.New( + anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), + anthropic.WithHTTPClient(&http.Client{Transport: r}), + ) + model, err := provider.LanguageModel("claude-sonnet-4-20250514") + if err != nil { + return nil, err + } + return model, nil +} diff --git a/providertests/provider_test.go b/providertests/provider_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0f77692dfc3d6af0910f49cfa1e4f83a5ea34ab9 --- /dev/null +++ b/providertests/provider_test.go @@ -0,0 +1,39 @@ +package providertests + +import ( + "strings" + "testing" + + "github.com/charmbracelet/ai/ai" + _ "github.com/joho/godotenv/autoload" +) + +func TestSimple(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "Say hi in Portuguese", + }) + if err != nil { + t.Fatalf("failed to generate: %v", err) + } + + want := "Olá" + got := result.Response.Content.Text() + if !strings.Contains(got, want) { + t.Fatalf("unexpected response: got %q, want %q", got, want) + } + }) + } +} diff --git a/providertests/recorder_test.go b/providertests/recorder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0ff64571e5a0a0e658c5c89821b1084b4abb9def --- /dev/null +++ b/providertests/recorder_test.go @@ -0,0 +1,74 @@ +package providertests + +import ( + "bytes" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + + "gopkg.in/dnaeon/go-vcr.v4/pkg/cassette" + "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" +) + +func newRecorder(t *testing.T) *recorder.Recorder { + cassetteName := filepath.Join("testdata", t.Name()) + + r, err := recorder.New( + cassetteName, + recorder.WithMode(recorder.ModeRecordOnce), + recorder.WithMatcher(customMatcher(t)), + recorder.WithHook(hookRemoveHeaders, recorder.AfterCaptureHook), + ) + if err != nil { + t.Fatalf("recorder: failed to create recorder: %v", err) + } + + t.Cleanup(func() { + if err := r.Stop(); err != nil { + t.Errorf("recorder: failed to stop recorder: %v", err) + } + }) + + return r +} + +func customMatcher(t *testing.T) recorder.MatcherFunc { + return func(r *http.Request, i cassette.Request) bool { + if r.Body == nil || r.Body == http.NoBody { + return cassette.DefaultMatcher(r, i) + } + + var reqBody []byte + var err error + reqBody, err = io.ReadAll(r.Body) + if err != nil { + t.Fatalf("recorder: failed to read request body") + } + r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(reqBody)) + + return r.Method == i.Method && r.URL.String() == i.URL && string(reqBody) == i.Body + } +} + +var headersToKeep = map[string]struct{}{ + "accept": {}, + "content-type": {}, + "user-agent": {}, +} + +func hookRemoveHeaders(i *cassette.Interaction) error { + for k := range i.Request.Headers { + if _, ok := headersToKeep[strings.ToLower(k)]; !ok { + delete(i.Request.Headers, k) + } + } + for k := range i.Response.Headers { + if _, ok := headersToKeep[strings.ToLower(k)]; !ok { + delete(i.Response.Headers, k) + } + } + return nil +} diff --git a/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml b/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00c4970125fd011cd782733a01411adcb151834c --- /dev/null +++ b/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml @@ -0,0 +1,33 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 205 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"Say hi in Portuguese\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_014AQFTJZeZ1KNGT5y9TSMSs\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"Oi! or Olá!\\n\\nBoth are common ways to say \\\"hi\\\" in Portuguese. \\\"Oi\\\" is more casual and commonly used in Brazilian Portuguese, while \\\"Olá\\\" is a bit more formal and used in both Brazilian and European Portuguese.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":16,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":60,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.412166125s diff --git a/providertests/testdata/TestSimple/openai-gpt-4o.yaml b/providertests/testdata/TestSimple/openai-gpt-4o.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72b7135ec8807e5c293ed3cbb6f4a43ac33f7da3 --- /dev/null +++ b/providertests/testdata/TestSimple/openai-gpt-4o.yaml @@ -0,0 +1,33 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 138 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Say hi in Portuguese\",\"role\":\"user\"}],\"model\":\"gpt-4o\"}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CBolfurp0H2jFXSwJHVGbLWOYHhbM\",\n \"object\": \"chat.completion\",\n \"created\": 1756932795,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"Olá! Como posso ajudar você hoje?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 20,\n \"completion_tokens\": 8,\n \"total_tokens\": 28,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_f33640a400\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 3.363218s