1package openai
2
3import (
4 "context"
5 "encoding/base64"
6 "encoding/json"
7 "errors"
8 "fmt"
9 "io"
10 "strings"
11
12 "github.com/charmbracelet/fantasy/ai"
13 xjson "github.com/charmbracelet/x/json"
14 "github.com/google/uuid"
15 "github.com/openai/openai-go/v2"
16 "github.com/openai/openai-go/v2/packages/param"
17 "github.com/openai/openai-go/v2/shared"
18)
19
20type languageModel struct {
21 provider string
22 modelID string
23 client openai.Client
24 prepareCallFunc LanguageModelPrepareCallFunc
25 mapFinishReasonFunc LanguageModelMapFinishReasonFunc
26 extraContentFunc LanguageModelExtraContentFunc
27 usageFunc LanguageModelUsageFunc
28 streamUsageFunc LanguageModelStreamUsageFunc
29 streamExtraFunc LanguageModelStreamExtraFunc
30 streamProviderMetadataFunc LanguageModelStreamProviderMetadataFunc
31}
32
33type LanguageModelOption = func(*languageModel)
34
35func WithLanguageModelPrepareCallFunc(fn LanguageModelPrepareCallFunc) LanguageModelOption {
36 return func(l *languageModel) {
37 l.prepareCallFunc = fn
38 }
39}
40
41func WithLanguageModelMapFinishReasonFunc(fn LanguageModelMapFinishReasonFunc) LanguageModelOption {
42 return func(l *languageModel) {
43 l.mapFinishReasonFunc = fn
44 }
45}
46
47func WithLanguageModelExtraContentFunc(fn LanguageModelExtraContentFunc) LanguageModelOption {
48 return func(l *languageModel) {
49 l.extraContentFunc = fn
50 }
51}
52
53func WithLanguageModelStreamExtraFunc(fn LanguageModelStreamExtraFunc) LanguageModelOption {
54 return func(l *languageModel) {
55 l.streamExtraFunc = fn
56 }
57}
58
59func WithLanguageModelUsageFunc(fn LanguageModelUsageFunc) LanguageModelOption {
60 return func(l *languageModel) {
61 l.usageFunc = fn
62 }
63}
64
65func WithLanguageModelStreamUsageFunc(fn LanguageModelStreamUsageFunc) LanguageModelOption {
66 return func(l *languageModel) {
67 l.streamUsageFunc = fn
68 }
69}
70
71func newLanguageModel(modelID string, provider string, client openai.Client, opts ...LanguageModelOption) languageModel {
72 model := languageModel{
73 modelID: modelID,
74 provider: provider,
75 client: client,
76 prepareCallFunc: DefaultPrepareCallFunc,
77 mapFinishReasonFunc: DefaultMapFinishReasonFunc,
78 usageFunc: DefaultUsageFunc,
79 streamUsageFunc: DefaultStreamUsageFunc,
80 streamProviderMetadataFunc: DefaultStreamProviderMetadataFunc,
81 }
82
83 for _, o := range opts {
84 o(&model)
85 }
86 return model
87}
88
89type streamToolCall struct {
90 id string
91 name string
92 arguments string
93 hasFinished bool
94}
95
96// Model implements ai.LanguageModel.
97func (o languageModel) Model() string {
98 return o.modelID
99}
100
101// Provider implements ai.LanguageModel.
102func (o languageModel) Provider() string {
103 return o.provider
104}
105
106func (o languageModel) prepareParams(call ai.Call) (*openai.ChatCompletionNewParams, []ai.CallWarning, error) {
107 params := &openai.ChatCompletionNewParams{}
108 messages, warnings := toPrompt(call.Prompt)
109 if call.TopK != nil {
110 warnings = append(warnings, ai.CallWarning{
111 Type: ai.CallWarningTypeUnsupportedSetting,
112 Setting: "top_k",
113 })
114 }
115
116 if call.MaxOutputTokens != nil {
117 params.MaxTokens = param.NewOpt(*call.MaxOutputTokens)
118 }
119 if call.Temperature != nil {
120 params.Temperature = param.NewOpt(*call.Temperature)
121 }
122 if call.TopP != nil {
123 params.TopP = param.NewOpt(*call.TopP)
124 }
125 if call.FrequencyPenalty != nil {
126 params.FrequencyPenalty = param.NewOpt(*call.FrequencyPenalty)
127 }
128 if call.PresencePenalty != nil {
129 params.PresencePenalty = param.NewOpt(*call.PresencePenalty)
130 }
131
132 if isReasoningModel(o.modelID) {
133 // remove unsupported settings for reasoning models
134 // see https://platform.openai.com/docs/guides/reasoning#limitations
135 if call.Temperature != nil {
136 params.Temperature = param.Opt[float64]{}
137 warnings = append(warnings, ai.CallWarning{
138 Type: ai.CallWarningTypeUnsupportedSetting,
139 Setting: "temperature",
140 Details: "temperature is not supported for reasoning models",
141 })
142 }
143 if call.TopP != nil {
144 params.TopP = param.Opt[float64]{}
145 warnings = append(warnings, ai.CallWarning{
146 Type: ai.CallWarningTypeUnsupportedSetting,
147 Setting: "TopP",
148 Details: "TopP is not supported for reasoning models",
149 })
150 }
151 if call.FrequencyPenalty != nil {
152 params.FrequencyPenalty = param.Opt[float64]{}
153 warnings = append(warnings, ai.CallWarning{
154 Type: ai.CallWarningTypeUnsupportedSetting,
155 Setting: "FrequencyPenalty",
156 Details: "FrequencyPenalty is not supported for reasoning models",
157 })
158 }
159 if call.PresencePenalty != nil {
160 params.PresencePenalty = param.Opt[float64]{}
161 warnings = append(warnings, ai.CallWarning{
162 Type: ai.CallWarningTypeUnsupportedSetting,
163 Setting: "PresencePenalty",
164 Details: "PresencePenalty is not supported for reasoning models",
165 })
166 }
167
168 // reasoning models use max_completion_tokens instead of max_tokens
169 if call.MaxOutputTokens != nil {
170 if !params.MaxCompletionTokens.Valid() {
171 params.MaxCompletionTokens = param.NewOpt(*call.MaxOutputTokens)
172 }
173 params.MaxTokens = param.Opt[int64]{}
174 }
175 }
176
177 // Handle search preview models
178 if isSearchPreviewModel(o.modelID) {
179 if call.Temperature != nil {
180 params.Temperature = param.Opt[float64]{}
181 warnings = append(warnings, ai.CallWarning{
182 Type: ai.CallWarningTypeUnsupportedSetting,
183 Setting: "temperature",
184 Details: "temperature is not supported for the search preview models and has been removed.",
185 })
186 }
187 }
188
189 optionsWarnings, err := o.prepareCallFunc(o, params, call)
190 if err != nil {
191 return nil, nil, err
192 }
193
194 if len(optionsWarnings) > 0 {
195 warnings = append(warnings, optionsWarnings...)
196 }
197
198 params.Messages = messages
199 params.Model = o.modelID
200
201 if len(call.Tools) > 0 {
202 tools, toolChoice, toolWarnings := toOpenAiTools(call.Tools, call.ToolChoice)
203 params.Tools = tools
204 if toolChoice != nil {
205 params.ToolChoice = *toolChoice
206 }
207 warnings = append(warnings, toolWarnings...)
208 }
209 return params, warnings, nil
210}
211
212func (o languageModel) handleError(err error) error {
213 var apiErr *openai.Error
214 if errors.As(err, &apiErr) {
215 requestDump := apiErr.DumpRequest(true)
216 responseDump := apiErr.DumpResponse(true)
217 headers := map[string]string{}
218 for k, h := range apiErr.Response.Header {
219 v := h[len(h)-1]
220 headers[strings.ToLower(k)] = v
221 }
222 return ai.NewAPICallError(
223 apiErr.Message,
224 apiErr.Request.URL.String(),
225 string(requestDump),
226 apiErr.StatusCode,
227 headers,
228 string(responseDump),
229 apiErr,
230 false,
231 )
232 }
233 return err
234}
235
236// Generate implements ai.LanguageModel.
237func (o languageModel) Generate(ctx context.Context, call ai.Call) (*ai.Response, error) {
238 params, warnings, err := o.prepareParams(call)
239 if err != nil {
240 return nil, err
241 }
242 response, err := o.client.Chat.Completions.New(ctx, *params)
243 if err != nil {
244 return nil, o.handleError(err)
245 }
246
247 if len(response.Choices) == 0 {
248 return nil, errors.New("no response generated")
249 }
250 choice := response.Choices[0]
251 content := make([]ai.Content, 0, 1+len(choice.Message.ToolCalls)+len(choice.Message.Annotations))
252 text := choice.Message.Content
253 if text != "" {
254 content = append(content, ai.TextContent{
255 Text: text,
256 })
257 }
258 if o.extraContentFunc != nil {
259 extraContent := o.extraContentFunc(choice)
260 content = append(content, extraContent...)
261 }
262 for _, tc := range choice.Message.ToolCalls {
263 toolCallID := tc.ID
264 content = append(content, ai.ToolCallContent{
265 ProviderExecuted: false, // TODO: update when handling other tools
266 ToolCallID: toolCallID,
267 ToolName: tc.Function.Name,
268 Input: tc.Function.Arguments,
269 })
270 }
271 // Handle annotations/citations
272 for _, annotation := range choice.Message.Annotations {
273 if annotation.Type == "url_citation" {
274 content = append(content, ai.SourceContent{
275 SourceType: ai.SourceTypeURL,
276 ID: uuid.NewString(),
277 URL: annotation.URLCitation.URL,
278 Title: annotation.URLCitation.Title,
279 })
280 }
281 }
282
283 usage, providerMetadata := o.usageFunc(*response)
284
285 mappedFinishReason := o.mapFinishReasonFunc(choice.FinishReason)
286 if len(choice.Message.ToolCalls) > 0 {
287 mappedFinishReason = ai.FinishReasonToolCalls
288 }
289 return &ai.Response{
290 Content: content,
291 Usage: usage,
292 FinishReason: mappedFinishReason,
293 ProviderMetadata: ai.ProviderMetadata{
294 Name: providerMetadata,
295 },
296 Warnings: warnings,
297 }, nil
298}
299
300// Stream implements ai.LanguageModel.
301func (o languageModel) Stream(ctx context.Context, call ai.Call) (ai.StreamResponse, error) {
302 params, warnings, err := o.prepareParams(call)
303 if err != nil {
304 return nil, err
305 }
306
307 params.StreamOptions = openai.ChatCompletionStreamOptionsParam{
308 IncludeUsage: openai.Bool(true),
309 }
310
311 stream := o.client.Chat.Completions.NewStreaming(ctx, *params)
312 isActiveText := false
313 toolCalls := make(map[int64]streamToolCall)
314
315 // Build provider metadata for streaming
316 providerMetadata := ai.ProviderMetadata{
317 Name: &ProviderMetadata{},
318 }
319 acc := openai.ChatCompletionAccumulator{}
320 extraContext := make(map[string]any)
321 var usage ai.Usage
322 var finishReason string
323 return func(yield func(ai.StreamPart) bool) {
324 if len(warnings) > 0 {
325 if !yield(ai.StreamPart{
326 Type: ai.StreamPartTypeWarnings,
327 Warnings: warnings,
328 }) {
329 return
330 }
331 }
332 for stream.Next() {
333 chunk := stream.Current()
334 acc.AddChunk(chunk)
335 usage, providerMetadata = o.streamUsageFunc(chunk, extraContext, providerMetadata)
336 if len(chunk.Choices) == 0 {
337 continue
338 }
339 for _, choice := range chunk.Choices {
340 if choice.FinishReason != "" {
341 finishReason = choice.FinishReason
342 }
343 switch {
344 case choice.Delta.Content != "":
345 if !isActiveText {
346 isActiveText = true
347 if !yield(ai.StreamPart{
348 Type: ai.StreamPartTypeTextStart,
349 ID: "0",
350 }) {
351 return
352 }
353 }
354 if !yield(ai.StreamPart{
355 Type: ai.StreamPartTypeTextDelta,
356 ID: "0",
357 Delta: choice.Delta.Content,
358 }) {
359 return
360 }
361 case len(choice.Delta.ToolCalls) > 0:
362 if isActiveText {
363 isActiveText = false
364 if !yield(ai.StreamPart{
365 Type: ai.StreamPartTypeTextEnd,
366 ID: "0",
367 }) {
368 return
369 }
370 }
371
372 for _, toolCallDelta := range choice.Delta.ToolCalls {
373 if existingToolCall, ok := toolCalls[toolCallDelta.Index]; ok {
374 if existingToolCall.hasFinished {
375 continue
376 }
377 if toolCallDelta.Function.Arguments != "" {
378 existingToolCall.arguments += toolCallDelta.Function.Arguments
379 }
380 if !yield(ai.StreamPart{
381 Type: ai.StreamPartTypeToolInputDelta,
382 ID: existingToolCall.id,
383 Delta: toolCallDelta.Function.Arguments,
384 }) {
385 return
386 }
387 toolCalls[toolCallDelta.Index] = existingToolCall
388 if xjson.IsValid(existingToolCall.arguments) {
389 if !yield(ai.StreamPart{
390 Type: ai.StreamPartTypeToolInputEnd,
391 ID: existingToolCall.id,
392 }) {
393 return
394 }
395
396 if !yield(ai.StreamPart{
397 Type: ai.StreamPartTypeToolCall,
398 ID: existingToolCall.id,
399 ToolCallName: existingToolCall.name,
400 ToolCallInput: existingToolCall.arguments,
401 }) {
402 return
403 }
404 existingToolCall.hasFinished = true
405 toolCalls[toolCallDelta.Index] = existingToolCall
406 }
407 } else {
408 // Does not exist
409 var err error
410 if toolCallDelta.Type != "function" {
411 err = ai.NewInvalidResponseDataError(toolCallDelta, "Expected 'function' type.")
412 }
413 if toolCallDelta.ID == "" {
414 err = ai.NewInvalidResponseDataError(toolCallDelta, "Expected 'id' to be a string.")
415 }
416 if toolCallDelta.Function.Name == "" {
417 err = ai.NewInvalidResponseDataError(toolCallDelta, "Expected 'function.name' to be a string.")
418 }
419 if err != nil {
420 yield(ai.StreamPart{
421 Type: ai.StreamPartTypeError,
422 Error: o.handleError(stream.Err()),
423 })
424 return
425 }
426
427 if !yield(ai.StreamPart{
428 Type: ai.StreamPartTypeToolInputStart,
429 ID: toolCallDelta.ID,
430 ToolCallName: toolCallDelta.Function.Name,
431 }) {
432 return
433 }
434 toolCalls[toolCallDelta.Index] = streamToolCall{
435 id: toolCallDelta.ID,
436 name: toolCallDelta.Function.Name,
437 arguments: toolCallDelta.Function.Arguments,
438 }
439
440 exTc := toolCalls[toolCallDelta.Index]
441 if exTc.arguments != "" {
442 if !yield(ai.StreamPart{
443 Type: ai.StreamPartTypeToolInputDelta,
444 ID: exTc.id,
445 Delta: exTc.arguments,
446 }) {
447 return
448 }
449 if xjson.IsValid(toolCalls[toolCallDelta.Index].arguments) {
450 if !yield(ai.StreamPart{
451 Type: ai.StreamPartTypeToolInputEnd,
452 ID: toolCallDelta.ID,
453 }) {
454 return
455 }
456
457 if !yield(ai.StreamPart{
458 Type: ai.StreamPartTypeToolCall,
459 ID: exTc.id,
460 ToolCallName: exTc.name,
461 ToolCallInput: exTc.arguments,
462 }) {
463 return
464 }
465 exTc.hasFinished = true
466 toolCalls[toolCallDelta.Index] = exTc
467 }
468 }
469 continue
470 }
471 }
472 }
473
474 if o.streamExtraFunc != nil {
475 updatedContext, shouldContinue := o.streamExtraFunc(chunk, yield, extraContext)
476 if !shouldContinue {
477 return
478 }
479 extraContext = updatedContext
480 }
481 }
482
483 // Check for annotations in the delta's raw JSON
484 for _, choice := range chunk.Choices {
485 if annotations := parseAnnotationsFromDelta(choice.Delta); len(annotations) > 0 {
486 for _, annotation := range annotations {
487 if annotation.Type == "url_citation" {
488 if !yield(ai.StreamPart{
489 Type: ai.StreamPartTypeSource,
490 ID: uuid.NewString(),
491 SourceType: ai.SourceTypeURL,
492 URL: annotation.URLCitation.URL,
493 Title: annotation.URLCitation.Title,
494 }) {
495 return
496 }
497 }
498 }
499 }
500 }
501 }
502 err := stream.Err()
503 if err == nil || errors.Is(err, io.EOF) {
504 // finished
505 if isActiveText {
506 isActiveText = false
507 if !yield(ai.StreamPart{
508 Type: ai.StreamPartTypeTextEnd,
509 ID: "0",
510 }) {
511 return
512 }
513 }
514
515 if len(acc.Choices) > 0 {
516 choice := acc.Choices[0]
517 // Add logprobs if available
518 providerMetadata = o.streamProviderMetadataFunc(choice, providerMetadata)
519
520 // Handle annotations/citations from accumulated response
521 for _, annotation := range choice.Message.Annotations {
522 if annotation.Type == "url_citation" {
523 if !yield(ai.StreamPart{
524 Type: ai.StreamPartTypeSource,
525 ID: acc.ID,
526 SourceType: ai.SourceTypeURL,
527 URL: annotation.URLCitation.URL,
528 Title: annotation.URLCitation.Title,
529 }) {
530 return
531 }
532 }
533 }
534 }
535 mappedFinishReason := o.mapFinishReasonFunc(finishReason)
536 if len(acc.Choices) > 0 {
537 choice := acc.Choices[0]
538 if len(choice.Message.ToolCalls) > 0 {
539 mappedFinishReason = ai.FinishReasonToolCalls
540 }
541 }
542 yield(ai.StreamPart{
543 Type: ai.StreamPartTypeFinish,
544 Usage: usage,
545 FinishReason: mappedFinishReason,
546 ProviderMetadata: providerMetadata,
547 })
548 return
549 } else {
550 yield(ai.StreamPart{
551 Type: ai.StreamPartTypeError,
552 Error: o.handleError(err),
553 })
554 return
555 }
556 }, nil
557}
558
559func isReasoningModel(modelID string) bool {
560 return strings.HasPrefix(modelID, "o") || strings.HasPrefix(modelID, "gpt-5") || strings.HasPrefix(modelID, "gpt-5-chat")
561}
562
563func isSearchPreviewModel(modelID string) bool {
564 return strings.Contains(modelID, "search-preview")
565}
566
567func supportsFlexProcessing(modelID string) bool {
568 return strings.HasPrefix(modelID, "o3") || strings.HasPrefix(modelID, "o4-mini") || strings.HasPrefix(modelID, "gpt-5")
569}
570
571func supportsPriorityProcessing(modelID string) bool {
572 return strings.HasPrefix(modelID, "gpt-4") || strings.HasPrefix(modelID, "gpt-5") ||
573 strings.HasPrefix(modelID, "gpt-5-mini") || strings.HasPrefix(modelID, "o3") ||
574 strings.HasPrefix(modelID, "o4-mini")
575}
576
577func toOpenAiTools(tools []ai.Tool, toolChoice *ai.ToolChoice) (openAiTools []openai.ChatCompletionToolUnionParam, openAiToolChoice *openai.ChatCompletionToolChoiceOptionUnionParam, warnings []ai.CallWarning) {
578 for _, tool := range tools {
579 if tool.GetType() == ai.ToolTypeFunction {
580 ft, ok := tool.(ai.FunctionTool)
581 if !ok {
582 continue
583 }
584 openAiTools = append(openAiTools, openai.ChatCompletionToolUnionParam{
585 OfFunction: &openai.ChatCompletionFunctionToolParam{
586 Function: shared.FunctionDefinitionParam{
587 Name: ft.Name,
588 Description: param.NewOpt(ft.Description),
589 Parameters: openai.FunctionParameters(ft.InputSchema),
590 Strict: param.NewOpt(false),
591 },
592 Type: "function",
593 },
594 })
595 continue
596 }
597
598 // TODO: handle provider tool calls
599 warnings = append(warnings, ai.CallWarning{
600 Type: ai.CallWarningTypeUnsupportedTool,
601 Tool: tool,
602 Message: "tool is not supported",
603 })
604 }
605 if toolChoice == nil {
606 return openAiTools, openAiToolChoice, warnings
607 }
608
609 switch *toolChoice {
610 case ai.ToolChoiceAuto:
611 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
612 OfAuto: param.NewOpt("auto"),
613 }
614 case ai.ToolChoiceNone:
615 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
616 OfAuto: param.NewOpt("none"),
617 }
618 default:
619 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
620 OfFunctionToolChoice: &openai.ChatCompletionNamedToolChoiceParam{
621 Type: "function",
622 Function: openai.ChatCompletionNamedToolChoiceFunctionParam{
623 Name: string(*toolChoice),
624 },
625 },
626 }
627 }
628 return openAiTools, openAiToolChoice, warnings
629}
630
631func toPrompt(prompt ai.Prompt) ([]openai.ChatCompletionMessageParamUnion, []ai.CallWarning) {
632 var messages []openai.ChatCompletionMessageParamUnion
633 var warnings []ai.CallWarning
634 for _, msg := range prompt {
635 switch msg.Role {
636 case ai.MessageRoleSystem:
637 var systemPromptParts []string
638 for _, c := range msg.Content {
639 if c.GetType() != ai.ContentTypeText {
640 warnings = append(warnings, ai.CallWarning{
641 Type: ai.CallWarningTypeOther,
642 Message: "system prompt can only have text content",
643 })
644 continue
645 }
646 textPart, ok := ai.AsContentType[ai.TextPart](c)
647 if !ok {
648 warnings = append(warnings, ai.CallWarning{
649 Type: ai.CallWarningTypeOther,
650 Message: "system prompt text part does not have the right type",
651 })
652 continue
653 }
654 text := textPart.Text
655 if strings.TrimSpace(text) != "" {
656 systemPromptParts = append(systemPromptParts, textPart.Text)
657 }
658 }
659 if len(systemPromptParts) == 0 {
660 warnings = append(warnings, ai.CallWarning{
661 Type: ai.CallWarningTypeOther,
662 Message: "system prompt has no text parts",
663 })
664 continue
665 }
666 messages = append(messages, openai.SystemMessage(strings.Join(systemPromptParts, "\n")))
667 case ai.MessageRoleUser:
668 // simple user message just text content
669 if len(msg.Content) == 1 && msg.Content[0].GetType() == ai.ContentTypeText {
670 textPart, ok := ai.AsContentType[ai.TextPart](msg.Content[0])
671 if !ok {
672 warnings = append(warnings, ai.CallWarning{
673 Type: ai.CallWarningTypeOther,
674 Message: "user message text part does not have the right type",
675 })
676 continue
677 }
678 messages = append(messages, openai.UserMessage(textPart.Text))
679 continue
680 }
681 // text content and attachments
682 // for now we only support image content later we need to check
683 // TODO: add the supported media types to the language model so we
684 // can use that to validate the data here.
685 var content []openai.ChatCompletionContentPartUnionParam
686 for _, c := range msg.Content {
687 switch c.GetType() {
688 case ai.ContentTypeText:
689 textPart, ok := ai.AsContentType[ai.TextPart](c)
690 if !ok {
691 warnings = append(warnings, ai.CallWarning{
692 Type: ai.CallWarningTypeOther,
693 Message: "user message text part does not have the right type",
694 })
695 continue
696 }
697 content = append(content, openai.ChatCompletionContentPartUnionParam{
698 OfText: &openai.ChatCompletionContentPartTextParam{
699 Text: textPart.Text,
700 },
701 })
702 case ai.ContentTypeFile:
703 filePart, ok := ai.AsContentType[ai.FilePart](c)
704 if !ok {
705 warnings = append(warnings, ai.CallWarning{
706 Type: ai.CallWarningTypeOther,
707 Message: "user message file part does not have the right type",
708 })
709 continue
710 }
711
712 switch {
713 case strings.HasPrefix(filePart.MediaType, "image/"):
714 // Handle image files
715 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
716 data := "data:" + filePart.MediaType + ";base64," + base64Encoded
717 imageURL := openai.ChatCompletionContentPartImageImageURLParam{URL: data}
718
719 // Check for provider-specific options like image detail
720 if providerOptions, ok := filePart.ProviderOptions[Name]; ok {
721 if detail, ok := providerOptions.(*ProviderFileOptions); ok {
722 imageURL.Detail = detail.ImageDetail
723 }
724 }
725
726 imageBlock := openai.ChatCompletionContentPartImageParam{ImageURL: imageURL}
727 content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
728
729 case filePart.MediaType == "audio/wav":
730 // Handle WAV audio files
731 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
732 audioBlock := openai.ChatCompletionContentPartInputAudioParam{
733 InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{
734 Data: base64Encoded,
735 Format: "wav",
736 },
737 }
738 content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
739
740 case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3":
741 // Handle MP3 audio files
742 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
743 audioBlock := openai.ChatCompletionContentPartInputAudioParam{
744 InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{
745 Data: base64Encoded,
746 Format: "mp3",
747 },
748 }
749 content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
750
751 case filePart.MediaType == "application/pdf":
752 // Handle PDF files
753 dataStr := string(filePart.Data)
754
755 // Check if data looks like a file ID (starts with "file-")
756 if strings.HasPrefix(dataStr, "file-") {
757 fileBlock := openai.ChatCompletionContentPartFileParam{
758 File: openai.ChatCompletionContentPartFileFileParam{
759 FileID: param.NewOpt(dataStr),
760 },
761 }
762 content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
763 } else {
764 // Handle as base64 data
765 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
766 data := "data:application/pdf;base64," + base64Encoded
767
768 filename := filePart.Filename
769 if filename == "" {
770 // Generate default filename based on content index
771 filename = fmt.Sprintf("part-%d.pdf", len(content))
772 }
773
774 fileBlock := openai.ChatCompletionContentPartFileParam{
775 File: openai.ChatCompletionContentPartFileFileParam{
776 Filename: param.NewOpt(filename),
777 FileData: param.NewOpt(data),
778 },
779 }
780 content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
781 }
782
783 default:
784 warnings = append(warnings, ai.CallWarning{
785 Type: ai.CallWarningTypeOther,
786 Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType),
787 })
788 }
789 }
790 }
791 messages = append(messages, openai.UserMessage(content))
792 case ai.MessageRoleAssistant:
793 // simple assistant message just text content
794 if len(msg.Content) == 1 && msg.Content[0].GetType() == ai.ContentTypeText {
795 textPart, ok := ai.AsContentType[ai.TextPart](msg.Content[0])
796 if !ok {
797 warnings = append(warnings, ai.CallWarning{
798 Type: ai.CallWarningTypeOther,
799 Message: "assistant message text part does not have the right type",
800 })
801 continue
802 }
803 messages = append(messages, openai.AssistantMessage(textPart.Text))
804 continue
805 }
806 assistantMsg := openai.ChatCompletionAssistantMessageParam{
807 Role: "assistant",
808 }
809 for _, c := range msg.Content {
810 switch c.GetType() {
811 case ai.ContentTypeText:
812 textPart, ok := ai.AsContentType[ai.TextPart](c)
813 if !ok {
814 warnings = append(warnings, ai.CallWarning{
815 Type: ai.CallWarningTypeOther,
816 Message: "assistant message text part does not have the right type",
817 })
818 continue
819 }
820 assistantMsg.Content = openai.ChatCompletionAssistantMessageParamContentUnion{
821 OfString: param.NewOpt(textPart.Text),
822 }
823 case ai.ContentTypeToolCall:
824 toolCallPart, ok := ai.AsContentType[ai.ToolCallPart](c)
825 if !ok {
826 warnings = append(warnings, ai.CallWarning{
827 Type: ai.CallWarningTypeOther,
828 Message: "assistant message tool part does not have the right type",
829 })
830 continue
831 }
832 assistantMsg.ToolCalls = append(assistantMsg.ToolCalls,
833 openai.ChatCompletionMessageToolCallUnionParam{
834 OfFunction: &openai.ChatCompletionMessageFunctionToolCallParam{
835 ID: toolCallPart.ToolCallID,
836 Type: "function",
837 Function: openai.ChatCompletionMessageFunctionToolCallFunctionParam{
838 Name: toolCallPart.ToolName,
839 Arguments: toolCallPart.Input,
840 },
841 },
842 })
843 }
844 }
845 messages = append(messages, openai.ChatCompletionMessageParamUnion{
846 OfAssistant: &assistantMsg,
847 })
848 case ai.MessageRoleTool:
849 for _, c := range msg.Content {
850 if c.GetType() != ai.ContentTypeToolResult {
851 warnings = append(warnings, ai.CallWarning{
852 Type: ai.CallWarningTypeOther,
853 Message: "tool message can only have tool result content",
854 })
855 continue
856 }
857
858 toolResultPart, ok := ai.AsContentType[ai.ToolResultPart](c)
859 if !ok {
860 warnings = append(warnings, ai.CallWarning{
861 Type: ai.CallWarningTypeOther,
862 Message: "tool message result part does not have the right type",
863 })
864 continue
865 }
866
867 switch toolResultPart.Output.GetType() {
868 case ai.ToolResultContentTypeText:
869 output, ok := ai.AsToolResultOutputType[ai.ToolResultOutputContentText](toolResultPart.Output)
870 if !ok {
871 warnings = append(warnings, ai.CallWarning{
872 Type: ai.CallWarningTypeOther,
873 Message: "tool result output does not have the right type",
874 })
875 continue
876 }
877 messages = append(messages, openai.ToolMessage(output.Text, toolResultPart.ToolCallID))
878 case ai.ToolResultContentTypeError:
879 // TODO: check if better handling is needed
880 output, ok := ai.AsToolResultOutputType[ai.ToolResultOutputContentError](toolResultPart.Output)
881 if !ok {
882 warnings = append(warnings, ai.CallWarning{
883 Type: ai.CallWarningTypeOther,
884 Message: "tool result output does not have the right type",
885 })
886 continue
887 }
888 messages = append(messages, openai.ToolMessage(output.Error.Error(), toolResultPart.ToolCallID))
889 }
890 }
891 }
892 }
893 return messages, warnings
894}
895
896// parseAnnotationsFromDelta parses annotations from the raw JSON of a delta.
897func parseAnnotationsFromDelta(delta openai.ChatCompletionChunkChoiceDelta) []openai.ChatCompletionMessageAnnotation {
898 var annotations []openai.ChatCompletionMessageAnnotation
899
900 // Parse the raw JSON to extract annotations
901 var deltaData map[string]any
902 if err := json.Unmarshal([]byte(delta.RawJSON()), &deltaData); err != nil {
903 return annotations
904 }
905
906 // Check if annotations exist in the delta
907 if annotationsData, ok := deltaData["annotations"].([]any); ok {
908 for _, annotationData := range annotationsData {
909 if annotationMap, ok := annotationData.(map[string]any); ok {
910 if annotationType, ok := annotationMap["type"].(string); ok && annotationType == "url_citation" {
911 if urlCitationData, ok := annotationMap["url_citation"].(map[string]any); ok {
912 annotation := openai.ChatCompletionMessageAnnotation{
913 Type: "url_citation",
914 URLCitation: openai.ChatCompletionMessageAnnotationURLCitation{
915 URL: urlCitationData["url"].(string),
916 Title: urlCitationData["title"].(string),
917 },
918 }
919 annotations = append(annotations, annotation)
920 }
921 }
922 }
923 }
924 }
925
926 return annotations
927}