1package openai
2
3import (
4 "context"
5 "encoding/base64"
6 "encoding/json"
7 "errors"
8 "fmt"
9 "io"
10 "strings"
11
12 "charm.land/fantasy"
13 xjson "github.com/charmbracelet/x/json"
14 "github.com/google/uuid"
15 "github.com/openai/openai-go/v2"
16 "github.com/openai/openai-go/v2/packages/param"
17 "github.com/openai/openai-go/v2/shared"
18)
19
20type languageModel struct {
21 provider string
22 modelID string
23 client openai.Client
24 prepareCallFunc LanguageModelPrepareCallFunc
25 mapFinishReasonFunc LanguageModelMapFinishReasonFunc
26 extraContentFunc LanguageModelExtraContentFunc
27 usageFunc LanguageModelUsageFunc
28 streamUsageFunc LanguageModelStreamUsageFunc
29 streamExtraFunc LanguageModelStreamExtraFunc
30 streamProviderMetadataFunc LanguageModelStreamProviderMetadataFunc
31}
32
33// LanguageModelOption is a function that configures a languageModel.
34type LanguageModelOption = func(*languageModel)
35
36// WithLanguageModelPrepareCallFunc sets the prepare call function for the language model.
37func WithLanguageModelPrepareCallFunc(fn LanguageModelPrepareCallFunc) LanguageModelOption {
38 return func(l *languageModel) {
39 l.prepareCallFunc = fn
40 }
41}
42
43// WithLanguageModelMapFinishReasonFunc sets the map finish reason function for the language model.
44func WithLanguageModelMapFinishReasonFunc(fn LanguageModelMapFinishReasonFunc) LanguageModelOption {
45 return func(l *languageModel) {
46 l.mapFinishReasonFunc = fn
47 }
48}
49
50// WithLanguageModelExtraContentFunc sets the extra content function for the language model.
51func WithLanguageModelExtraContentFunc(fn LanguageModelExtraContentFunc) LanguageModelOption {
52 return func(l *languageModel) {
53 l.extraContentFunc = fn
54 }
55}
56
57// WithLanguageModelStreamExtraFunc sets the stream extra function for the language model.
58func WithLanguageModelStreamExtraFunc(fn LanguageModelStreamExtraFunc) LanguageModelOption {
59 return func(l *languageModel) {
60 l.streamExtraFunc = fn
61 }
62}
63
64// WithLanguageModelUsageFunc sets the usage function for the language model.
65func WithLanguageModelUsageFunc(fn LanguageModelUsageFunc) LanguageModelOption {
66 return func(l *languageModel) {
67 l.usageFunc = fn
68 }
69}
70
71// WithLanguageModelStreamUsageFunc sets the stream usage function for the language model.
72func WithLanguageModelStreamUsageFunc(fn LanguageModelStreamUsageFunc) LanguageModelOption {
73 return func(l *languageModel) {
74 l.streamUsageFunc = fn
75 }
76}
77
78func newLanguageModel(modelID string, provider string, client openai.Client, opts ...LanguageModelOption) languageModel {
79 model := languageModel{
80 modelID: modelID,
81 provider: provider,
82 client: client,
83 prepareCallFunc: DefaultPrepareCallFunc,
84 mapFinishReasonFunc: DefaultMapFinishReasonFunc,
85 usageFunc: DefaultUsageFunc,
86 streamUsageFunc: DefaultStreamUsageFunc,
87 streamProviderMetadataFunc: DefaultStreamProviderMetadataFunc,
88 }
89
90 for _, o := range opts {
91 o(&model)
92 }
93 return model
94}
95
96type streamToolCall struct {
97 id string
98 name string
99 arguments string
100 hasFinished bool
101}
102
103// Model implements fantasy.LanguageModel.
104func (o languageModel) Model() string {
105 return o.modelID
106}
107
108// Provider implements fantasy.LanguageModel.
109func (o languageModel) Provider() string {
110 return o.provider
111}
112
113func (o languageModel) prepareParams(call fantasy.Call) (*openai.ChatCompletionNewParams, []fantasy.CallWarning, error) {
114 params := &openai.ChatCompletionNewParams{}
115 messages, warnings := toPrompt(call.Prompt)
116 if call.TopK != nil {
117 warnings = append(warnings, fantasy.CallWarning{
118 Type: fantasy.CallWarningTypeUnsupportedSetting,
119 Setting: "top_k",
120 })
121 }
122
123 if call.MaxOutputTokens != nil {
124 params.MaxTokens = param.NewOpt(*call.MaxOutputTokens)
125 }
126 if call.Temperature != nil {
127 params.Temperature = param.NewOpt(*call.Temperature)
128 }
129 if call.TopP != nil {
130 params.TopP = param.NewOpt(*call.TopP)
131 }
132 if call.FrequencyPenalty != nil {
133 params.FrequencyPenalty = param.NewOpt(*call.FrequencyPenalty)
134 }
135 if call.PresencePenalty != nil {
136 params.PresencePenalty = param.NewOpt(*call.PresencePenalty)
137 }
138
139 if isReasoningModel(o.modelID) {
140 // remove unsupported settings for reasoning models
141 // see https://platform.openai.com/docs/guides/reasoning#limitations
142 if call.Temperature != nil {
143 params.Temperature = param.Opt[float64]{}
144 warnings = append(warnings, fantasy.CallWarning{
145 Type: fantasy.CallWarningTypeUnsupportedSetting,
146 Setting: "temperature",
147 Details: "temperature is not supported for reasoning models",
148 })
149 }
150 if call.TopP != nil {
151 params.TopP = param.Opt[float64]{}
152 warnings = append(warnings, fantasy.CallWarning{
153 Type: fantasy.CallWarningTypeUnsupportedSetting,
154 Setting: "TopP",
155 Details: "TopP is not supported for reasoning models",
156 })
157 }
158 if call.FrequencyPenalty != nil {
159 params.FrequencyPenalty = param.Opt[float64]{}
160 warnings = append(warnings, fantasy.CallWarning{
161 Type: fantasy.CallWarningTypeUnsupportedSetting,
162 Setting: "FrequencyPenalty",
163 Details: "FrequencyPenalty is not supported for reasoning models",
164 })
165 }
166 if call.PresencePenalty != nil {
167 params.PresencePenalty = param.Opt[float64]{}
168 warnings = append(warnings, fantasy.CallWarning{
169 Type: fantasy.CallWarningTypeUnsupportedSetting,
170 Setting: "PresencePenalty",
171 Details: "PresencePenalty is not supported for reasoning models",
172 })
173 }
174
175 // reasoning models use max_completion_tokens instead of max_tokens
176 if call.MaxOutputTokens != nil {
177 if !params.MaxCompletionTokens.Valid() {
178 params.MaxCompletionTokens = param.NewOpt(*call.MaxOutputTokens)
179 }
180 params.MaxTokens = param.Opt[int64]{}
181 }
182 }
183
184 // Handle search preview models
185 if isSearchPreviewModel(o.modelID) {
186 if call.Temperature != nil {
187 params.Temperature = param.Opt[float64]{}
188 warnings = append(warnings, fantasy.CallWarning{
189 Type: fantasy.CallWarningTypeUnsupportedSetting,
190 Setting: "temperature",
191 Details: "temperature is not supported for the search preview models and has been removed.",
192 })
193 }
194 }
195
196 optionsWarnings, err := o.prepareCallFunc(o, params, call)
197 if err != nil {
198 return nil, nil, err
199 }
200
201 if len(optionsWarnings) > 0 {
202 warnings = append(warnings, optionsWarnings...)
203 }
204
205 params.Messages = messages
206 params.Model = o.modelID
207
208 if len(call.Tools) > 0 {
209 tools, toolChoice, toolWarnings := toOpenAiTools(call.Tools, call.ToolChoice)
210 params.Tools = tools
211 if toolChoice != nil {
212 params.ToolChoice = *toolChoice
213 }
214 warnings = append(warnings, toolWarnings...)
215 }
216 return params, warnings, nil
217}
218
219func (o languageModel) handleError(err error) error {
220 var apiErr *openai.Error
221 if errors.As(err, &apiErr) {
222 requestDump := apiErr.DumpRequest(true)
223 responseDump := apiErr.DumpResponse(true)
224 headers := map[string]string{}
225 for k, h := range apiErr.Response.Header {
226 v := h[len(h)-1]
227 headers[strings.ToLower(k)] = v
228 }
229 return fantasy.NewAPICallError(
230 apiErr.Message,
231 apiErr.Request.URL.String(),
232 string(requestDump),
233 apiErr.StatusCode,
234 headers,
235 string(responseDump),
236 apiErr,
237 false,
238 )
239 }
240 return err
241}
242
243// Generate implements fantasy.LanguageModel.
244func (o languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) {
245 params, warnings, err := o.prepareParams(call)
246 if err != nil {
247 return nil, err
248 }
249 response, err := o.client.Chat.Completions.New(ctx, *params)
250 if err != nil {
251 return nil, o.handleError(err)
252 }
253
254 if len(response.Choices) == 0 {
255 return nil, errors.New("no response generated")
256 }
257 choice := response.Choices[0]
258 content := make([]fantasy.Content, 0, 1+len(choice.Message.ToolCalls)+len(choice.Message.Annotations))
259 text := choice.Message.Content
260 if text != "" {
261 content = append(content, fantasy.TextContent{
262 Text: text,
263 })
264 }
265 if o.extraContentFunc != nil {
266 extraContent := o.extraContentFunc(choice)
267 content = append(content, extraContent...)
268 }
269 for _, tc := range choice.Message.ToolCalls {
270 toolCallID := tc.ID
271 content = append(content, fantasy.ToolCallContent{
272 ProviderExecuted: false, // TODO: update when handling other tools
273 ToolCallID: toolCallID,
274 ToolName: tc.Function.Name,
275 Input: tc.Function.Arguments,
276 })
277 }
278 // Handle annotations/citations
279 for _, annotation := range choice.Message.Annotations {
280 if annotation.Type == "url_citation" {
281 content = append(content, fantasy.SourceContent{
282 SourceType: fantasy.SourceTypeURL,
283 ID: uuid.NewString(),
284 URL: annotation.URLCitation.URL,
285 Title: annotation.URLCitation.Title,
286 })
287 }
288 }
289
290 usage, providerMetadata := o.usageFunc(*response)
291
292 mappedFinishReason := o.mapFinishReasonFunc(choice.FinishReason)
293 if len(choice.Message.ToolCalls) > 0 {
294 mappedFinishReason = fantasy.FinishReasonToolCalls
295 }
296 return &fantasy.Response{
297 Content: content,
298 Usage: usage,
299 FinishReason: mappedFinishReason,
300 ProviderMetadata: fantasy.ProviderMetadata{
301 Name: providerMetadata,
302 },
303 Warnings: warnings,
304 }, nil
305}
306
307// Stream implements fantasy.LanguageModel.
308func (o languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
309 params, warnings, err := o.prepareParams(call)
310 if err != nil {
311 return nil, err
312 }
313
314 params.StreamOptions = openai.ChatCompletionStreamOptionsParam{
315 IncludeUsage: openai.Bool(true),
316 }
317
318 stream := o.client.Chat.Completions.NewStreaming(ctx, *params)
319 isActiveText := false
320 toolCalls := make(map[int64]streamToolCall)
321
322 // Build provider metadata for streaming
323 providerMetadata := fantasy.ProviderMetadata{
324 Name: &ProviderMetadata{},
325 }
326 acc := openai.ChatCompletionAccumulator{}
327 extraContext := make(map[string]any)
328 var usage fantasy.Usage
329 var finishReason string
330 return func(yield func(fantasy.StreamPart) bool) {
331 if len(warnings) > 0 {
332 if !yield(fantasy.StreamPart{
333 Type: fantasy.StreamPartTypeWarnings,
334 Warnings: warnings,
335 }) {
336 return
337 }
338 }
339 for stream.Next() {
340 chunk := stream.Current()
341 acc.AddChunk(chunk)
342 usage, providerMetadata = o.streamUsageFunc(chunk, extraContext, providerMetadata)
343 if len(chunk.Choices) == 0 {
344 continue
345 }
346 for _, choice := range chunk.Choices {
347 if choice.FinishReason != "" {
348 finishReason = choice.FinishReason
349 }
350 switch {
351 case choice.Delta.Content != "":
352 if !isActiveText {
353 isActiveText = true
354 if !yield(fantasy.StreamPart{
355 Type: fantasy.StreamPartTypeTextStart,
356 ID: "0",
357 }) {
358 return
359 }
360 }
361 if !yield(fantasy.StreamPart{
362 Type: fantasy.StreamPartTypeTextDelta,
363 ID: "0",
364 Delta: choice.Delta.Content,
365 }) {
366 return
367 }
368 case len(choice.Delta.ToolCalls) > 0:
369 if isActiveText {
370 isActiveText = false
371 if !yield(fantasy.StreamPart{
372 Type: fantasy.StreamPartTypeTextEnd,
373 ID: "0",
374 }) {
375 return
376 }
377 }
378
379 for _, toolCallDelta := range choice.Delta.ToolCalls {
380 if existingToolCall, ok := toolCalls[toolCallDelta.Index]; ok {
381 if existingToolCall.hasFinished {
382 continue
383 }
384 if toolCallDelta.Function.Arguments != "" {
385 existingToolCall.arguments += toolCallDelta.Function.Arguments
386 }
387 if !yield(fantasy.StreamPart{
388 Type: fantasy.StreamPartTypeToolInputDelta,
389 ID: existingToolCall.id,
390 Delta: toolCallDelta.Function.Arguments,
391 }) {
392 return
393 }
394 toolCalls[toolCallDelta.Index] = existingToolCall
395 if xjson.IsValid(existingToolCall.arguments) {
396 if !yield(fantasy.StreamPart{
397 Type: fantasy.StreamPartTypeToolInputEnd,
398 ID: existingToolCall.id,
399 }) {
400 return
401 }
402
403 if !yield(fantasy.StreamPart{
404 Type: fantasy.StreamPartTypeToolCall,
405 ID: existingToolCall.id,
406 ToolCallName: existingToolCall.name,
407 ToolCallInput: existingToolCall.arguments,
408 }) {
409 return
410 }
411 existingToolCall.hasFinished = true
412 toolCalls[toolCallDelta.Index] = existingToolCall
413 }
414 } else {
415 // Does not exist
416 var err error
417 if toolCallDelta.Type != "function" {
418 err = fantasy.NewInvalidResponseDataError(toolCallDelta, "Expected 'function' type.")
419 }
420 if toolCallDelta.ID == "" {
421 err = fantasy.NewInvalidResponseDataError(toolCallDelta, "Expected 'id' to be a string.")
422 }
423 if toolCallDelta.Function.Name == "" {
424 err = fantasy.NewInvalidResponseDataError(toolCallDelta, "Expected 'function.name' to be a string.")
425 }
426 if err != nil {
427 yield(fantasy.StreamPart{
428 Type: fantasy.StreamPartTypeError,
429 Error: o.handleError(stream.Err()),
430 })
431 return
432 }
433
434 if !yield(fantasy.StreamPart{
435 Type: fantasy.StreamPartTypeToolInputStart,
436 ID: toolCallDelta.ID,
437 ToolCallName: toolCallDelta.Function.Name,
438 }) {
439 return
440 }
441 toolCalls[toolCallDelta.Index] = streamToolCall{
442 id: toolCallDelta.ID,
443 name: toolCallDelta.Function.Name,
444 arguments: toolCallDelta.Function.Arguments,
445 }
446
447 exTc := toolCalls[toolCallDelta.Index]
448 if exTc.arguments != "" {
449 if !yield(fantasy.StreamPart{
450 Type: fantasy.StreamPartTypeToolInputDelta,
451 ID: exTc.id,
452 Delta: exTc.arguments,
453 }) {
454 return
455 }
456 if xjson.IsValid(toolCalls[toolCallDelta.Index].arguments) {
457 if !yield(fantasy.StreamPart{
458 Type: fantasy.StreamPartTypeToolInputEnd,
459 ID: toolCallDelta.ID,
460 }) {
461 return
462 }
463
464 if !yield(fantasy.StreamPart{
465 Type: fantasy.StreamPartTypeToolCall,
466 ID: exTc.id,
467 ToolCallName: exTc.name,
468 ToolCallInput: exTc.arguments,
469 }) {
470 return
471 }
472 exTc.hasFinished = true
473 toolCalls[toolCallDelta.Index] = exTc
474 }
475 }
476 continue
477 }
478 }
479 }
480
481 if o.streamExtraFunc != nil {
482 updatedContext, shouldContinue := o.streamExtraFunc(chunk, yield, extraContext)
483 if !shouldContinue {
484 return
485 }
486 extraContext = updatedContext
487 }
488 }
489
490 // Check for annotations in the delta's raw JSON
491 for _, choice := range chunk.Choices {
492 if annotations := parseAnnotationsFromDelta(choice.Delta); len(annotations) > 0 {
493 for _, annotation := range annotations {
494 if annotation.Type == "url_citation" {
495 if !yield(fantasy.StreamPart{
496 Type: fantasy.StreamPartTypeSource,
497 ID: uuid.NewString(),
498 SourceType: fantasy.SourceTypeURL,
499 URL: annotation.URLCitation.URL,
500 Title: annotation.URLCitation.Title,
501 }) {
502 return
503 }
504 }
505 }
506 }
507 }
508 }
509 err := stream.Err()
510 if err == nil || errors.Is(err, io.EOF) {
511 // finished
512 if isActiveText {
513 isActiveText = false
514 if !yield(fantasy.StreamPart{
515 Type: fantasy.StreamPartTypeTextEnd,
516 ID: "0",
517 }) {
518 return
519 }
520 }
521
522 if len(acc.Choices) > 0 {
523 choice := acc.Choices[0]
524 // Add logprobs if available
525 providerMetadata = o.streamProviderMetadataFunc(choice, providerMetadata)
526
527 // Handle annotations/citations from accumulated response
528 for _, annotation := range choice.Message.Annotations {
529 if annotation.Type == "url_citation" {
530 if !yield(fantasy.StreamPart{
531 Type: fantasy.StreamPartTypeSource,
532 ID: acc.ID,
533 SourceType: fantasy.SourceTypeURL,
534 URL: annotation.URLCitation.URL,
535 Title: annotation.URLCitation.Title,
536 }) {
537 return
538 }
539 }
540 }
541 }
542 mappedFinishReason := o.mapFinishReasonFunc(finishReason)
543 if len(acc.Choices) > 0 {
544 choice := acc.Choices[0]
545 if len(choice.Message.ToolCalls) > 0 {
546 mappedFinishReason = fantasy.FinishReasonToolCalls
547 }
548 }
549 yield(fantasy.StreamPart{
550 Type: fantasy.StreamPartTypeFinish,
551 Usage: usage,
552 FinishReason: mappedFinishReason,
553 ProviderMetadata: providerMetadata,
554 })
555 return
556 } else { //nolint: revive
557 yield(fantasy.StreamPart{
558 Type: fantasy.StreamPartTypeError,
559 Error: o.handleError(err),
560 })
561 return
562 }
563 }, nil
564}
565
566func isReasoningModel(modelID string) bool {
567 return strings.HasPrefix(modelID, "o") || strings.HasPrefix(modelID, "gpt-5") || strings.HasPrefix(modelID, "gpt-5-chat")
568}
569
570func isSearchPreviewModel(modelID string) bool {
571 return strings.Contains(modelID, "search-preview")
572}
573
574func supportsFlexProcessing(modelID string) bool {
575 return strings.HasPrefix(modelID, "o3") || strings.HasPrefix(modelID, "o4-mini") || strings.HasPrefix(modelID, "gpt-5")
576}
577
578func supportsPriorityProcessing(modelID string) bool {
579 return strings.HasPrefix(modelID, "gpt-4") || strings.HasPrefix(modelID, "gpt-5") ||
580 strings.HasPrefix(modelID, "gpt-5-mini") || strings.HasPrefix(modelID, "o3") ||
581 strings.HasPrefix(modelID, "o4-mini")
582}
583
584func toOpenAiTools(tools []fantasy.Tool, toolChoice *fantasy.ToolChoice) (openAiTools []openai.ChatCompletionToolUnionParam, openAiToolChoice *openai.ChatCompletionToolChoiceOptionUnionParam, warnings []fantasy.CallWarning) {
585 for _, tool := range tools {
586 if tool.GetType() == fantasy.ToolTypeFunction {
587 ft, ok := tool.(fantasy.FunctionTool)
588 if !ok {
589 continue
590 }
591 openAiTools = append(openAiTools, openai.ChatCompletionToolUnionParam{
592 OfFunction: &openai.ChatCompletionFunctionToolParam{
593 Function: shared.FunctionDefinitionParam{
594 Name: ft.Name,
595 Description: param.NewOpt(ft.Description),
596 Parameters: openai.FunctionParameters(ft.InputSchema),
597 Strict: param.NewOpt(false),
598 },
599 Type: "function",
600 },
601 })
602 continue
603 }
604
605 // TODO: handle provider tool calls
606 warnings = append(warnings, fantasy.CallWarning{
607 Type: fantasy.CallWarningTypeUnsupportedTool,
608 Tool: tool,
609 Message: "tool is not supported",
610 })
611 }
612 if toolChoice == nil {
613 return openAiTools, openAiToolChoice, warnings
614 }
615
616 switch *toolChoice {
617 case fantasy.ToolChoiceAuto:
618 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
619 OfAuto: param.NewOpt("auto"),
620 }
621 case fantasy.ToolChoiceNone:
622 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
623 OfAuto: param.NewOpt("none"),
624 }
625 default:
626 openAiToolChoice = &openai.ChatCompletionToolChoiceOptionUnionParam{
627 OfFunctionToolChoice: &openai.ChatCompletionNamedToolChoiceParam{
628 Type: "function",
629 Function: openai.ChatCompletionNamedToolChoiceFunctionParam{
630 Name: string(*toolChoice),
631 },
632 },
633 }
634 }
635 return openAiTools, openAiToolChoice, warnings
636}
637
638func toPrompt(prompt fantasy.Prompt) ([]openai.ChatCompletionMessageParamUnion, []fantasy.CallWarning) {
639 var messages []openai.ChatCompletionMessageParamUnion
640 var warnings []fantasy.CallWarning
641 for _, msg := range prompt {
642 switch msg.Role {
643 case fantasy.MessageRoleSystem:
644 var systemPromptParts []string
645 for _, c := range msg.Content {
646 if c.GetType() != fantasy.ContentTypeText {
647 warnings = append(warnings, fantasy.CallWarning{
648 Type: fantasy.CallWarningTypeOther,
649 Message: "system prompt can only have text content",
650 })
651 continue
652 }
653 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
654 if !ok {
655 warnings = append(warnings, fantasy.CallWarning{
656 Type: fantasy.CallWarningTypeOther,
657 Message: "system prompt text part does not have the right type",
658 })
659 continue
660 }
661 text := textPart.Text
662 if strings.TrimSpace(text) != "" {
663 systemPromptParts = append(systemPromptParts, textPart.Text)
664 }
665 }
666 if len(systemPromptParts) == 0 {
667 warnings = append(warnings, fantasy.CallWarning{
668 Type: fantasy.CallWarningTypeOther,
669 Message: "system prompt has no text parts",
670 })
671 continue
672 }
673 messages = append(messages, openai.SystemMessage(strings.Join(systemPromptParts, "\n")))
674 case fantasy.MessageRoleUser:
675 // simple user message just text content
676 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
677 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
678 if !ok {
679 warnings = append(warnings, fantasy.CallWarning{
680 Type: fantasy.CallWarningTypeOther,
681 Message: "user message text part does not have the right type",
682 })
683 continue
684 }
685 messages = append(messages, openai.UserMessage(textPart.Text))
686 continue
687 }
688 // text content and attachments
689 // for now we only support image content later we need to check
690 // TODO: add the supported media types to the language model so we
691 // can use that to validate the data here.
692 var content []openai.ChatCompletionContentPartUnionParam
693 for _, c := range msg.Content {
694 switch c.GetType() {
695 case fantasy.ContentTypeText:
696 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
697 if !ok {
698 warnings = append(warnings, fantasy.CallWarning{
699 Type: fantasy.CallWarningTypeOther,
700 Message: "user message text part does not have the right type",
701 })
702 continue
703 }
704 content = append(content, openai.ChatCompletionContentPartUnionParam{
705 OfText: &openai.ChatCompletionContentPartTextParam{
706 Text: textPart.Text,
707 },
708 })
709 case fantasy.ContentTypeFile:
710 filePart, ok := fantasy.AsContentType[fantasy.FilePart](c)
711 if !ok {
712 warnings = append(warnings, fantasy.CallWarning{
713 Type: fantasy.CallWarningTypeOther,
714 Message: "user message file part does not have the right type",
715 })
716 continue
717 }
718
719 switch {
720 case strings.HasPrefix(filePart.MediaType, "image/"):
721 // Handle image files
722 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
723 data := "data:" + filePart.MediaType + ";base64," + base64Encoded
724 imageURL := openai.ChatCompletionContentPartImageImageURLParam{URL: data}
725
726 // Check for provider-specific options like image detail
727 if providerOptions, ok := filePart.ProviderOptions[Name]; ok {
728 if detail, ok := providerOptions.(*ProviderFileOptions); ok {
729 imageURL.Detail = detail.ImageDetail
730 }
731 }
732
733 imageBlock := openai.ChatCompletionContentPartImageParam{ImageURL: imageURL}
734 content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
735
736 case filePart.MediaType == "audio/wav":
737 // Handle WAV audio files
738 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
739 audioBlock := openai.ChatCompletionContentPartInputAudioParam{
740 InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{
741 Data: base64Encoded,
742 Format: "wav",
743 },
744 }
745 content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
746
747 case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3":
748 // Handle MP3 audio files
749 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
750 audioBlock := openai.ChatCompletionContentPartInputAudioParam{
751 InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{
752 Data: base64Encoded,
753 Format: "mp3",
754 },
755 }
756 content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
757
758 case filePart.MediaType == "application/pdf":
759 // Handle PDF files
760 dataStr := string(filePart.Data)
761
762 // Check if data looks like a file ID (starts with "file-")
763 if strings.HasPrefix(dataStr, "file-") {
764 fileBlock := openai.ChatCompletionContentPartFileParam{
765 File: openai.ChatCompletionContentPartFileFileParam{
766 FileID: param.NewOpt(dataStr),
767 },
768 }
769 content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
770 } else {
771 // Handle as base64 data
772 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
773 data := "data:application/pdf;base64," + base64Encoded
774
775 filename := filePart.Filename
776 if filename == "" {
777 // Generate default filename based on content index
778 filename = fmt.Sprintf("part-%d.pdf", len(content))
779 }
780
781 fileBlock := openai.ChatCompletionContentPartFileParam{
782 File: openai.ChatCompletionContentPartFileFileParam{
783 Filename: param.NewOpt(filename),
784 FileData: param.NewOpt(data),
785 },
786 }
787 content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
788 }
789
790 default:
791 warnings = append(warnings, fantasy.CallWarning{
792 Type: fantasy.CallWarningTypeOther,
793 Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType),
794 })
795 }
796 }
797 }
798 messages = append(messages, openai.UserMessage(content))
799 case fantasy.MessageRoleAssistant:
800 // simple assistant message just text content
801 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
802 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
803 if !ok {
804 warnings = append(warnings, fantasy.CallWarning{
805 Type: fantasy.CallWarningTypeOther,
806 Message: "assistant message text part does not have the right type",
807 })
808 continue
809 }
810 messages = append(messages, openai.AssistantMessage(textPart.Text))
811 continue
812 }
813 assistantMsg := openai.ChatCompletionAssistantMessageParam{
814 Role: "assistant",
815 }
816 for _, c := range msg.Content {
817 switch c.GetType() {
818 case fantasy.ContentTypeText:
819 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
820 if !ok {
821 warnings = append(warnings, fantasy.CallWarning{
822 Type: fantasy.CallWarningTypeOther,
823 Message: "assistant message text part does not have the right type",
824 })
825 continue
826 }
827 assistantMsg.Content = openai.ChatCompletionAssistantMessageParamContentUnion{
828 OfString: param.NewOpt(textPart.Text),
829 }
830 case fantasy.ContentTypeToolCall:
831 toolCallPart, ok := fantasy.AsContentType[fantasy.ToolCallPart](c)
832 if !ok {
833 warnings = append(warnings, fantasy.CallWarning{
834 Type: fantasy.CallWarningTypeOther,
835 Message: "assistant message tool part does not have the right type",
836 })
837 continue
838 }
839 assistantMsg.ToolCalls = append(assistantMsg.ToolCalls,
840 openai.ChatCompletionMessageToolCallUnionParam{
841 OfFunction: &openai.ChatCompletionMessageFunctionToolCallParam{
842 ID: toolCallPart.ToolCallID,
843 Type: "function",
844 Function: openai.ChatCompletionMessageFunctionToolCallFunctionParam{
845 Name: toolCallPart.ToolName,
846 Arguments: toolCallPart.Input,
847 },
848 },
849 })
850 }
851 }
852 messages = append(messages, openai.ChatCompletionMessageParamUnion{
853 OfAssistant: &assistantMsg,
854 })
855 case fantasy.MessageRoleTool:
856 for _, c := range msg.Content {
857 if c.GetType() != fantasy.ContentTypeToolResult {
858 warnings = append(warnings, fantasy.CallWarning{
859 Type: fantasy.CallWarningTypeOther,
860 Message: "tool message can only have tool result content",
861 })
862 continue
863 }
864
865 toolResultPart, ok := fantasy.AsContentType[fantasy.ToolResultPart](c)
866 if !ok {
867 warnings = append(warnings, fantasy.CallWarning{
868 Type: fantasy.CallWarningTypeOther,
869 Message: "tool message result part does not have the right type",
870 })
871 continue
872 }
873
874 switch toolResultPart.Output.GetType() {
875 case fantasy.ToolResultContentTypeText:
876 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](toolResultPart.Output)
877 if !ok {
878 warnings = append(warnings, fantasy.CallWarning{
879 Type: fantasy.CallWarningTypeOther,
880 Message: "tool result output does not have the right type",
881 })
882 continue
883 }
884 messages = append(messages, openai.ToolMessage(output.Text, toolResultPart.ToolCallID))
885 case fantasy.ToolResultContentTypeError:
886 // TODO: check if better handling is needed
887 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentError](toolResultPart.Output)
888 if !ok {
889 warnings = append(warnings, fantasy.CallWarning{
890 Type: fantasy.CallWarningTypeOther,
891 Message: "tool result output does not have the right type",
892 })
893 continue
894 }
895 messages = append(messages, openai.ToolMessage(output.Error.Error(), toolResultPart.ToolCallID))
896 }
897 }
898 }
899 }
900 return messages, warnings
901}
902
903// parseAnnotationsFromDelta parses annotations from the raw JSON of a delta.
904func parseAnnotationsFromDelta(delta openai.ChatCompletionChunkChoiceDelta) []openai.ChatCompletionMessageAnnotation {
905 var annotations []openai.ChatCompletionMessageAnnotation
906
907 // Parse the raw JSON to extract annotations
908 var deltaData map[string]any
909 if err := json.Unmarshal([]byte(delta.RawJSON()), &deltaData); err != nil {
910 return annotations
911 }
912
913 // Check if annotations exist in the delta
914 if annotationsData, ok := deltaData["annotations"].([]any); ok {
915 for _, annotationData := range annotationsData {
916 if annotationMap, ok := annotationData.(map[string]any); ok {
917 if annotationType, ok := annotationMap["type"].(string); ok && annotationType == "url_citation" {
918 if urlCitationData, ok := annotationMap["url_citation"].(map[string]any); ok {
919 annotation := openai.ChatCompletionMessageAnnotation{
920 Type: "url_citation",
921 URLCitation: openai.ChatCompletionMessageAnnotationURLCitation{
922 URL: urlCitationData["url"].(string),
923 Title: urlCitationData["title"].(string),
924 },
925 }
926 annotations = append(annotations, annotation)
927 }
928 }
929 }
930 }
931 }
932
933 return annotations
934}