1package openrouter
2
3import (
4 "encoding/base64"
5 "encoding/json"
6 "fmt"
7 "maps"
8 "strings"
9
10 "charm.land/fantasy"
11 "charm.land/fantasy/providers/anthropic"
12 "charm.land/fantasy/providers/google"
13 "charm.land/fantasy/providers/openai"
14 openaisdk "github.com/charmbracelet/openai-go"
15 "github.com/charmbracelet/openai-go/packages/param"
16 xstrings "github.com/charmbracelet/x/exp/strings"
17)
18
19const reasoningStartedCtx = "reasoning_started"
20
21func languagePrepareModelCall(_ fantasy.LanguageModel, params *openaisdk.ChatCompletionNewParams, call fantasy.Call) ([]fantasy.CallWarning, error) {
22 providerOptions := &ProviderOptions{}
23 if v, ok := call.ProviderOptions[Name]; ok {
24 providerOptions, ok = v.(*ProviderOptions)
25 if !ok {
26 return nil, &fantasy.Error{Title: "invalid argument", Message: "openrouter provider options should be *openrouter.ProviderOptions"}
27 }
28 }
29
30 extraFields := make(map[string]any)
31
32 if providerOptions.Provider != nil {
33 data, err := structToMapJSON(providerOptions.Provider)
34 if err != nil {
35 return nil, err
36 }
37 extraFields["provider"] = data
38 }
39
40 if providerOptions.Reasoning != nil {
41 data, err := structToMapJSON(providerOptions.Reasoning)
42 if err != nil {
43 return nil, err
44 }
45 extraFields["reasoning"] = data
46 }
47
48 if providerOptions.IncludeUsage != nil {
49 extraFields["usage"] = map[string]any{
50 "include": *providerOptions.IncludeUsage,
51 }
52 } else { // default include usage
53 extraFields["usage"] = map[string]any{
54 "include": true,
55 }
56 }
57 if providerOptions.LogitBias != nil {
58 params.LogitBias = providerOptions.LogitBias
59 }
60 if providerOptions.LogProbs != nil {
61 params.Logprobs = param.NewOpt(*providerOptions.LogProbs)
62 }
63 if providerOptions.User != nil {
64 params.User = param.NewOpt(*providerOptions.User)
65 }
66 if providerOptions.ParallelToolCalls != nil {
67 params.ParallelToolCalls = param.NewOpt(*providerOptions.ParallelToolCalls)
68 }
69
70 maps.Copy(extraFields, providerOptions.ExtraBody)
71 params.SetExtraFields(extraFields)
72 return nil, nil
73}
74
75func languageModelExtraContent(choice openaisdk.ChatCompletionChoice) []fantasy.Content {
76 content := make([]fantasy.Content, 0)
77 reasoningData := ReasoningData{}
78 err := json.Unmarshal([]byte(choice.Message.RawJSON()), &reasoningData)
79 if err != nil {
80 return content
81 }
82 type anthropicReasoningBlock struct {
83 text string
84 metadata *anthropic.ReasoningOptionMetadata
85 }
86 type googleReasoningBlock struct {
87 text string
88 metadata *google.ReasoningMetadata
89 }
90
91 responsesReasoningBlocks := make([]openai.ResponsesReasoningMetadata, 0)
92 anthropicReasoningBlocks := make([]anthropicReasoningBlock, 0)
93 googleReasoningBlocks := make([]googleReasoningBlock, 0)
94 otherReasoning := make([]string, 0)
95 for _, detail := range reasoningData.ReasoningDetails {
96 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
97 var thinkingBlock openai.ResponsesReasoningMetadata
98 if len(responsesReasoningBlocks)-1 >= detail.Index {
99 thinkingBlock = responsesReasoningBlocks[detail.Index]
100 } else {
101 thinkingBlock = openai.ResponsesReasoningMetadata{}
102 responsesReasoningBlocks = append(responsesReasoningBlocks, thinkingBlock)
103 }
104
105 switch detail.Type {
106 case "reasoning.summary":
107 thinkingBlock.Summary = append(thinkingBlock.Summary, detail.Summary)
108 case "reasoning.encrypted":
109 thinkingBlock.EncryptedContent = &detail.Data
110 }
111 if detail.ID != "" {
112 thinkingBlock.ItemID = detail.ID
113 }
114
115 responsesReasoningBlocks[detail.Index] = thinkingBlock
116 continue
117 }
118 if strings.HasPrefix(detail.Format, "google-gemini") {
119 var thinkingBlock googleReasoningBlock
120 if len(googleReasoningBlocks)-1 >= detail.Index {
121 thinkingBlock = googleReasoningBlocks[detail.Index]
122 } else {
123 thinkingBlock = googleReasoningBlock{metadata: &google.ReasoningMetadata{}}
124 googleReasoningBlocks = append(googleReasoningBlocks, thinkingBlock)
125 }
126
127 switch detail.Type {
128 case "reasoning.text":
129 thinkingBlock.text = detail.Text
130 case "reasoning.encrypted":
131 thinkingBlock.metadata.Signature = detail.Data
132 thinkingBlock.metadata.ToolID = detail.ID
133 }
134
135 googleReasoningBlocks[detail.Index] = thinkingBlock
136 continue
137 }
138
139 if strings.HasPrefix(detail.Format, "anthropic-claude") {
140 anthropicReasoningBlocks = append(anthropicReasoningBlocks, anthropicReasoningBlock{
141 text: detail.Text,
142 metadata: &anthropic.ReasoningOptionMetadata{
143 Signature: detail.Signature,
144 },
145 })
146 continue
147 }
148
149 otherReasoning = append(otherReasoning, detail.Text)
150 }
151
152 for _, block := range responsesReasoningBlocks {
153 if len(block.Summary) == 0 {
154 block.Summary = []string{""}
155 }
156 content = append(content, fantasy.ReasoningContent{
157 Text: strings.Join(block.Summary, "\n"),
158 ProviderMetadata: fantasy.ProviderMetadata{
159 openai.Name: &block,
160 },
161 })
162 }
163 for _, block := range anthropicReasoningBlocks {
164 content = append(content, fantasy.ReasoningContent{
165 Text: block.text,
166 ProviderMetadata: fantasy.ProviderMetadata{
167 anthropic.Name: block.metadata,
168 },
169 })
170 }
171 for _, block := range googleReasoningBlocks {
172 content = append(content, fantasy.ReasoningContent{
173 Text: block.text,
174 ProviderMetadata: fantasy.ProviderMetadata{
175 google.Name: block.metadata,
176 },
177 })
178 }
179
180 for _, reasoning := range otherReasoning {
181 content = append(content, fantasy.ReasoningContent{
182 Text: reasoning,
183 })
184 }
185 return content
186}
187
188type currentReasoningState struct {
189 metadata *openai.ResponsesReasoningMetadata
190 googleMetadata *google.ReasoningMetadata
191 googleText string
192 format string
193}
194
195func extractReasoningContext(ctx map[string]any) *currentReasoningState {
196 reasoningStarted, ok := ctx[reasoningStartedCtx]
197 if !ok {
198 return nil
199 }
200 state, ok := reasoningStarted.(*currentReasoningState)
201 if !ok {
202 return nil
203 }
204 return state
205}
206
207func languageModelStreamExtra(chunk openaisdk.ChatCompletionChunk, yield func(fantasy.StreamPart) bool, ctx map[string]any) (map[string]any, bool) {
208 if len(chunk.Choices) == 0 {
209 return ctx, true
210 }
211
212 currentState := extractReasoningContext(ctx)
213
214 inx := 0
215 choice := chunk.Choices[inx]
216 reasoningData := ReasoningData{}
217 err := json.Unmarshal([]byte(choice.Delta.RawJSON()), &reasoningData)
218 if err != nil {
219 yield(fantasy.StreamPart{
220 Type: fantasy.StreamPartTypeError,
221 Error: &fantasy.Error{Title: "stream error", Message: "error unmarshalling delta", Cause: err},
222 })
223 return ctx, false
224 }
225
226 // Reasoning Start
227 if currentState == nil {
228 if len(reasoningData.ReasoningDetails) == 0 {
229 return ctx, true
230 }
231
232 var metadata fantasy.ProviderMetadata
233 currentState = ¤tReasoningState{}
234
235 detail := reasoningData.ReasoningDetails[0]
236 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
237 currentState.metadata = &openai.ResponsesReasoningMetadata{
238 Summary: []string{detail.Summary},
239 }
240 metadata = fantasy.ProviderMetadata{
241 openai.Name: currentState.metadata,
242 }
243 // There was no summary just thinking we just send this as if it ended alredy
244 if detail.Data != "" {
245 shouldContinue := yield(fantasy.StreamPart{
246 Type: fantasy.StreamPartTypeReasoningStart,
247 ID: fmt.Sprintf("%d", inx),
248 Delta: detail.Summary,
249 ProviderMetadata: metadata,
250 })
251 if !shouldContinue {
252 return ctx, false
253 }
254 return ctx, yield(fantasy.StreamPart{
255 Type: fantasy.StreamPartTypeReasoningEnd,
256 ID: fmt.Sprintf("%d", inx),
257 ProviderMetadata: fantasy.ProviderMetadata{
258 openai.Name: &openai.ResponsesReasoningMetadata{
259 Summary: []string{detail.Summary},
260 EncryptedContent: &detail.Data,
261 ItemID: detail.ID,
262 },
263 },
264 })
265 }
266 }
267
268 if strings.HasPrefix(detail.Format, "google-gemini") {
269 // this means there is only encrypted data available start and finish right away
270 if detail.Type == "reasoning.encrypted" {
271 ctx[reasoningStartedCtx] = nil
272 if !yield(fantasy.StreamPart{
273 Type: fantasy.StreamPartTypeReasoningStart,
274 ID: fmt.Sprintf("%d", inx),
275 }) {
276 return ctx, false
277 }
278 return ctx, yield(fantasy.StreamPart{
279 Type: fantasy.StreamPartTypeReasoningEnd,
280 ID: fmt.Sprintf("%d", inx),
281 ProviderMetadata: fantasy.ProviderMetadata{
282 google.Name: &google.ReasoningMetadata{
283 Signature: detail.Data,
284 ToolID: detail.ID,
285 },
286 },
287 })
288 }
289 currentState.googleMetadata = &google.ReasoningMetadata{}
290 currentState.googleText = detail.Text
291 metadata = fantasy.ProviderMetadata{
292 google.Name: currentState.googleMetadata,
293 }
294 }
295
296 currentState.format = detail.Format
297 ctx[reasoningStartedCtx] = currentState
298 delta := detail.Summary
299 if xstrings.ContainsAnyOf(detail.Format, "google-gemini", "anthropic-claude") {
300 delta = detail.Text
301 }
302 return ctx, yield(fantasy.StreamPart{
303 Type: fantasy.StreamPartTypeReasoningStart,
304 ID: fmt.Sprintf("%d", inx),
305 Delta: delta,
306 ProviderMetadata: metadata,
307 })
308 }
309 if len(reasoningData.ReasoningDetails) == 0 {
310 // Anthropic sends the signature after tool_calls, so don't end reasoning early
311 if strings.HasPrefix(currentState.format, "anthropic-claude") {
312 return ctx, true
313 }
314 // this means its a model different from openai/anthropic that ended reasoning
315 if choice.Delta.Content != "" || len(choice.Delta.ToolCalls) > 0 {
316 ctx[reasoningStartedCtx] = nil
317 return ctx, yield(fantasy.StreamPart{
318 Type: fantasy.StreamPartTypeReasoningEnd,
319 ID: fmt.Sprintf("%d", inx),
320 })
321 }
322 return ctx, true
323 }
324 // Reasoning delta
325 detail := reasoningData.ReasoningDetails[0]
326 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
327 // Reasoning has ended
328 if detail.Data != "" {
329 currentState.metadata.EncryptedContent = &detail.Data
330 currentState.metadata.ItemID = detail.ID
331 ctx[reasoningStartedCtx] = nil
332 return ctx, yield(fantasy.StreamPart{
333 Type: fantasy.StreamPartTypeReasoningEnd,
334 ID: fmt.Sprintf("%d", inx),
335 ProviderMetadata: fantasy.ProviderMetadata{
336 openai.Name: currentState.metadata,
337 },
338 })
339 }
340 var textDelta string
341 // add to existing summary
342 if len(currentState.metadata.Summary)-1 >= detail.Index {
343 currentState.metadata.Summary[detail.Index] += detail.Summary
344 textDelta = detail.Summary
345 } else { // add new summary
346 currentState.metadata.Summary = append(currentState.metadata.Summary, detail.Summary)
347 textDelta = "\n" + detail.Summary
348 }
349 ctx[reasoningStartedCtx] = currentState
350 return ctx, yield(fantasy.StreamPart{
351 Type: fantasy.StreamPartTypeReasoningDelta,
352 ID: fmt.Sprintf("%d", inx),
353 Delta: textDelta,
354 ProviderMetadata: fantasy.ProviderMetadata{
355 openai.Name: currentState.metadata,
356 },
357 })
358 }
359 if strings.HasPrefix(detail.Format, "anthropic-claude") {
360 // the reasoning has ended
361 if detail.Signature != "" {
362 metadata := fantasy.ProviderMetadata{
363 anthropic.Name: &anthropic.ReasoningOptionMetadata{
364 Signature: detail.Signature,
365 },
366 }
367 // initial update
368 shouldContinue := yield(fantasy.StreamPart{
369 Type: fantasy.StreamPartTypeReasoningDelta,
370 ID: fmt.Sprintf("%d", inx),
371 Delta: detail.Text,
372 ProviderMetadata: metadata,
373 })
374 if !shouldContinue {
375 return ctx, false
376 }
377 ctx[reasoningStartedCtx] = nil
378 return ctx, yield(fantasy.StreamPart{
379 Type: fantasy.StreamPartTypeReasoningEnd,
380 ID: fmt.Sprintf("%d", inx),
381 })
382 }
383
384 return ctx, yield(fantasy.StreamPart{
385 Type: fantasy.StreamPartTypeReasoningDelta,
386 ID: fmt.Sprintf("%d", inx),
387 Delta: detail.Text,
388 })
389 }
390
391 if strings.HasPrefix(detail.Format, "google-gemini") {
392 // reasoning.text type - accumulate text
393 if detail.Type == "reasoning.text" {
394 currentState.googleText += detail.Text
395 ctx[reasoningStartedCtx] = currentState
396 return ctx, yield(fantasy.StreamPart{
397 Type: fantasy.StreamPartTypeReasoningDelta,
398 ID: fmt.Sprintf("%d", inx),
399 Delta: detail.Text,
400 })
401 }
402
403 // reasoning.encrypted type - end reasoning with signature
404 if detail.Type == "reasoning.encrypted" {
405 currentState.googleMetadata.Signature = detail.Data
406 currentState.googleMetadata.ToolID = detail.ID
407 metadata := fantasy.ProviderMetadata{
408 google.Name: currentState.googleMetadata,
409 }
410 ctx[reasoningStartedCtx] = nil
411 return ctx, yield(fantasy.StreamPart{
412 Type: fantasy.StreamPartTypeReasoningEnd,
413 ID: fmt.Sprintf("%d", inx),
414 ProviderMetadata: metadata,
415 })
416 }
417 }
418
419 return ctx, yield(fantasy.StreamPart{
420 Type: fantasy.StreamPartTypeReasoningDelta,
421 ID: fmt.Sprintf("%d", inx),
422 Delta: detail.Text,
423 })
424}
425
426func languageModelUsage(response openaisdk.ChatCompletion) (fantasy.Usage, fantasy.ProviderOptionsData) {
427 if len(response.Choices) == 0 {
428 return fantasy.Usage{}, nil
429 }
430 openrouterUsage := UsageAccounting{}
431 usage := response.Usage
432
433 _ = json.Unmarshal([]byte(usage.RawJSON()), &openrouterUsage)
434
435 completionTokenDetails := usage.CompletionTokensDetails
436 promptTokenDetails := usage.PromptTokensDetails
437
438 var provider string
439 if p, ok := response.JSON.ExtraFields["provider"]; ok {
440 provider = p.Raw()
441 }
442
443 // Build provider metadata
444 providerMetadata := &ProviderMetadata{
445 Provider: provider,
446 Usage: openrouterUsage,
447 }
448
449 return fantasy.Usage{
450 InputTokens: usage.PromptTokens,
451 OutputTokens: usage.CompletionTokens,
452 TotalTokens: usage.TotalTokens,
453 ReasoningTokens: completionTokenDetails.ReasoningTokens,
454 CacheReadTokens: promptTokenDetails.CachedTokens,
455 }, providerMetadata
456}
457
458func languageModelStreamUsage(chunk openaisdk.ChatCompletionChunk, _ map[string]any, metadata fantasy.ProviderMetadata) (fantasy.Usage, fantasy.ProviderMetadata) {
459 usage := chunk.Usage
460 if usage.TotalTokens == 0 {
461 return fantasy.Usage{}, nil
462 }
463
464 streamProviderMetadata := &ProviderMetadata{}
465 if metadata != nil {
466 if providerMetadata, ok := metadata[Name]; ok {
467 converted, ok := providerMetadata.(*ProviderMetadata)
468 if ok {
469 streamProviderMetadata = converted
470 }
471 }
472 }
473 openrouterUsage := UsageAccounting{}
474 _ = json.Unmarshal([]byte(usage.RawJSON()), &openrouterUsage)
475 streamProviderMetadata.Usage = openrouterUsage
476
477 if p, ok := chunk.JSON.ExtraFields["provider"]; ok {
478 streamProviderMetadata.Provider = p.Raw()
479 }
480
481 // we do this here because the acc does not add prompt details
482 completionTokenDetails := usage.CompletionTokensDetails
483 promptTokenDetails := usage.PromptTokensDetails
484 aiUsage := fantasy.Usage{
485 InputTokens: usage.PromptTokens,
486 OutputTokens: usage.CompletionTokens,
487 TotalTokens: usage.TotalTokens,
488 ReasoningTokens: completionTokenDetails.ReasoningTokens,
489 CacheReadTokens: promptTokenDetails.CachedTokens,
490 }
491
492 return aiUsage, fantasy.ProviderMetadata{
493 Name: streamProviderMetadata,
494 }
495}
496
497func languageModelToPrompt(prompt fantasy.Prompt, _, model string) ([]openaisdk.ChatCompletionMessageParamUnion, []fantasy.CallWarning) {
498 var messages []openaisdk.ChatCompletionMessageParamUnion
499 var warnings []fantasy.CallWarning
500 for _, msg := range prompt {
501 switch msg.Role {
502 case fantasy.MessageRoleSystem:
503 var systemPromptParts []string
504 for _, c := range msg.Content {
505 if c.GetType() != fantasy.ContentTypeText {
506 warnings = append(warnings, fantasy.CallWarning{
507 Type: fantasy.CallWarningTypeOther,
508 Message: "system prompt can only have text content",
509 })
510 continue
511 }
512 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
513 if !ok {
514 warnings = append(warnings, fantasy.CallWarning{
515 Type: fantasy.CallWarningTypeOther,
516 Message: "system prompt text part does not have the right type",
517 })
518 continue
519 }
520 text := textPart.Text
521 if strings.TrimSpace(text) != "" {
522 systemPromptParts = append(systemPromptParts, textPart.Text)
523 }
524 }
525 if len(systemPromptParts) == 0 {
526 warnings = append(warnings, fantasy.CallWarning{
527 Type: fantasy.CallWarningTypeOther,
528 Message: "system prompt has no text parts",
529 })
530 continue
531 }
532 systemMsg := openaisdk.SystemMessage(strings.Join(systemPromptParts, "\n"))
533 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
534 if anthropicCache != nil {
535 systemMsg.OfSystem.SetExtraFields(map[string]any{
536 "cache_control": map[string]string{
537 "type": anthropicCache.Type,
538 },
539 })
540 }
541 messages = append(messages, systemMsg)
542 case fantasy.MessageRoleUser:
543 // simple user message just text content
544 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
545 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
546 if !ok {
547 warnings = append(warnings, fantasy.CallWarning{
548 Type: fantasy.CallWarningTypeOther,
549 Message: "user message text part does not have the right type",
550 })
551 continue
552 }
553 userMsg := openaisdk.UserMessage(textPart.Text)
554
555 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
556 if anthropicCache != nil {
557 userMsg.OfUser.SetExtraFields(map[string]any{
558 "cache_control": map[string]string{
559 "type": anthropicCache.Type,
560 },
561 })
562 }
563 messages = append(messages, userMsg)
564 continue
565 }
566 // text content and attachments
567 // for now we only support image content later we need to check
568 // TODO: add the supported media types to the language model so we
569 // can use that to validate the data here.
570 var content []openaisdk.ChatCompletionContentPartUnionParam
571 for i, c := range msg.Content {
572 isLastPart := i == len(msg.Content)-1
573 cacheControl := anthropic.GetCacheControl(c.Options())
574 if cacheControl == nil && isLastPart {
575 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
576 }
577 switch c.GetType() {
578 case fantasy.ContentTypeText:
579 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
580 if !ok {
581 warnings = append(warnings, fantasy.CallWarning{
582 Type: fantasy.CallWarningTypeOther,
583 Message: "user message text part does not have the right type",
584 })
585 continue
586 }
587 part := openaisdk.ChatCompletionContentPartUnionParam{
588 OfText: &openaisdk.ChatCompletionContentPartTextParam{
589 Text: textPart.Text,
590 },
591 }
592 if cacheControl != nil {
593 part.OfText.SetExtraFields(map[string]any{
594 "cache_control": map[string]string{
595 "type": cacheControl.Type,
596 },
597 })
598 }
599 content = append(content, part)
600 case fantasy.ContentTypeFile:
601 filePart, ok := fantasy.AsContentType[fantasy.FilePart](c)
602 if !ok {
603 warnings = append(warnings, fantasy.CallWarning{
604 Type: fantasy.CallWarningTypeOther,
605 Message: "user message file part does not have the right type",
606 })
607 continue
608 }
609
610 switch {
611 case strings.HasPrefix(filePart.MediaType, "image/"):
612 // Handle image files
613 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
614 data := "data:" + filePart.MediaType + ";base64," + base64Encoded
615 imageURL := openaisdk.ChatCompletionContentPartImageImageURLParam{URL: data}
616
617 // Check for provider-specific options like image detail
618 if providerOptions, ok := filePart.ProviderOptions[Name]; ok {
619 if detail, ok := providerOptions.(*openai.ProviderFileOptions); ok {
620 imageURL.Detail = detail.ImageDetail
621 }
622 }
623
624 imageBlock := openaisdk.ChatCompletionContentPartImageParam{ImageURL: imageURL}
625 if cacheControl != nil {
626 imageBlock.SetExtraFields(map[string]any{
627 "cache_control": map[string]string{
628 "type": cacheControl.Type,
629 },
630 })
631 }
632 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
633
634 case filePart.MediaType == "audio/wav":
635 // Handle WAV audio files
636 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
637 audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{
638 InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{
639 Data: base64Encoded,
640 Format: "wav",
641 },
642 }
643 if cacheControl != nil {
644 audioBlock.SetExtraFields(map[string]any{
645 "cache_control": map[string]string{
646 "type": cacheControl.Type,
647 },
648 })
649 }
650 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
651
652 case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3":
653 // Handle MP3 audio files
654 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
655 audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{
656 InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{
657 Data: base64Encoded,
658 Format: "mp3",
659 },
660 }
661 if cacheControl != nil {
662 audioBlock.SetExtraFields(map[string]any{
663 "cache_control": map[string]string{
664 "type": cacheControl.Type,
665 },
666 })
667 }
668 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
669
670 case filePart.MediaType == "application/pdf":
671 // Handle PDF files
672 dataStr := string(filePart.Data)
673
674 // Check if data looks like a file ID (starts with "file-")
675 if strings.HasPrefix(dataStr, "file-") {
676 fileBlock := openaisdk.ChatCompletionContentPartFileParam{
677 File: openaisdk.ChatCompletionContentPartFileFileParam{
678 FileID: param.NewOpt(dataStr),
679 },
680 }
681
682 if cacheControl != nil {
683 fileBlock.SetExtraFields(map[string]any{
684 "cache_control": map[string]string{
685 "type": cacheControl.Type,
686 },
687 })
688 }
689 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
690 } else {
691 // Handle as base64 data
692 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
693 data := "data:application/pdf;base64," + base64Encoded
694
695 filename := filePart.Filename
696 if filename == "" {
697 // Generate default filename based on content index
698 filename = fmt.Sprintf("part-%d.pdf", len(content))
699 }
700
701 fileBlock := openaisdk.ChatCompletionContentPartFileParam{
702 File: openaisdk.ChatCompletionContentPartFileFileParam{
703 Filename: param.NewOpt(filename),
704 FileData: param.NewOpt(data),
705 },
706 }
707 if cacheControl != nil {
708 fileBlock.SetExtraFields(map[string]any{
709 "cache_control": map[string]string{
710 "type": cacheControl.Type,
711 },
712 })
713 }
714 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
715 }
716
717 default:
718 warnings = append(warnings, fantasy.CallWarning{
719 Type: fantasy.CallWarningTypeOther,
720 Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType),
721 })
722 }
723 }
724 }
725 messages = append(messages, openaisdk.UserMessage(content))
726 case fantasy.MessageRoleAssistant:
727 // simple assistant message just text content
728 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
729 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
730 if !ok {
731 warnings = append(warnings, fantasy.CallWarning{
732 Type: fantasy.CallWarningTypeOther,
733 Message: "assistant message text part does not have the right type",
734 })
735 continue
736 }
737
738 assistantMsg := openaisdk.AssistantMessage(textPart.Text)
739 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
740 if anthropicCache != nil {
741 assistantMsg.OfAssistant.SetExtraFields(map[string]any{
742 "cache_control": map[string]string{
743 "type": anthropicCache.Type,
744 },
745 })
746 }
747 messages = append(messages, assistantMsg)
748 continue
749 }
750 assistantMsg := openaisdk.ChatCompletionAssistantMessageParam{
751 Role: "assistant",
752 }
753 for i, c := range msg.Content {
754 isLastPart := i == len(msg.Content)-1
755 cacheControl := anthropic.GetCacheControl(c.Options())
756 if cacheControl == nil && isLastPart {
757 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
758 }
759 switch c.GetType() {
760 case fantasy.ContentTypeText:
761 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
762 if !ok {
763 warnings = append(warnings, fantasy.CallWarning{
764 Type: fantasy.CallWarningTypeOther,
765 Message: "assistant message text part does not have the right type",
766 })
767 continue
768 }
769 // there is some text already there
770 if assistantMsg.Content.OfString.Valid() {
771 textPart.Text = assistantMsg.Content.OfString.Value + "\n" + textPart.Text
772 }
773 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
774 OfString: param.NewOpt(textPart.Text),
775 }
776 if cacheControl != nil {
777 assistantMsg.Content.SetExtraFields(map[string]any{
778 "cache_control": map[string]string{
779 "type": cacheControl.Type,
780 },
781 })
782 }
783 case fantasy.ContentTypeReasoning:
784 reasoningPart, ok := fantasy.AsContentType[fantasy.ReasoningPart](c)
785 if !ok {
786 warnings = append(warnings, fantasy.CallWarning{
787 Type: fantasy.CallWarningTypeOther,
788 Message: "assistant message reasoning part does not have the right type",
789 })
790 continue
791 }
792 var reasoningDetails []ReasoningDetail
793 switch {
794 case strings.HasPrefix(model, "anthropic/") && reasoningPart.Text != "":
795 metadata := anthropic.GetReasoningMetadata(reasoningPart.Options())
796 if metadata == nil {
797 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
798 if assistantMsg.Content.OfString.Valid() {
799 text = assistantMsg.Content.OfString.Value + "\n" + text
800 }
801 // this reasoning did not come from anthropic just add a text content
802 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
803 OfString: param.NewOpt(text),
804 }
805 if cacheControl != nil {
806 assistantMsg.Content.SetExtraFields(map[string]any{
807 "cache_control": map[string]string{
808 "type": cacheControl.Type,
809 },
810 })
811 }
812 continue
813 }
814 reasoningDetails = append(reasoningDetails, ReasoningDetail{
815 Format: "anthropic-claude-v1",
816 Type: "reasoning.text",
817 Text: reasoningPart.Text,
818 Signature: metadata.Signature,
819 })
820 data, _ := json.Marshal(reasoningDetails)
821 reasoningDetailsMap := []map[string]any{}
822 _ = json.Unmarshal(data, &reasoningDetailsMap)
823 assistantMsg.SetExtraFields(map[string]any{
824 "reasoning_details": reasoningDetailsMap,
825 "reasoning": reasoningPart.Text,
826 })
827 case strings.HasPrefix(model, "openai/"):
828 metadata := openai.GetReasoningMetadata(reasoningPart.Options())
829 if metadata == nil {
830 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
831 if assistantMsg.Content.OfString.Valid() {
832 text = assistantMsg.Content.OfString.Value + "\n" + text
833 }
834 // this reasoning did not come from anthropic just add a text content
835 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
836 OfString: param.NewOpt(text),
837 }
838 continue
839 }
840 for inx, summary := range metadata.Summary {
841 if summary == "" {
842 continue
843 }
844 reasoningDetails = append(reasoningDetails, ReasoningDetail{
845 Type: "reasoning.summary",
846 Format: "openai-responses-v1",
847 Summary: summary,
848 Index: inx,
849 })
850 }
851 if metadata.EncryptedContent != nil {
852 reasoningDetails = append(reasoningDetails, ReasoningDetail{
853 Type: "reasoning.encrypted",
854 Format: "openai-responses-v1",
855 Data: *metadata.EncryptedContent,
856 ID: metadata.ItemID,
857 })
858 }
859 data, _ := json.Marshal(reasoningDetails)
860 reasoningDetailsMap := []map[string]any{}
861 _ = json.Unmarshal(data, &reasoningDetailsMap)
862 assistantMsg.SetExtraFields(map[string]any{
863 "reasoning_details": reasoningDetailsMap,
864 })
865 case strings.HasPrefix(model, "xai/"):
866 metadata := openai.GetReasoningMetadata(reasoningPart.Options())
867 if metadata == nil {
868 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
869 if assistantMsg.Content.OfString.Valid() {
870 text = assistantMsg.Content.OfString.Value + "\n" + text
871 }
872 // this reasoning did not come from anthropic just add a text content
873 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
874 OfString: param.NewOpt(text),
875 }
876 continue
877 }
878 for inx, summary := range metadata.Summary {
879 if summary == "" {
880 continue
881 }
882 reasoningDetails = append(reasoningDetails, ReasoningDetail{
883 Type: "reasoning.summary",
884 Format: "xai-responses-v1",
885 Summary: summary,
886 Index: inx,
887 })
888 }
889 if metadata.EncryptedContent != nil {
890 reasoningDetails = append(reasoningDetails, ReasoningDetail{
891 Type: "reasoning.encrypted",
892 Format: "xai-responses-v1",
893 Data: *metadata.EncryptedContent,
894 ID: metadata.ItemID,
895 })
896 }
897 data, _ := json.Marshal(reasoningDetails)
898 reasoningDetailsMap := []map[string]any{}
899 _ = json.Unmarshal(data, &reasoningDetailsMap)
900 assistantMsg.SetExtraFields(map[string]any{
901 "reasoning_details": reasoningDetailsMap,
902 })
903 case strings.HasPrefix(model, "google/"):
904 metadata := google.GetReasoningMetadata(reasoningPart.Options())
905 if metadata == nil {
906 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
907 if assistantMsg.Content.OfString.Valid() {
908 text = assistantMsg.Content.OfString.Value + "\n" + text
909 }
910 // this reasoning did not come from anthropic just add a text content
911 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
912 OfString: param.NewOpt(text),
913 }
914 continue
915 }
916 if reasoningPart.Text != "" {
917 reasoningDetails = append(reasoningDetails, ReasoningDetail{
918 Type: "reasoning.text",
919 Format: "google-gemini-v1",
920 Text: reasoningPart.Text,
921 })
922 }
923 reasoningDetails = append(reasoningDetails, ReasoningDetail{
924 Type: "reasoning.encrypted",
925 Format: "google-gemini-v1",
926 Data: metadata.Signature,
927 ID: metadata.ToolID,
928 })
929 data, _ := json.Marshal(reasoningDetails)
930 reasoningDetailsMap := []map[string]any{}
931 _ = json.Unmarshal(data, &reasoningDetailsMap)
932 assistantMsg.SetExtraFields(map[string]any{
933 "reasoning_details": reasoningDetailsMap,
934 })
935 default:
936 reasoningDetails = append(reasoningDetails, ReasoningDetail{
937 Type: "reasoning.text",
938 Text: reasoningPart.Text,
939 Format: "unknown",
940 })
941 data, _ := json.Marshal(reasoningDetails)
942 reasoningDetailsMap := []map[string]any{}
943 _ = json.Unmarshal(data, &reasoningDetailsMap)
944 assistantMsg.SetExtraFields(map[string]any{
945 "reasoning_details": reasoningDetailsMap,
946 })
947 }
948 case fantasy.ContentTypeToolCall:
949 toolCallPart, ok := fantasy.AsContentType[fantasy.ToolCallPart](c)
950 if !ok {
951 warnings = append(warnings, fantasy.CallWarning{
952 Type: fantasy.CallWarningTypeOther,
953 Message: "assistant message tool part does not have the right type",
954 })
955 continue
956 }
957 tc := openaisdk.ChatCompletionMessageToolCallUnionParam{
958 OfFunction: &openaisdk.ChatCompletionMessageFunctionToolCallParam{
959 ID: toolCallPart.ToolCallID,
960 Type: "function",
961 Function: openaisdk.ChatCompletionMessageFunctionToolCallFunctionParam{
962 Name: toolCallPart.ToolName,
963 Arguments: toolCallPart.Input,
964 },
965 },
966 }
967 if cacheControl != nil {
968 tc.OfFunction.SetExtraFields(map[string]any{
969 "cache_control": map[string]string{
970 "type": cacheControl.Type,
971 },
972 })
973 }
974 assistantMsg.ToolCalls = append(assistantMsg.ToolCalls, tc)
975 }
976 }
977 messages = append(messages, openaisdk.ChatCompletionMessageParamUnion{
978 OfAssistant: &assistantMsg,
979 })
980 case fantasy.MessageRoleTool:
981 for i, c := range msg.Content {
982 isLastPart := i == len(msg.Content)-1
983 cacheControl := anthropic.GetCacheControl(c.Options())
984 if cacheControl == nil && isLastPart {
985 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
986 }
987 if c.GetType() != fantasy.ContentTypeToolResult {
988 warnings = append(warnings, fantasy.CallWarning{
989 Type: fantasy.CallWarningTypeOther,
990 Message: "tool message can only have tool result content",
991 })
992 continue
993 }
994
995 toolResultPart, ok := fantasy.AsContentType[fantasy.ToolResultPart](c)
996 if !ok {
997 warnings = append(warnings, fantasy.CallWarning{
998 Type: fantasy.CallWarningTypeOther,
999 Message: "tool message result part does not have the right type",
1000 })
1001 continue
1002 }
1003
1004 switch toolResultPart.Output.GetType() {
1005 case fantasy.ToolResultContentTypeText:
1006 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](toolResultPart.Output)
1007 if !ok {
1008 warnings = append(warnings, fantasy.CallWarning{
1009 Type: fantasy.CallWarningTypeOther,
1010 Message: "tool result output does not have the right type",
1011 })
1012 continue
1013 }
1014 tr := openaisdk.ToolMessage(output.Text, toolResultPart.ToolCallID)
1015 if cacheControl != nil {
1016 tr.SetExtraFields(map[string]any{
1017 "cache_control": map[string]string{
1018 "type": cacheControl.Type,
1019 },
1020 })
1021 }
1022 messages = append(messages, tr)
1023 case fantasy.ToolResultContentTypeError:
1024 // TODO: check if better handling is needed
1025 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentError](toolResultPart.Output)
1026 if !ok {
1027 warnings = append(warnings, fantasy.CallWarning{
1028 Type: fantasy.CallWarningTypeOther,
1029 Message: "tool result output does not have the right type",
1030 })
1031 continue
1032 }
1033 tr := openaisdk.ToolMessage(output.Error.Error(), toolResultPart.ToolCallID)
1034 if cacheControl != nil {
1035 tr.SetExtraFields(map[string]any{
1036 "cache_control": map[string]string{
1037 "type": cacheControl.Type,
1038 },
1039 })
1040 }
1041 messages = append(messages, tr)
1042 }
1043 }
1044 }
1045 }
1046 return messages, warnings
1047}