1package openrouter
2
3import (
4 "encoding/base64"
5 "encoding/json"
6 "fmt"
7 "maps"
8 "strings"
9
10 "charm.land/fantasy"
11 "charm.land/fantasy/providers/anthropic"
12 "charm.land/fantasy/providers/google"
13 "charm.land/fantasy/providers/openai"
14 openaisdk "github.com/openai/openai-go/v2"
15 "github.com/openai/openai-go/v2/packages/param"
16)
17
18const reasoningStartedCtx = "reasoning_started"
19
20func languagePrepareModelCall(_ fantasy.LanguageModel, params *openaisdk.ChatCompletionNewParams, call fantasy.Call) ([]fantasy.CallWarning, error) {
21 providerOptions := &ProviderOptions{}
22 if v, ok := call.ProviderOptions[Name]; ok {
23 providerOptions, ok = v.(*ProviderOptions)
24 if !ok {
25 return nil, &fantasy.Error{Title: "invalid argument", Message: "openrouter provider options should be *openrouter.ProviderOptions"}
26 }
27 }
28
29 extraFields := make(map[string]any)
30
31 if providerOptions.Provider != nil {
32 data, err := structToMapJSON(providerOptions.Provider)
33 if err != nil {
34 return nil, err
35 }
36 extraFields["provider"] = data
37 }
38
39 if providerOptions.Reasoning != nil {
40 data, err := structToMapJSON(providerOptions.Reasoning)
41 if err != nil {
42 return nil, err
43 }
44 extraFields["reasoning"] = data
45 }
46
47 if providerOptions.IncludeUsage != nil {
48 extraFields["usage"] = map[string]any{
49 "include": *providerOptions.IncludeUsage,
50 }
51 } else { // default include usage
52 extraFields["usage"] = map[string]any{
53 "include": true,
54 }
55 }
56 if providerOptions.LogitBias != nil {
57 params.LogitBias = providerOptions.LogitBias
58 }
59 if providerOptions.LogProbs != nil {
60 params.Logprobs = param.NewOpt(*providerOptions.LogProbs)
61 }
62 if providerOptions.User != nil {
63 params.User = param.NewOpt(*providerOptions.User)
64 }
65 if providerOptions.ParallelToolCalls != nil {
66 params.ParallelToolCalls = param.NewOpt(*providerOptions.ParallelToolCalls)
67 }
68
69 maps.Copy(extraFields, providerOptions.ExtraBody)
70 params.SetExtraFields(extraFields)
71 return nil, nil
72}
73
74func languageModelExtraContent(choice openaisdk.ChatCompletionChoice) []fantasy.Content {
75 content := make([]fantasy.Content, 0)
76 reasoningData := ReasoningData{}
77 err := json.Unmarshal([]byte(choice.Message.RawJSON()), &reasoningData)
78 if err != nil {
79 return content
80 }
81 type anthropicReasoningBlock struct {
82 text string
83 metadata *anthropic.ReasoningOptionMetadata
84 }
85 type googleReasoningBlock struct {
86 text string
87 metadata *google.ReasoningMetadata
88 }
89
90 responsesReasoningBlocks := make([]openai.ResponsesReasoningMetadata, 0)
91 anthropicReasoningBlocks := make([]anthropicReasoningBlock, 0)
92 googleReasoningBlocks := make([]googleReasoningBlock, 0)
93 otherReasoning := make([]string, 0)
94 for _, detail := range reasoningData.ReasoningDetails {
95 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
96 var thinkingBlock openai.ResponsesReasoningMetadata
97 if len(responsesReasoningBlocks)-1 >= detail.Index {
98 thinkingBlock = responsesReasoningBlocks[detail.Index]
99 } else {
100 thinkingBlock = openai.ResponsesReasoningMetadata{}
101 responsesReasoningBlocks = append(responsesReasoningBlocks, thinkingBlock)
102 }
103
104 switch detail.Type {
105 case "reasoning.summary":
106 thinkingBlock.Summary = append(thinkingBlock.Summary, detail.Summary)
107 case "reasoning.encrypted":
108 thinkingBlock.EncryptedContent = &detail.Data
109 }
110 if detail.ID != "" {
111 thinkingBlock.ItemID = detail.ID
112 }
113
114 responsesReasoningBlocks[detail.Index] = thinkingBlock
115 continue
116 }
117 if strings.HasPrefix(detail.Format, "google-gemini") {
118 var thinkingBlock googleReasoningBlock
119 if len(googleReasoningBlocks)-1 >= detail.Index {
120 thinkingBlock = googleReasoningBlocks[detail.Index]
121 } else {
122 thinkingBlock = googleReasoningBlock{metadata: &google.ReasoningMetadata{}}
123 googleReasoningBlocks = append(googleReasoningBlocks, thinkingBlock)
124 }
125
126 switch detail.Type {
127 case "reasoning.text":
128 thinkingBlock.text = detail.Text
129 case "reasoning.encrypted":
130 thinkingBlock.metadata.Signature = detail.Data
131 thinkingBlock.metadata.ToolID = detail.ID
132 }
133
134 googleReasoningBlocks[detail.Index] = thinkingBlock
135 continue
136 }
137
138 if strings.HasPrefix(detail.Format, "anthropic-claude") {
139 anthropicReasoningBlocks = append(anthropicReasoningBlocks, anthropicReasoningBlock{
140 text: detail.Text,
141 metadata: &anthropic.ReasoningOptionMetadata{
142 Signature: detail.Signature,
143 },
144 })
145 continue
146 }
147
148 otherReasoning = append(otherReasoning, detail.Text)
149 }
150
151 for _, block := range responsesReasoningBlocks {
152 if len(block.Summary) == 0 {
153 block.Summary = []string{""}
154 }
155 content = append(content, fantasy.ReasoningContent{
156 Text: strings.Join(block.Summary, "\n"),
157 ProviderMetadata: fantasy.ProviderMetadata{
158 openai.Name: &block,
159 },
160 })
161 }
162 for _, block := range anthropicReasoningBlocks {
163 content = append(content, fantasy.ReasoningContent{
164 Text: block.text,
165 ProviderMetadata: fantasy.ProviderMetadata{
166 anthropic.Name: block.metadata,
167 },
168 })
169 }
170 for _, block := range googleReasoningBlocks {
171 content = append(content, fantasy.ReasoningContent{
172 Text: block.text,
173 ProviderMetadata: fantasy.ProviderMetadata{
174 google.Name: block.metadata,
175 },
176 })
177 }
178
179 for _, reasoning := range otherReasoning {
180 content = append(content, fantasy.ReasoningContent{
181 Text: reasoning,
182 })
183 }
184 return content
185}
186
187type currentReasoningState struct {
188 metadata *openai.ResponsesReasoningMetadata
189 googleMetadata *google.ReasoningMetadata
190 googleText string
191}
192
193func extractReasoningContext(ctx map[string]any) *currentReasoningState {
194 reasoningStarted, ok := ctx[reasoningStartedCtx]
195 if !ok {
196 return nil
197 }
198 state, ok := reasoningStarted.(*currentReasoningState)
199 if !ok {
200 return nil
201 }
202 return state
203}
204
205func languageModelStreamExtra(chunk openaisdk.ChatCompletionChunk, yield func(fantasy.StreamPart) bool, ctx map[string]any) (map[string]any, bool) {
206 if len(chunk.Choices) == 0 {
207 return ctx, true
208 }
209
210 currentState := extractReasoningContext(ctx)
211
212 inx := 0
213 choice := chunk.Choices[inx]
214 reasoningData := ReasoningData{}
215 err := json.Unmarshal([]byte(choice.Delta.RawJSON()), &reasoningData)
216 if err != nil {
217 yield(fantasy.StreamPart{
218 Type: fantasy.StreamPartTypeError,
219 Error: &fantasy.Error{Title: "stream error", Message: "error unmarshalling delta", Cause: err},
220 })
221 return ctx, false
222 }
223
224 // Reasoning Start
225 if currentState == nil {
226 if len(reasoningData.ReasoningDetails) == 0 {
227 return ctx, true
228 }
229
230 var metadata fantasy.ProviderMetadata
231 currentState = ¤tReasoningState{}
232
233 detail := reasoningData.ReasoningDetails[0]
234 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
235 currentState.metadata = &openai.ResponsesReasoningMetadata{
236 Summary: []string{detail.Summary},
237 }
238 metadata = fantasy.ProviderMetadata{
239 openai.Name: currentState.metadata,
240 }
241 // There was no summary just thinking we just send this as if it ended alredy
242 if detail.Data != "" {
243 shouldContinue := yield(fantasy.StreamPart{
244 Type: fantasy.StreamPartTypeReasoningStart,
245 ID: fmt.Sprintf("%d", inx),
246 Delta: detail.Summary,
247 ProviderMetadata: metadata,
248 })
249 if !shouldContinue {
250 return ctx, false
251 }
252 return ctx, yield(fantasy.StreamPart{
253 Type: fantasy.StreamPartTypeReasoningEnd,
254 ID: fmt.Sprintf("%d", inx),
255 ProviderMetadata: fantasy.ProviderMetadata{
256 openai.Name: &openai.ResponsesReasoningMetadata{
257 Summary: []string{detail.Summary},
258 EncryptedContent: &detail.Data,
259 ItemID: detail.ID,
260 },
261 },
262 })
263 }
264 }
265
266 if strings.HasPrefix(detail.Format, "google-gemini") {
267 // this means there is only encrypted data available start and finish right away
268 if detail.Type == "reasoning.encrypted" {
269 ctx[reasoningStartedCtx] = nil
270 if !yield(fantasy.StreamPart{
271 Type: fantasy.StreamPartTypeReasoningStart,
272 ID: fmt.Sprintf("%d", inx),
273 }) {
274 return ctx, false
275 }
276 return ctx, yield(fantasy.StreamPart{
277 Type: fantasy.StreamPartTypeReasoningEnd,
278 ID: fmt.Sprintf("%d", inx),
279 ProviderMetadata: fantasy.ProviderMetadata{
280 google.Name: &google.ReasoningMetadata{
281 Signature: detail.Data,
282 ToolID: detail.ID,
283 },
284 },
285 })
286 }
287 currentState.googleMetadata = &google.ReasoningMetadata{}
288 currentState.googleText = detail.Text
289 metadata = fantasy.ProviderMetadata{
290 google.Name: currentState.googleMetadata,
291 }
292 }
293
294 ctx[reasoningStartedCtx] = currentState
295 delta := detail.Summary
296 if strings.HasPrefix(detail.Format, "google-gemini") {
297 delta = detail.Text
298 }
299 return ctx, yield(fantasy.StreamPart{
300 Type: fantasy.StreamPartTypeReasoningStart,
301 ID: fmt.Sprintf("%d", inx),
302 Delta: delta,
303 ProviderMetadata: metadata,
304 })
305 }
306 if len(reasoningData.ReasoningDetails) == 0 {
307 // this means its a model different from openai/anthropic that ended reasoning
308 if choice.Delta.Content != "" || len(choice.Delta.ToolCalls) > 0 {
309 ctx[reasoningStartedCtx] = nil
310 return ctx, yield(fantasy.StreamPart{
311 Type: fantasy.StreamPartTypeReasoningEnd,
312 ID: fmt.Sprintf("%d", inx),
313 })
314 }
315 return ctx, true
316 }
317 // Reasoning delta
318 detail := reasoningData.ReasoningDetails[0]
319 if strings.HasPrefix(detail.Format, "openai-responses") || strings.HasPrefix(detail.Format, "xai-responses") {
320 // Reasoning has ended
321 if detail.Data != "" {
322 currentState.metadata.EncryptedContent = &detail.Data
323 currentState.metadata.ItemID = detail.ID
324 ctx[reasoningStartedCtx] = nil
325 return ctx, yield(fantasy.StreamPart{
326 Type: fantasy.StreamPartTypeReasoningEnd,
327 ID: fmt.Sprintf("%d", inx),
328 ProviderMetadata: fantasy.ProviderMetadata{
329 openai.Name: currentState.metadata,
330 },
331 })
332 }
333 var textDelta string
334 // add to existing summary
335 if len(currentState.metadata.Summary)-1 >= detail.Index {
336 currentState.metadata.Summary[detail.Index] += detail.Summary
337 textDelta = detail.Summary
338 } else { // add new summary
339 currentState.metadata.Summary = append(currentState.metadata.Summary, detail.Summary)
340 textDelta = "\n" + detail.Summary
341 }
342 ctx[reasoningStartedCtx] = currentState
343 return ctx, yield(fantasy.StreamPart{
344 Type: fantasy.StreamPartTypeReasoningDelta,
345 ID: fmt.Sprintf("%d", inx),
346 Delta: textDelta,
347 ProviderMetadata: fantasy.ProviderMetadata{
348 openai.Name: currentState.metadata,
349 },
350 })
351 }
352 if strings.HasPrefix(detail.Format, "anthropic-claude") {
353 // the reasoning has ended
354 if detail.Signature != "" {
355 metadata := fantasy.ProviderMetadata{
356 anthropic.Name: &anthropic.ReasoningOptionMetadata{
357 Signature: detail.Signature,
358 },
359 }
360 // initial update
361 shouldContinue := yield(fantasy.StreamPart{
362 Type: fantasy.StreamPartTypeReasoningDelta,
363 ID: fmt.Sprintf("%d", inx),
364 Delta: detail.Text,
365 ProviderMetadata: metadata,
366 })
367 if !shouldContinue {
368 return ctx, false
369 }
370 ctx[reasoningStartedCtx] = nil
371 return ctx, yield(fantasy.StreamPart{
372 Type: fantasy.StreamPartTypeReasoningEnd,
373 ID: fmt.Sprintf("%d", inx),
374 })
375 }
376
377 return ctx, yield(fantasy.StreamPart{
378 Type: fantasy.StreamPartTypeReasoningDelta,
379 ID: fmt.Sprintf("%d", inx),
380 Delta: detail.Text,
381 })
382 }
383
384 if strings.HasPrefix(detail.Format, "google-gemini") {
385 // reasoning.text type - accumulate text
386 if detail.Type == "reasoning.text" {
387 currentState.googleText += detail.Text
388 ctx[reasoningStartedCtx] = currentState
389 return ctx, yield(fantasy.StreamPart{
390 Type: fantasy.StreamPartTypeReasoningDelta,
391 ID: fmt.Sprintf("%d", inx),
392 Delta: detail.Text,
393 })
394 }
395
396 // reasoning.encrypted type - end reasoning with signature
397 if detail.Type == "reasoning.encrypted" {
398 currentState.googleMetadata.Signature = detail.Data
399 currentState.googleMetadata.ToolID = detail.ID
400 metadata := fantasy.ProviderMetadata{
401 google.Name: currentState.googleMetadata,
402 }
403 ctx[reasoningStartedCtx] = nil
404 return ctx, yield(fantasy.StreamPart{
405 Type: fantasy.StreamPartTypeReasoningEnd,
406 ID: fmt.Sprintf("%d", inx),
407 ProviderMetadata: metadata,
408 })
409 }
410 }
411
412 return ctx, yield(fantasy.StreamPart{
413 Type: fantasy.StreamPartTypeReasoningDelta,
414 ID: fmt.Sprintf("%d", inx),
415 Delta: detail.Text,
416 })
417}
418
419func languageModelUsage(response openaisdk.ChatCompletion) (fantasy.Usage, fantasy.ProviderOptionsData) {
420 if len(response.Choices) == 0 {
421 return fantasy.Usage{}, nil
422 }
423 openrouterUsage := UsageAccounting{}
424 usage := response.Usage
425
426 _ = json.Unmarshal([]byte(usage.RawJSON()), &openrouterUsage)
427
428 completionTokenDetails := usage.CompletionTokensDetails
429 promptTokenDetails := usage.PromptTokensDetails
430
431 var provider string
432 if p, ok := response.JSON.ExtraFields["provider"]; ok {
433 provider = p.Raw()
434 }
435
436 // Build provider metadata
437 providerMetadata := &ProviderMetadata{
438 Provider: provider,
439 Usage: openrouterUsage,
440 }
441
442 return fantasy.Usage{
443 InputTokens: usage.PromptTokens,
444 OutputTokens: usage.CompletionTokens,
445 TotalTokens: usage.TotalTokens,
446 ReasoningTokens: completionTokenDetails.ReasoningTokens,
447 CacheReadTokens: promptTokenDetails.CachedTokens,
448 }, providerMetadata
449}
450
451func languageModelStreamUsage(chunk openaisdk.ChatCompletionChunk, _ map[string]any, metadata fantasy.ProviderMetadata) (fantasy.Usage, fantasy.ProviderMetadata) {
452 usage := chunk.Usage
453 if usage.TotalTokens == 0 {
454 return fantasy.Usage{}, nil
455 }
456
457 streamProviderMetadata := &ProviderMetadata{}
458 if metadata != nil {
459 if providerMetadata, ok := metadata[Name]; ok {
460 converted, ok := providerMetadata.(*ProviderMetadata)
461 if ok {
462 streamProviderMetadata = converted
463 }
464 }
465 }
466 openrouterUsage := UsageAccounting{}
467 _ = json.Unmarshal([]byte(usage.RawJSON()), &openrouterUsage)
468 streamProviderMetadata.Usage = openrouterUsage
469
470 if p, ok := chunk.JSON.ExtraFields["provider"]; ok {
471 streamProviderMetadata.Provider = p.Raw()
472 }
473
474 // we do this here because the acc does not add prompt details
475 completionTokenDetails := usage.CompletionTokensDetails
476 promptTokenDetails := usage.PromptTokensDetails
477 aiUsage := fantasy.Usage{
478 InputTokens: usage.PromptTokens,
479 OutputTokens: usage.CompletionTokens,
480 TotalTokens: usage.TotalTokens,
481 ReasoningTokens: completionTokenDetails.ReasoningTokens,
482 CacheReadTokens: promptTokenDetails.CachedTokens,
483 }
484
485 return aiUsage, fantasy.ProviderMetadata{
486 Name: streamProviderMetadata,
487 }
488}
489
490func languageModelToPrompt(prompt fantasy.Prompt, _, model string) ([]openaisdk.ChatCompletionMessageParamUnion, []fantasy.CallWarning) {
491 var messages []openaisdk.ChatCompletionMessageParamUnion
492 var warnings []fantasy.CallWarning
493 for _, msg := range prompt {
494 switch msg.Role {
495 case fantasy.MessageRoleSystem:
496 var systemPromptParts []string
497 for _, c := range msg.Content {
498 if c.GetType() != fantasy.ContentTypeText {
499 warnings = append(warnings, fantasy.CallWarning{
500 Type: fantasy.CallWarningTypeOther,
501 Message: "system prompt can only have text content",
502 })
503 continue
504 }
505 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
506 if !ok {
507 warnings = append(warnings, fantasy.CallWarning{
508 Type: fantasy.CallWarningTypeOther,
509 Message: "system prompt text part does not have the right type",
510 })
511 continue
512 }
513 text := textPart.Text
514 if strings.TrimSpace(text) != "" {
515 systemPromptParts = append(systemPromptParts, textPart.Text)
516 }
517 }
518 if len(systemPromptParts) == 0 {
519 warnings = append(warnings, fantasy.CallWarning{
520 Type: fantasy.CallWarningTypeOther,
521 Message: "system prompt has no text parts",
522 })
523 continue
524 }
525 systemMsg := openaisdk.SystemMessage(strings.Join(systemPromptParts, "\n"))
526 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
527 if anthropicCache != nil {
528 systemMsg.OfSystem.SetExtraFields(map[string]any{
529 "cache_control": map[string]string{
530 "type": anthropicCache.Type,
531 },
532 })
533 }
534 messages = append(messages, systemMsg)
535 case fantasy.MessageRoleUser:
536 // simple user message just text content
537 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
538 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
539 if !ok {
540 warnings = append(warnings, fantasy.CallWarning{
541 Type: fantasy.CallWarningTypeOther,
542 Message: "user message text part does not have the right type",
543 })
544 continue
545 }
546 userMsg := openaisdk.UserMessage(textPart.Text)
547
548 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
549 if anthropicCache != nil {
550 userMsg.OfUser.SetExtraFields(map[string]any{
551 "cache_control": map[string]string{
552 "type": anthropicCache.Type,
553 },
554 })
555 }
556 messages = append(messages, userMsg)
557 continue
558 }
559 // text content and attachments
560 // for now we only support image content later we need to check
561 // TODO: add the supported media types to the language model so we
562 // can use that to validate the data here.
563 var content []openaisdk.ChatCompletionContentPartUnionParam
564 for i, c := range msg.Content {
565 isLastPart := i == len(msg.Content)-1
566 cacheControl := anthropic.GetCacheControl(c.Options())
567 if cacheControl == nil && isLastPart {
568 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
569 }
570 switch c.GetType() {
571 case fantasy.ContentTypeText:
572 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
573 if !ok {
574 warnings = append(warnings, fantasy.CallWarning{
575 Type: fantasy.CallWarningTypeOther,
576 Message: "user message text part does not have the right type",
577 })
578 continue
579 }
580 part := openaisdk.ChatCompletionContentPartUnionParam{
581 OfText: &openaisdk.ChatCompletionContentPartTextParam{
582 Text: textPart.Text,
583 },
584 }
585 if cacheControl != nil {
586 part.OfText.SetExtraFields(map[string]any{
587 "cache_control": map[string]string{
588 "type": cacheControl.Type,
589 },
590 })
591 }
592 content = append(content, part)
593 case fantasy.ContentTypeFile:
594 filePart, ok := fantasy.AsContentType[fantasy.FilePart](c)
595 if !ok {
596 warnings = append(warnings, fantasy.CallWarning{
597 Type: fantasy.CallWarningTypeOther,
598 Message: "user message file part does not have the right type",
599 })
600 continue
601 }
602
603 switch {
604 case strings.HasPrefix(filePart.MediaType, "image/"):
605 // Handle image files
606 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
607 data := "data:" + filePart.MediaType + ";base64," + base64Encoded
608 imageURL := openaisdk.ChatCompletionContentPartImageImageURLParam{URL: data}
609
610 // Check for provider-specific options like image detail
611 if providerOptions, ok := filePart.ProviderOptions[Name]; ok {
612 if detail, ok := providerOptions.(*openai.ProviderFileOptions); ok {
613 imageURL.Detail = detail.ImageDetail
614 }
615 }
616
617 imageBlock := openaisdk.ChatCompletionContentPartImageParam{ImageURL: imageURL}
618 if cacheControl != nil {
619 imageBlock.SetExtraFields(map[string]any{
620 "cache_control": map[string]string{
621 "type": cacheControl.Type,
622 },
623 })
624 }
625 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
626
627 case filePart.MediaType == "audio/wav":
628 // Handle WAV audio files
629 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
630 audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{
631 InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{
632 Data: base64Encoded,
633 Format: "wav",
634 },
635 }
636 if cacheControl != nil {
637 audioBlock.SetExtraFields(map[string]any{
638 "cache_control": map[string]string{
639 "type": cacheControl.Type,
640 },
641 })
642 }
643 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
644
645 case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3":
646 // Handle MP3 audio files
647 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
648 audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{
649 InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{
650 Data: base64Encoded,
651 Format: "mp3",
652 },
653 }
654 if cacheControl != nil {
655 audioBlock.SetExtraFields(map[string]any{
656 "cache_control": map[string]string{
657 "type": cacheControl.Type,
658 },
659 })
660 }
661 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock})
662
663 case filePart.MediaType == "application/pdf":
664 // Handle PDF files
665 dataStr := string(filePart.Data)
666
667 // Check if data looks like a file ID (starts with "file-")
668 if strings.HasPrefix(dataStr, "file-") {
669 fileBlock := openaisdk.ChatCompletionContentPartFileParam{
670 File: openaisdk.ChatCompletionContentPartFileFileParam{
671 FileID: param.NewOpt(dataStr),
672 },
673 }
674
675 if cacheControl != nil {
676 fileBlock.SetExtraFields(map[string]any{
677 "cache_control": map[string]string{
678 "type": cacheControl.Type,
679 },
680 })
681 }
682 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
683 } else {
684 // Handle as base64 data
685 base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data)
686 data := "data:application/pdf;base64," + base64Encoded
687
688 filename := filePart.Filename
689 if filename == "" {
690 // Generate default filename based on content index
691 filename = fmt.Sprintf("part-%d.pdf", len(content))
692 }
693
694 fileBlock := openaisdk.ChatCompletionContentPartFileParam{
695 File: openaisdk.ChatCompletionContentPartFileFileParam{
696 Filename: param.NewOpt(filename),
697 FileData: param.NewOpt(data),
698 },
699 }
700 if cacheControl != nil {
701 fileBlock.SetExtraFields(map[string]any{
702 "cache_control": map[string]string{
703 "type": cacheControl.Type,
704 },
705 })
706 }
707 content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock})
708 }
709
710 default:
711 warnings = append(warnings, fantasy.CallWarning{
712 Type: fantasy.CallWarningTypeOther,
713 Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType),
714 })
715 }
716 }
717 }
718 messages = append(messages, openaisdk.UserMessage(content))
719 case fantasy.MessageRoleAssistant:
720 // simple assistant message just text content
721 if len(msg.Content) == 1 && msg.Content[0].GetType() == fantasy.ContentTypeText {
722 textPart, ok := fantasy.AsContentType[fantasy.TextPart](msg.Content[0])
723 if !ok {
724 warnings = append(warnings, fantasy.CallWarning{
725 Type: fantasy.CallWarningTypeOther,
726 Message: "assistant message text part does not have the right type",
727 })
728 continue
729 }
730
731 assistantMsg := openaisdk.AssistantMessage(textPart.Text)
732 anthropicCache := anthropic.GetCacheControl(msg.ProviderOptions)
733 if anthropicCache != nil {
734 assistantMsg.OfAssistant.SetExtraFields(map[string]any{
735 "cache_control": map[string]string{
736 "type": anthropicCache.Type,
737 },
738 })
739 }
740 messages = append(messages, assistantMsg)
741 continue
742 }
743 assistantMsg := openaisdk.ChatCompletionAssistantMessageParam{
744 Role: "assistant",
745 }
746 for i, c := range msg.Content {
747 isLastPart := i == len(msg.Content)-1
748 cacheControl := anthropic.GetCacheControl(c.Options())
749 if cacheControl == nil && isLastPart {
750 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
751 }
752 switch c.GetType() {
753 case fantasy.ContentTypeText:
754 textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
755 if !ok {
756 warnings = append(warnings, fantasy.CallWarning{
757 Type: fantasy.CallWarningTypeOther,
758 Message: "assistant message text part does not have the right type",
759 })
760 continue
761 }
762 // there is some text already there
763 if assistantMsg.Content.OfString.Valid() {
764 textPart.Text = assistantMsg.Content.OfString.Value + "\n" + textPart.Text
765 }
766 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
767 OfString: param.NewOpt(textPart.Text),
768 }
769 if cacheControl != nil {
770 assistantMsg.Content.SetExtraFields(map[string]any{
771 "cache_control": map[string]string{
772 "type": cacheControl.Type,
773 },
774 })
775 }
776 case fantasy.ContentTypeReasoning:
777 reasoningPart, ok := fantasy.AsContentType[fantasy.ReasoningPart](c)
778 if !ok {
779 warnings = append(warnings, fantasy.CallWarning{
780 Type: fantasy.CallWarningTypeOther,
781 Message: "assistant message reasoning part does not have the right type",
782 })
783 continue
784 }
785 var reasoningDetails []ReasoningDetail
786 switch {
787 case strings.HasPrefix(model, "anthropic/") && reasoningPart.Text != "":
788 metadata := anthropic.GetReasoningMetadata(reasoningPart.Options())
789 if metadata == nil {
790 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
791 if assistantMsg.Content.OfString.Valid() {
792 text = assistantMsg.Content.OfString.Value + "\n" + text
793 }
794 // this reasoning did not come from anthropic just add a text content
795 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
796 OfString: param.NewOpt(text),
797 }
798 if cacheControl != nil {
799 assistantMsg.Content.SetExtraFields(map[string]any{
800 "cache_control": map[string]string{
801 "type": cacheControl.Type,
802 },
803 })
804 }
805 continue
806 }
807 reasoningDetails = append(reasoningDetails, ReasoningDetail{
808 Format: "anthropic-claude-v1",
809 Type: "reasoning.text",
810 Text: reasoningPart.Text,
811 Signature: metadata.Signature,
812 })
813 data, _ := json.Marshal(reasoningDetails)
814 reasoningDetailsMap := []map[string]any{}
815 _ = json.Unmarshal(data, &reasoningDetailsMap)
816 assistantMsg.SetExtraFields(map[string]any{
817 "reasoning_details": reasoningDetailsMap,
818 "reasoning": reasoningPart.Text,
819 })
820 case strings.HasPrefix(model, "openai/"):
821 metadata := openai.GetReasoningMetadata(reasoningPart.Options())
822 if metadata == nil {
823 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
824 if assistantMsg.Content.OfString.Valid() {
825 text = assistantMsg.Content.OfString.Value + "\n" + text
826 }
827 // this reasoning did not come from anthropic just add a text content
828 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
829 OfString: param.NewOpt(text),
830 }
831 continue
832 }
833 for inx, summary := range metadata.Summary {
834 if summary == "" {
835 continue
836 }
837 reasoningDetails = append(reasoningDetails, ReasoningDetail{
838 Type: "reasoning.summary",
839 Format: "openai-responses-v1",
840 Summary: summary,
841 Index: inx,
842 })
843 }
844 reasoningDetails = append(reasoningDetails, ReasoningDetail{
845 Type: "reasoning.encrypted",
846 Format: "openai-responses-v1",
847 Data: *metadata.EncryptedContent,
848 ID: metadata.ItemID,
849 })
850 data, _ := json.Marshal(reasoningDetails)
851 reasoningDetailsMap := []map[string]any{}
852 _ = json.Unmarshal(data, &reasoningDetailsMap)
853 assistantMsg.SetExtraFields(map[string]any{
854 "reasoning_details": reasoningDetailsMap,
855 })
856 case strings.HasPrefix(model, "xai/"):
857 metadata := openai.GetReasoningMetadata(reasoningPart.Options())
858 if metadata == nil {
859 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
860 if assistantMsg.Content.OfString.Valid() {
861 text = assistantMsg.Content.OfString.Value + "\n" + text
862 }
863 // this reasoning did not come from anthropic just add a text content
864 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
865 OfString: param.NewOpt(text),
866 }
867 continue
868 }
869 for inx, summary := range metadata.Summary {
870 if summary == "" {
871 continue
872 }
873 reasoningDetails = append(reasoningDetails, ReasoningDetail{
874 Type: "reasoning.summary",
875 Format: "xai-responses-v1",
876 Summary: summary,
877 Index: inx,
878 })
879 }
880 reasoningDetails = append(reasoningDetails, ReasoningDetail{
881 Type: "reasoning.encrypted",
882 Format: "xai-responses-v1",
883 Data: *metadata.EncryptedContent,
884 ID: metadata.ItemID,
885 })
886 data, _ := json.Marshal(reasoningDetails)
887 reasoningDetailsMap := []map[string]any{}
888 _ = json.Unmarshal(data, &reasoningDetailsMap)
889 assistantMsg.SetExtraFields(map[string]any{
890 "reasoning_details": reasoningDetailsMap,
891 })
892 case strings.HasPrefix(model, "google/"):
893 metadata := google.GetReasoningMetadata(reasoningPart.Options())
894 if metadata == nil {
895 text := fmt.Sprintf("<thoughts>%s</thoughts>", reasoningPart.Text)
896 if assistantMsg.Content.OfString.Valid() {
897 text = assistantMsg.Content.OfString.Value + "\n" + text
898 }
899 // this reasoning did not come from anthropic just add a text content
900 assistantMsg.Content = openaisdk.ChatCompletionAssistantMessageParamContentUnion{
901 OfString: param.NewOpt(text),
902 }
903 continue
904 }
905 if reasoningPart.Text != "" {
906 reasoningDetails = append(reasoningDetails, ReasoningDetail{
907 Type: "reasoning.text",
908 Format: "google-gemini-v1",
909 Text: reasoningPart.Text,
910 })
911 }
912 reasoningDetails = append(reasoningDetails, ReasoningDetail{
913 Type: "reasoning.encrypted",
914 Format: "google-gemini-v1",
915 Data: metadata.Signature,
916 ID: metadata.ToolID,
917 })
918 data, _ := json.Marshal(reasoningDetails)
919 reasoningDetailsMap := []map[string]any{}
920 _ = json.Unmarshal(data, &reasoningDetailsMap)
921 assistantMsg.SetExtraFields(map[string]any{
922 "reasoning_details": reasoningDetailsMap,
923 })
924 default:
925 reasoningDetails = append(reasoningDetails, ReasoningDetail{
926 Type: "reasoning.text",
927 Text: reasoningPart.Text,
928 Format: "unknown",
929 })
930 data, _ := json.Marshal(reasoningDetails)
931 reasoningDetailsMap := []map[string]any{}
932 _ = json.Unmarshal(data, &reasoningDetailsMap)
933 assistantMsg.SetExtraFields(map[string]any{
934 "reasoning_details": reasoningDetailsMap,
935 })
936 }
937 case fantasy.ContentTypeToolCall:
938 toolCallPart, ok := fantasy.AsContentType[fantasy.ToolCallPart](c)
939 if !ok {
940 warnings = append(warnings, fantasy.CallWarning{
941 Type: fantasy.CallWarningTypeOther,
942 Message: "assistant message tool part does not have the right type",
943 })
944 continue
945 }
946 tc := openaisdk.ChatCompletionMessageToolCallUnionParam{
947 OfFunction: &openaisdk.ChatCompletionMessageFunctionToolCallParam{
948 ID: toolCallPart.ToolCallID,
949 Type: "function",
950 Function: openaisdk.ChatCompletionMessageFunctionToolCallFunctionParam{
951 Name: toolCallPart.ToolName,
952 Arguments: toolCallPart.Input,
953 },
954 },
955 }
956 if cacheControl != nil {
957 tc.OfFunction.SetExtraFields(map[string]any{
958 "cache_control": map[string]string{
959 "type": cacheControl.Type,
960 },
961 })
962 }
963 assistantMsg.ToolCalls = append(assistantMsg.ToolCalls, tc)
964 }
965 }
966 messages = append(messages, openaisdk.ChatCompletionMessageParamUnion{
967 OfAssistant: &assistantMsg,
968 })
969 case fantasy.MessageRoleTool:
970 for i, c := range msg.Content {
971 isLastPart := i == len(msg.Content)-1
972 cacheControl := anthropic.GetCacheControl(c.Options())
973 if cacheControl == nil && isLastPart {
974 cacheControl = anthropic.GetCacheControl(msg.ProviderOptions)
975 }
976 if c.GetType() != fantasy.ContentTypeToolResult {
977 warnings = append(warnings, fantasy.CallWarning{
978 Type: fantasy.CallWarningTypeOther,
979 Message: "tool message can only have tool result content",
980 })
981 continue
982 }
983
984 toolResultPart, ok := fantasy.AsContentType[fantasy.ToolResultPart](c)
985 if !ok {
986 warnings = append(warnings, fantasy.CallWarning{
987 Type: fantasy.CallWarningTypeOther,
988 Message: "tool message result part does not have the right type",
989 })
990 continue
991 }
992
993 switch toolResultPart.Output.GetType() {
994 case fantasy.ToolResultContentTypeText:
995 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](toolResultPart.Output)
996 if !ok {
997 warnings = append(warnings, fantasy.CallWarning{
998 Type: fantasy.CallWarningTypeOther,
999 Message: "tool result output does not have the right type",
1000 })
1001 continue
1002 }
1003 tr := openaisdk.ToolMessage(output.Text, toolResultPart.ToolCallID)
1004 if cacheControl != nil {
1005 tr.SetExtraFields(map[string]any{
1006 "cache_control": map[string]string{
1007 "type": cacheControl.Type,
1008 },
1009 })
1010 }
1011 messages = append(messages, tr)
1012 case fantasy.ToolResultContentTypeError:
1013 // TODO: check if better handling is needed
1014 output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentError](toolResultPart.Output)
1015 if !ok {
1016 warnings = append(warnings, fantasy.CallWarning{
1017 Type: fantasy.CallWarningTypeOther,
1018 Message: "tool result output does not have the right type",
1019 })
1020 continue
1021 }
1022 tr := openaisdk.ToolMessage(output.Error.Error(), toolResultPart.ToolCallID)
1023 if cacheControl != nil {
1024 tr.SetExtraFields(map[string]any{
1025 "cache_control": map[string]string{
1026 "type": cacheControl.Type,
1027 },
1028 })
1029 }
1030 messages = append(messages, tr)
1031 }
1032 }
1033 }
1034 }
1035 return messages, warnings
1036}