1// Package agent is the core orchestration layer for Crush AI agents.
2//
3// It provides session-based AI agent functionality for managing
4// conversations, tool execution, and message handling. It coordinates
5// interactions between language models, messages, sessions, and tools while
6// handling features like automatic summarization, queuing, and token
7// management.
8package agent
9
10import (
11 "cmp"
12 "context"
13 _ "embed"
14 "encoding/base64"
15 "errors"
16 "fmt"
17 "log/slog"
18 "net/http"
19 "os"
20 "regexp"
21 "strconv"
22 "strings"
23 "sync"
24 "time"
25
26 "charm.land/catwalk/pkg/catwalk"
27 "charm.land/fantasy"
28 "charm.land/fantasy/providers/anthropic"
29 "charm.land/fantasy/providers/bedrock"
30 "charm.land/fantasy/providers/google"
31 "charm.land/fantasy/providers/openai"
32 "charm.land/fantasy/providers/openrouter"
33 "charm.land/fantasy/providers/vercel"
34 "charm.land/lipgloss/v2"
35 "github.com/charmbracelet/crush/internal/agent/hyper"
36 "github.com/charmbracelet/crush/internal/agent/notify"
37 "github.com/charmbracelet/crush/internal/agent/tools"
38 "github.com/charmbracelet/crush/internal/agent/tools/mcp"
39 "github.com/charmbracelet/crush/internal/config"
40 "github.com/charmbracelet/crush/internal/csync"
41 "github.com/charmbracelet/crush/internal/message"
42 "github.com/charmbracelet/crush/internal/pubsub"
43 "github.com/charmbracelet/crush/internal/session"
44 "github.com/charmbracelet/crush/internal/stringext"
45 "github.com/charmbracelet/crush/internal/version"
46 "github.com/charmbracelet/x/exp/charmtone"
47)
48
49const (
50 DefaultSessionName = "Untitled Session"
51
52 // Constants for auto-summarization thresholds
53 largeContextWindowThreshold = 200_000
54 largeContextWindowBuffer = 20_000
55 smallContextWindowRatio = 0.2
56)
57
58var userAgent = fmt.Sprintf("Charm-Crush/%s (https://charm.land/crush)", version.Version)
59
60//go:embed templates/title.md
61var titlePrompt []byte
62
63//go:embed templates/summary.md
64var summaryPrompt []byte
65
66// Used to remove <think> tags from generated titles.
67var (
68 thinkTagRegex = regexp.MustCompile(`(?s)<think>.*?</think>`)
69 orphanThinkTagRegex = regexp.MustCompile(`</?think>`)
70)
71
72type SessionAgentCall struct {
73 SessionID string
74 Prompt string
75 ProviderOptions fantasy.ProviderOptions
76 Attachments []message.Attachment
77 MaxOutputTokens int64
78 Temperature *float64
79 TopP *float64
80 TopK *int64
81 FrequencyPenalty *float64
82 PresencePenalty *float64
83 NonInteractive bool
84}
85
86type SessionAgent interface {
87 Run(context.Context, SessionAgentCall) (*fantasy.AgentResult, error)
88 SetModels(large Model, small Model)
89 SetTools(tools []fantasy.AgentTool)
90 SetSystemPrompt(systemPrompt string)
91 Cancel(sessionID string)
92 CancelAll()
93 IsSessionBusy(sessionID string) bool
94 IsBusy() bool
95 QueuedPrompts(sessionID string) int
96 QueuedPromptsList(sessionID string) []string
97 ClearQueue(sessionID string)
98 Summarize(context.Context, string, fantasy.ProviderOptions) error
99 Model() Model
100}
101
102type Model struct {
103 Model fantasy.LanguageModel
104 CatwalkCfg catwalk.Model
105 ModelCfg config.SelectedModel
106 FlatRate bool
107}
108
109type sessionAgent struct {
110 largeModel *csync.Value[Model]
111 smallModel *csync.Value[Model]
112 systemPromptPrefix *csync.Value[string]
113 systemPrompt *csync.Value[string]
114 tools *csync.Slice[fantasy.AgentTool]
115
116 isSubAgent bool
117 sessions session.Service
118 messages message.Service
119 disableAutoSummarize bool
120 isYolo bool
121 notify pubsub.Publisher[notify.Notification]
122
123 messageQueue *csync.Map[string, []SessionAgentCall]
124 activeRequests *csync.Map[string, context.CancelFunc]
125}
126
127type SessionAgentOptions struct {
128 LargeModel Model
129 SmallModel Model
130 SystemPromptPrefix string
131 SystemPrompt string
132 IsSubAgent bool
133 DisableAutoSummarize bool
134 IsYolo bool
135 Sessions session.Service
136 Messages message.Service
137 Tools []fantasy.AgentTool
138 Notify pubsub.Publisher[notify.Notification]
139}
140
141func NewSessionAgent(
142 opts SessionAgentOptions,
143) SessionAgent {
144 return &sessionAgent{
145 largeModel: csync.NewValue(opts.LargeModel),
146 smallModel: csync.NewValue(opts.SmallModel),
147 systemPromptPrefix: csync.NewValue(opts.SystemPromptPrefix),
148 systemPrompt: csync.NewValue(opts.SystemPrompt),
149 isSubAgent: opts.IsSubAgent,
150 sessions: opts.Sessions,
151 messages: opts.Messages,
152 disableAutoSummarize: opts.DisableAutoSummarize,
153 tools: csync.NewSliceFrom(opts.Tools),
154 isYolo: opts.IsYolo,
155 notify: opts.Notify,
156 messageQueue: csync.NewMap[string, []SessionAgentCall](),
157 activeRequests: csync.NewMap[string, context.CancelFunc](),
158 }
159}
160
161func (a *sessionAgent) Run(ctx context.Context, call SessionAgentCall) (*fantasy.AgentResult, error) {
162 if call.Prompt == "" && !message.ContainsTextAttachment(call.Attachments) {
163 return nil, ErrEmptyPrompt
164 }
165 if call.SessionID == "" {
166 return nil, ErrSessionMissing
167 }
168
169 // Queue the message if busy
170 if a.IsSessionBusy(call.SessionID) {
171 existing, ok := a.messageQueue.Get(call.SessionID)
172 if !ok {
173 existing = []SessionAgentCall{}
174 }
175 existing = append(existing, call)
176 a.messageQueue.Set(call.SessionID, existing)
177 return nil, nil
178 }
179
180 // Copy mutable fields under lock to avoid races with SetTools/SetModels.
181 agentTools := a.tools.Copy()
182 largeModel := a.largeModel.Get()
183 systemPrompt := a.systemPrompt.Get()
184 promptPrefix := a.systemPromptPrefix.Get()
185 var instructions strings.Builder
186
187 for _, server := range mcp.GetStates() {
188 if server.State != mcp.StateConnected {
189 continue
190 }
191 if s := server.Client.InitializeResult().Instructions; s != "" {
192 instructions.WriteString(s)
193 instructions.WriteString("\n\n")
194 }
195 }
196
197 if s := instructions.String(); s != "" {
198 systemPrompt += "\n\n<mcp-instructions>\n" + s + "\n</mcp-instructions>"
199 }
200
201 if len(agentTools) > 0 {
202 // Add Anthropic caching to the last tool.
203 agentTools[len(agentTools)-1].SetProviderOptions(a.getCacheControlOptions())
204 }
205
206 agent := fantasy.NewAgent(
207 largeModel.Model,
208 fantasy.WithSystemPrompt(systemPrompt),
209 fantasy.WithTools(agentTools...),
210 fantasy.WithUserAgent(userAgent),
211 )
212
213 sessionLock := sync.Mutex{}
214 currentSession, err := a.sessions.Get(ctx, call.SessionID)
215 if err != nil {
216 return nil, fmt.Errorf("failed to get session: %w", err)
217 }
218
219 msgs, err := a.getSessionMessages(ctx, currentSession)
220 if err != nil {
221 return nil, fmt.Errorf("failed to get session messages: %w", err)
222 }
223
224 var wg sync.WaitGroup
225 // Generate title if first message.
226 if len(msgs) == 0 {
227 titleCtx := ctx // Copy to avoid race with ctx reassignment below.
228 wg.Go(func() {
229 a.generateTitle(titleCtx, call.SessionID, call.Prompt)
230 })
231 }
232 defer wg.Wait()
233
234 // Add the user message to the session.
235 _, err = a.createUserMessage(ctx, call)
236 if err != nil {
237 return nil, err
238 }
239
240 // Add the session to the context.
241 ctx = context.WithValue(ctx, tools.SessionIDContextKey, call.SessionID)
242
243 genCtx, cancel := context.WithCancel(ctx)
244 a.activeRequests.Set(call.SessionID, cancel)
245
246 defer cancel()
247 defer a.activeRequests.Del(call.SessionID)
248
249 history, files := a.preparePrompt(msgs, largeModel.CatwalkCfg.SupportsImages, call.Attachments...)
250
251 startTime := time.Now()
252 a.eventPromptSent(call.SessionID)
253
254 var currentAssistant *message.Message
255 var shouldSummarize bool
256 // Don't send MaxOutputTokens if 0 — some providers (e.g. LM Studio) reject it
257 var maxOutputTokens *int64
258 if call.MaxOutputTokens > 0 {
259 maxOutputTokens = &call.MaxOutputTokens
260 }
261 result, err := agent.Stream(genCtx, fantasy.AgentStreamCall{
262 Prompt: message.PromptWithTextAttachments(call.Prompt, call.Attachments),
263 Files: files,
264 Messages: history,
265 ProviderOptions: call.ProviderOptions,
266 MaxOutputTokens: maxOutputTokens,
267 TopP: call.TopP,
268 Temperature: call.Temperature,
269 PresencePenalty: call.PresencePenalty,
270 TopK: call.TopK,
271 FrequencyPenalty: call.FrequencyPenalty,
272 PrepareStep: func(callContext context.Context, options fantasy.PrepareStepFunctionOptions) (_ context.Context, prepared fantasy.PrepareStepResult, err error) {
273 prepared.Messages = options.Messages
274 for i := range prepared.Messages {
275 prepared.Messages[i].ProviderOptions = nil
276 }
277
278 // Use latest tools (updated by SetTools when MCP tools change).
279 prepared.Tools = a.tools.Copy()
280
281 queuedCalls, _ := a.messageQueue.Get(call.SessionID)
282 a.messageQueue.Del(call.SessionID)
283 for _, queued := range queuedCalls {
284 userMessage, createErr := a.createUserMessage(callContext, queued)
285 if createErr != nil {
286 return callContext, prepared, createErr
287 }
288 prepared.Messages = append(prepared.Messages, userMessage.ToAIMessage()...)
289 }
290
291 prepared.Messages = a.workaroundProviderMediaLimitations(prepared.Messages, largeModel)
292
293 lastSystemRoleInx := 0
294 systemMessageUpdated := false
295 for i, msg := range prepared.Messages {
296 // Only add cache control to the last message.
297 if msg.Role == fantasy.MessageRoleSystem {
298 lastSystemRoleInx = i
299 } else if !systemMessageUpdated {
300 prepared.Messages[lastSystemRoleInx].ProviderOptions = a.getCacheControlOptions()
301 systemMessageUpdated = true
302 }
303 // Than add cache control to the last 2 messages.
304 if i > len(prepared.Messages)-3 {
305 prepared.Messages[i].ProviderOptions = a.getCacheControlOptions()
306 }
307 }
308
309 if promptPrefix != "" {
310 prepared.Messages = append([]fantasy.Message{fantasy.NewSystemMessage(promptPrefix)}, prepared.Messages...)
311 }
312
313 var assistantMsg message.Message
314 assistantMsg, err = a.messages.Create(callContext, call.SessionID, message.CreateMessageParams{
315 Role: message.Assistant,
316 Parts: []message.ContentPart{},
317 Model: largeModel.ModelCfg.Model,
318 Provider: largeModel.ModelCfg.Provider,
319 })
320 if err != nil {
321 return callContext, prepared, err
322 }
323 callContext = context.WithValue(callContext, tools.MessageIDContextKey, assistantMsg.ID)
324 callContext = context.WithValue(callContext, tools.SupportsImagesContextKey, largeModel.CatwalkCfg.SupportsImages)
325 callContext = context.WithValue(callContext, tools.ModelNameContextKey, largeModel.CatwalkCfg.Name)
326 currentAssistant = &assistantMsg
327 return callContext, prepared, err
328 },
329 OnReasoningStart: func(id string, reasoning fantasy.ReasoningContent) error {
330 currentAssistant.AppendReasoningContent(reasoning.Text)
331 return a.messages.Update(genCtx, *currentAssistant)
332 },
333 OnReasoningDelta: func(id string, text string) error {
334 currentAssistant.AppendReasoningContent(text)
335 return a.messages.Update(genCtx, *currentAssistant)
336 },
337 OnReasoningEnd: func(id string, reasoning fantasy.ReasoningContent) error {
338 // handle anthropic signature
339 if anthropicData, ok := reasoning.ProviderMetadata[anthropic.Name]; ok {
340 if reasoning, ok := anthropicData.(*anthropic.ReasoningOptionMetadata); ok {
341 currentAssistant.AppendReasoningSignature(reasoning.Signature)
342 }
343 }
344 if googleData, ok := reasoning.ProviderMetadata[google.Name]; ok {
345 if reasoning, ok := googleData.(*google.ReasoningMetadata); ok {
346 currentAssistant.AppendThoughtSignature(reasoning.Signature, reasoning.ToolID)
347 }
348 }
349 if openaiData, ok := reasoning.ProviderMetadata[openai.Name]; ok {
350 if reasoning, ok := openaiData.(*openai.ResponsesReasoningMetadata); ok {
351 currentAssistant.SetReasoningResponsesData(reasoning)
352 }
353 }
354 currentAssistant.FinishThinking()
355 return a.messages.Update(genCtx, *currentAssistant)
356 },
357 OnTextDelta: func(id string, text string) error {
358 // Strip leading newline from initial text content. This is is
359 // particularly important in non-interactive mode where leading
360 // newlines are very visible.
361 if len(currentAssistant.Parts) == 0 {
362 text = strings.TrimPrefix(text, "\n")
363 }
364
365 currentAssistant.AppendContent(text)
366 return a.messages.Update(genCtx, *currentAssistant)
367 },
368 OnToolInputStart: func(id string, toolName string) error {
369 toolCall := message.ToolCall{
370 ID: id,
371 Name: toolName,
372 ProviderExecuted: false,
373 Finished: false,
374 }
375 currentAssistant.AddToolCall(toolCall)
376 // Use parent ctx instead of genCtx to ensure the update succeeds
377 // even if the request is canceled mid-stream
378 return a.messages.Update(ctx, *currentAssistant)
379 },
380 OnRetry: func(err *fantasy.ProviderError, delay time.Duration) {
381 slog.Warn("Provider request failed, retrying", providerRetryLogFields(err, delay)...)
382 },
383 OnToolCall: func(tc fantasy.ToolCallContent) error {
384 toolCall := message.ToolCall{
385 ID: tc.ToolCallID,
386 Name: tc.ToolName,
387 Input: tc.Input,
388 ProviderExecuted: false,
389 Finished: true,
390 }
391 currentAssistant.AddToolCall(toolCall)
392 // Use parent ctx instead of genCtx to ensure the update succeeds
393 // even if the request is canceled mid-stream
394 return a.messages.Update(ctx, *currentAssistant)
395 },
396 OnToolResult: func(result fantasy.ToolResultContent) error {
397 toolResult := a.convertToToolResult(result)
398 // Use parent ctx instead of genCtx to ensure the message is created
399 // even if the request is canceled mid-stream
400 _, createMsgErr := a.messages.Create(ctx, currentAssistant.SessionID, message.CreateMessageParams{
401 Role: message.Tool,
402 Parts: []message.ContentPart{
403 toolResult,
404 },
405 })
406 return createMsgErr
407 },
408 OnStepFinish: func(stepResult fantasy.StepResult) error {
409 finishReason := message.FinishReasonUnknown
410 switch stepResult.FinishReason {
411 case fantasy.FinishReasonLength:
412 finishReason = message.FinishReasonMaxTokens
413 case fantasy.FinishReasonStop:
414 finishReason = message.FinishReasonEndTurn
415 case fantasy.FinishReasonToolCalls:
416 finishReason = message.FinishReasonToolUse
417 }
418 // If a tool result halted the turn (e.g. a hook halt or a
419 // permission denial), the step ends on FinishReasonToolCalls but
420 // the model will not be called again. Treat it as the end of the
421 // turn so the UI can render the assistant footer.
422 if finishReason == message.FinishReasonToolUse {
423 for _, tr := range stepResult.Content.ToolResults() {
424 if tr.StopTurn {
425 finishReason = message.FinishReasonEndTurn
426 break
427 }
428 }
429 }
430 currentAssistant.AddFinish(finishReason, "", "")
431 sessionLock.Lock()
432 defer sessionLock.Unlock()
433
434 updatedSession, getSessionErr := a.sessions.Get(ctx, call.SessionID)
435 if getSessionErr != nil {
436 return getSessionErr
437 }
438 a.updateSessionUsage(largeModel, &updatedSession, stepResult.Usage, a.openrouterCost(stepResult.ProviderMetadata))
439 _, sessionErr := a.sessions.Save(ctx, updatedSession)
440 if sessionErr != nil {
441 return sessionErr
442 }
443 currentSession = updatedSession
444 return a.messages.Update(genCtx, *currentAssistant)
445 },
446 StopWhen: []fantasy.StopCondition{
447 func(_ []fantasy.StepResult) bool {
448 cw := int64(largeModel.CatwalkCfg.ContextWindow)
449 // If context window is unknown (0), skip auto-summarize
450 // to avoid immediately truncating custom/local models.
451 if cw == 0 {
452 return false
453 }
454 tokens := currentSession.CompletionTokens + currentSession.PromptTokens
455 remaining := cw - tokens
456 var threshold int64
457 if cw > largeContextWindowThreshold {
458 threshold = largeContextWindowBuffer
459 } else {
460 threshold = int64(float64(cw) * smallContextWindowRatio)
461 }
462 if (remaining <= threshold) && !a.disableAutoSummarize {
463 shouldSummarize = true
464 return true
465 }
466 return false
467 },
468 func(steps []fantasy.StepResult) bool {
469 return hasRepeatedToolCalls(steps, loopDetectionWindowSize, loopDetectionMaxRepeats)
470 },
471 },
472 })
473
474 a.eventPromptResponded(call.SessionID, time.Since(startTime).Truncate(time.Second))
475
476 if err != nil {
477 isHyper := largeModel.ModelCfg.Provider == hyper.Name
478 isCancelErr := errors.Is(err, context.Canceled)
479 if currentAssistant == nil {
480 return result, err
481 }
482 // Ensure we finish thinking on error to close the reasoning state.
483 currentAssistant.FinishThinking()
484 toolCalls := currentAssistant.ToolCalls()
485 // INFO: we use the parent context here because the genCtx has been cancelled.
486 msgs, createErr := a.messages.List(ctx, currentAssistant.SessionID)
487 if createErr != nil {
488 return nil, createErr
489 }
490 for _, tc := range toolCalls {
491 if !tc.Finished {
492 tc.Finished = true
493 tc.Input = "{}"
494 currentAssistant.AddToolCall(tc)
495 updateErr := a.messages.Update(ctx, *currentAssistant)
496 if updateErr != nil {
497 return nil, updateErr
498 }
499 }
500
501 found := false
502 for _, msg := range msgs {
503 if msg.Role == message.Tool {
504 for _, tr := range msg.ToolResults() {
505 if tr.ToolCallID == tc.ID {
506 found = true
507 break
508 }
509 }
510 }
511 if found {
512 break
513 }
514 }
515 if found {
516 continue
517 }
518 content := "There was an error while executing the tool"
519 if isCancelErr {
520 content = "Error: user cancelled assistant tool calling"
521 }
522 toolResult := message.ToolResult{
523 ToolCallID: tc.ID,
524 Name: tc.Name,
525 Content: content,
526 IsError: true,
527 }
528 _, createErr = a.messages.Create(ctx, currentAssistant.SessionID, message.CreateMessageParams{
529 Role: message.Tool,
530 Parts: []message.ContentPart{
531 toolResult,
532 },
533 })
534 if createErr != nil {
535 return nil, createErr
536 }
537 }
538 var fantasyErr *fantasy.Error
539 var providerErr *fantasy.ProviderError
540 const defaultTitle = "Provider Error"
541 linkStyle := lipgloss.NewStyle().Foreground(charmtone.Guac).Underline(true)
542 if isCancelErr {
543 currentAssistant.AddFinish(message.FinishReasonCanceled, "User canceled request", "")
544 } else if isHyper && errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusUnauthorized {
545 currentAssistant.AddFinish(message.FinishReasonError, "Unauthorized", `Please re-authenticate with Hyper. You can also run "crush auth" to re-authenticate.`)
546 if a.notify != nil {
547 a.notify.Publish(pubsub.CreatedEvent, notify.Notification{
548 SessionID: call.SessionID,
549 SessionTitle: currentSession.Title,
550 Type: notify.TypeReAuthenticate,
551 ProviderID: largeModel.ModelCfg.Provider,
552 })
553 }
554 } else if isHyper && errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusPaymentRequired {
555 url := hyper.BaseURL()
556 link := linkStyle.Hyperlink(url, "id=hyper").Render(url)
557 currentAssistant.AddFinish(message.FinishReasonError, "No credits", "You're out of credits. Add more at "+link)
558 } else if errors.As(err, &providerErr) {
559 if providerErr.Message == "The requested model is not supported." {
560 url := "https://github.com/settings/copilot/features"
561 link := linkStyle.Hyperlink(url, "id=copilot").Render(url)
562 currentAssistant.AddFinish(
563 message.FinishReasonError,
564 "Copilot model not enabled",
565 fmt.Sprintf("%q is not enabled in Copilot. Go to the following page to enable it. Then, wait 5 minutes before trying again. %s", largeModel.CatwalkCfg.Name, link),
566 )
567 } else {
568 currentAssistant.AddFinish(message.FinishReasonError, cmp.Or(stringext.Capitalize(providerErr.Title), defaultTitle), providerErr.Message)
569 }
570 } else if errors.As(err, &fantasyErr) {
571 currentAssistant.AddFinish(message.FinishReasonError, cmp.Or(stringext.Capitalize(fantasyErr.Title), defaultTitle), fantasyErr.Message)
572 } else {
573 currentAssistant.AddFinish(message.FinishReasonError, defaultTitle, err.Error())
574 }
575 // Note: we use the parent context here because the genCtx has been
576 // cancelled.
577 updateErr := a.messages.Update(ctx, *currentAssistant)
578 if updateErr != nil {
579 return nil, updateErr
580 }
581 return nil, err
582 }
583
584 // Send notification that agent has finished its turn (skip for
585 // nested/non-interactive sessions).
586 if !call.NonInteractive && a.notify != nil {
587 a.notify.Publish(pubsub.CreatedEvent, notify.Notification{
588 SessionID: call.SessionID,
589 SessionTitle: currentSession.Title,
590 Type: notify.TypeAgentFinished,
591 })
592 }
593
594 if shouldSummarize {
595 a.activeRequests.Del(call.SessionID)
596 if summarizeErr := a.Summarize(genCtx, call.SessionID, call.ProviderOptions); summarizeErr != nil {
597 return nil, summarizeErr
598 }
599 // If the agent wasn't done...
600 if len(currentAssistant.ToolCalls()) > 0 {
601 existing, ok := a.messageQueue.Get(call.SessionID)
602 if !ok {
603 existing = []SessionAgentCall{}
604 }
605 call.Prompt = fmt.Sprintf("The previous session was interrupted because it got too long, the initial user request was: `%s`", call.Prompt)
606 existing = append(existing, call)
607 a.messageQueue.Set(call.SessionID, existing)
608 }
609 }
610
611 // Release active request before processing queued messages.
612 a.activeRequests.Del(call.SessionID)
613 cancel()
614
615 queuedMessages, ok := a.messageQueue.Get(call.SessionID)
616 if !ok || len(queuedMessages) == 0 {
617 return result, err
618 }
619 // There are queued messages restart the loop.
620 firstQueuedMessage := queuedMessages[0]
621 a.messageQueue.Set(call.SessionID, queuedMessages[1:])
622 return a.Run(ctx, firstQueuedMessage)
623}
624
625func (a *sessionAgent) Summarize(ctx context.Context, sessionID string, opts fantasy.ProviderOptions) error {
626 if a.IsSessionBusy(sessionID) {
627 return ErrSessionBusy
628 }
629
630 // Copy mutable fields under lock to avoid races with SetModels.
631 largeModel := a.largeModel.Get()
632 systemPromptPrefix := a.systemPromptPrefix.Get()
633
634 currentSession, err := a.sessions.Get(ctx, sessionID)
635 if err != nil {
636 return fmt.Errorf("failed to get session: %w", err)
637 }
638 msgs, err := a.getSessionMessages(ctx, currentSession)
639 if err != nil {
640 return err
641 }
642 if len(msgs) == 0 {
643 // Nothing to summarize.
644 return nil
645 }
646
647 aiMsgs, _ := a.preparePrompt(msgs, largeModel.CatwalkCfg.SupportsImages)
648
649 genCtx, cancel := context.WithCancel(ctx)
650 a.activeRequests.Set(sessionID, cancel)
651 defer a.activeRequests.Del(sessionID)
652 defer cancel()
653
654 agent := fantasy.NewAgent(largeModel.Model,
655 fantasy.WithSystemPrompt(string(summaryPrompt)),
656 fantasy.WithUserAgent(userAgent),
657 )
658 summaryMessage, err := a.messages.Create(ctx, sessionID, message.CreateMessageParams{
659 Role: message.Assistant,
660 Model: largeModel.Model.Model(),
661 Provider: largeModel.Model.Provider(),
662 IsSummaryMessage: true,
663 })
664 if err != nil {
665 return err
666 }
667
668 summaryPromptText := buildSummaryPrompt(currentSession.Todos)
669
670 resp, err := agent.Stream(genCtx, fantasy.AgentStreamCall{
671 Prompt: summaryPromptText,
672 Messages: aiMsgs,
673 ProviderOptions: opts,
674 PrepareStep: func(callContext context.Context, options fantasy.PrepareStepFunctionOptions) (_ context.Context, prepared fantasy.PrepareStepResult, err error) {
675 prepared.Messages = options.Messages
676 if systemPromptPrefix != "" {
677 prepared.Messages = append([]fantasy.Message{fantasy.NewSystemMessage(systemPromptPrefix)}, prepared.Messages...)
678 }
679 return callContext, prepared, nil
680 },
681 OnReasoningDelta: func(id string, text string) error {
682 summaryMessage.AppendReasoningContent(text)
683 return a.messages.Update(genCtx, summaryMessage)
684 },
685 OnReasoningEnd: func(id string, reasoning fantasy.ReasoningContent) error {
686 // Handle anthropic signature.
687 if anthropicData, ok := reasoning.ProviderMetadata["anthropic"]; ok {
688 if signature, ok := anthropicData.(*anthropic.ReasoningOptionMetadata); ok && signature.Signature != "" {
689 summaryMessage.AppendReasoningSignature(signature.Signature)
690 }
691 }
692 summaryMessage.FinishThinking()
693 return a.messages.Update(genCtx, summaryMessage)
694 },
695 OnTextDelta: func(id, text string) error {
696 summaryMessage.AppendContent(text)
697 return a.messages.Update(genCtx, summaryMessage)
698 },
699 })
700 if err != nil {
701 isCancelErr := errors.Is(err, context.Canceled)
702 if isCancelErr {
703 // User cancelled summarize we need to remove the summary message.
704 deleteErr := a.messages.Delete(ctx, summaryMessage.ID)
705 return deleteErr
706 }
707 // Mark the summary message as finished with an error so the UI
708 // stops spinning.
709 summaryMessage.AddFinish(message.FinishReasonError, "Summarization Error", err.Error())
710 if updateErr := a.messages.Update(ctx, summaryMessage); updateErr != nil {
711 return updateErr
712 }
713 return err
714 }
715
716 summaryMessage.AddFinish(message.FinishReasonEndTurn, "", "")
717 err = a.messages.Update(genCtx, summaryMessage)
718 if err != nil {
719 return err
720 }
721
722 var openrouterCost *float64
723 for _, step := range resp.Steps {
724 stepCost := a.openrouterCost(step.ProviderMetadata)
725 if stepCost != nil {
726 newCost := *stepCost
727 if openrouterCost != nil {
728 newCost += *openrouterCost
729 }
730 openrouterCost = &newCost
731 }
732 }
733
734 a.updateSessionUsage(largeModel, ¤tSession, resp.TotalUsage, openrouterCost)
735
736 // Just in case, get just the last usage info.
737 usage := resp.Response.Usage
738 currentSession.SummaryMessageID = summaryMessage.ID
739 currentSession.CompletionTokens = usage.OutputTokens
740 currentSession.PromptTokens = 0
741 _, err = a.sessions.Save(genCtx, currentSession)
742 if err != nil {
743 return err
744 }
745
746 // Release the active request before processing queued messages so that
747 // Run() does not see the session as busy.
748 a.activeRequests.Del(sessionID)
749 cancel()
750
751 // Process any messages that were queued while summarizing.
752 queuedMessages, ok := a.messageQueue.Get(sessionID)
753 if !ok || len(queuedMessages) == 0 {
754 return nil
755 }
756 firstQueuedMessage := queuedMessages[0]
757 a.messageQueue.Set(sessionID, queuedMessages[1:])
758 _, qErr := a.Run(ctx, firstQueuedMessage)
759 return qErr
760}
761
762func (a *sessionAgent) getCacheControlOptions() fantasy.ProviderOptions {
763 if t, _ := strconv.ParseBool(os.Getenv("CRUSH_DISABLE_ANTHROPIC_CACHE")); t {
764 return fantasy.ProviderOptions{}
765 }
766 return fantasy.ProviderOptions{
767 anthropic.Name: &anthropic.ProviderCacheControlOptions{
768 CacheControl: anthropic.CacheControl{Type: "ephemeral"},
769 },
770 bedrock.Name: &anthropic.ProviderCacheControlOptions{
771 CacheControl: anthropic.CacheControl{Type: "ephemeral"},
772 },
773 vercel.Name: &anthropic.ProviderCacheControlOptions{
774 CacheControl: anthropic.CacheControl{Type: "ephemeral"},
775 },
776 }
777}
778
779func (a *sessionAgent) createUserMessage(ctx context.Context, call SessionAgentCall) (message.Message, error) {
780 parts := []message.ContentPart{message.TextContent{Text: call.Prompt}}
781 var attachmentParts []message.ContentPart
782 for _, attachment := range call.Attachments {
783 attachmentParts = append(attachmentParts, message.BinaryContent{Path: attachment.FilePath, MIMEType: attachment.MimeType, Data: attachment.Content})
784 }
785 parts = append(parts, attachmentParts...)
786 msg, err := a.messages.Create(ctx, call.SessionID, message.CreateMessageParams{
787 Role: message.User,
788 Parts: parts,
789 })
790 if err != nil {
791 return message.Message{}, fmt.Errorf("failed to create user message: %w", err)
792 }
793 return msg, nil
794}
795
796func (a *sessionAgent) preparePrompt(msgs []message.Message, supportsImages bool, attachments ...message.Attachment) ([]fantasy.Message, []fantasy.FilePart) {
797 var history []fantasy.Message
798 if !a.isSubAgent {
799 history = append(history, fantasy.NewUserMessage(
800 fmt.Sprintf("<system_reminder>%s</system_reminder>",
801 `This is a reminder that your todo list is currently empty. DO NOT mention this to the user explicitly because they are already aware.
802If you are working on tasks that would benefit from a todo list please use the "todos" tool to create one.
803If not, please feel free to ignore. Again do not mention this message to the user.`,
804 ),
805 ))
806 }
807 // Collect all tool call IDs present in assistant messages and all tool
808 // result IDs present in tool messages. This lets us detect both orphaned
809 // tool results (result without a call) and orphaned tool calls (call
810 // without a result).
811 knownToolCallIDs := make(map[string]struct{})
812 knownToolResultIDs := make(map[string]struct{})
813 for _, m := range msgs {
814 switch m.Role {
815 case message.Assistant:
816 for _, tc := range m.ToolCalls() {
817 knownToolCallIDs[tc.ID] = struct{}{}
818 }
819 case message.Tool:
820 for _, tr := range m.ToolResults() {
821 knownToolResultIDs[tr.ToolCallID] = struct{}{}
822 }
823 }
824 }
825
826 for _, m := range msgs {
827 if len(m.Parts) == 0 {
828 continue
829 }
830 // Assistant message without content or tool calls (cancelled before it returned anything).
831 if m.Role == message.Assistant && len(m.ToolCalls()) == 0 && m.Content().Text == "" && m.ReasoningContent().String() == "" {
832 continue
833 }
834 if m.Role == message.Tool {
835 if msg, ok := filterOrphanedToolResults(m, knownToolCallIDs); ok {
836 history = append(history, msg)
837 }
838 continue
839 }
840 aiMsgs := m.ToAIMessage()
841 if !supportsImages {
842 for i := range aiMsgs {
843 if aiMsgs[i].Role == fantasy.MessageRoleUser {
844 aiMsgs[i].Content = filterFileParts(aiMsgs[i].Content)
845 }
846 }
847 }
848 history = append(history, aiMsgs...)
849
850 if m.Role == message.Assistant {
851 if msg, ok := syntheticToolResultsForOrphanedCalls(m, knownToolResultIDs); ok {
852 history = append(history, msg)
853 }
854 }
855 }
856
857 var files []fantasy.FilePart
858 for _, attachment := range attachments {
859 if attachment.IsText() {
860 continue
861 }
862 files = append(files, fantasy.FilePart{
863 Filename: attachment.FileName,
864 Data: attachment.Content,
865 MediaType: attachment.MimeType,
866 })
867 }
868
869 return history, files
870}
871
872// filterFileParts removes fantasy.FilePart entries from a slice of message
873// parts. Used to strip image attachments from historical user messages when
874// the current model does not support them.
875func filterFileParts(parts []fantasy.MessagePart) []fantasy.MessagePart {
876 filtered := make([]fantasy.MessagePart, 0, len(parts))
877 for _, part := range parts {
878 if _, ok := fantasy.AsMessagePart[fantasy.FilePart](part); ok {
879 continue
880 }
881 filtered = append(filtered, part)
882 }
883 return filtered
884}
885
886// filterOrphanedToolResults converts a tool message to a fantasy.Message,
887// dropping any tool result parts whose tool_call_id has no matching tool call
888// in the known set. An orphaned result causes API validation to fail on every
889// subsequent turn, permanently locking the session. Returns the filtered
890// message and true if at least one valid part remains.
891func filterOrphanedToolResults(m message.Message, knownToolCallIDs map[string]struct{}) (fantasy.Message, bool) {
892 aiMsgs := m.ToAIMessage()
893 if len(aiMsgs) == 0 {
894 return fantasy.Message{}, false
895 }
896 var validParts []fantasy.MessagePart
897 for _, part := range aiMsgs[0].Content {
898 tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part)
899 if !ok {
900 validParts = append(validParts, part)
901 continue
902 }
903 if _, known := knownToolCallIDs[tr.ToolCallID]; known {
904 validParts = append(validParts, part)
905 } else {
906 slog.Warn("Dropping orphaned tool result with no matching tool call",
907 "tool_call_id", tr.ToolCallID,
908 )
909 }
910 }
911 if len(validParts) == 0 {
912 return fantasy.Message{}, false
913 }
914 msg := aiMsgs[0]
915 msg.Content = validParts
916 return msg, true
917}
918
919// syntheticToolResultsForOrphanedCalls returns a tool message containing
920// synthetic tool results for any tool calls in the assistant message that
921// have no matching result in knownToolResultIDs. LLM APIs require every
922// tool_use to be immediately followed by a tool_result; an interrupted
923// session can leave orphaned tool_use blocks that permanently lock the
924// conversation. Returns the message and true if any synthetic results were
925// produced.
926func syntheticToolResultsForOrphanedCalls(m message.Message, knownToolResultIDs map[string]struct{}) (fantasy.Message, bool) {
927 var syntheticParts []fantasy.MessagePart
928 for _, tc := range m.ToolCalls() {
929 if _, hasResult := knownToolResultIDs[tc.ID]; hasResult {
930 continue
931 }
932 slog.Warn("Injecting synthetic tool result for orphaned tool call",
933 "tool_call_id", tc.ID,
934 "tool_name", tc.Name,
935 )
936 syntheticParts = append(syntheticParts, fantasy.ToolResultPart{
937 ToolCallID: tc.ID,
938 Output: fantasy.ToolResultOutputContentError{
939 Error: errors.New("tool call was interrupted and did not produce a result, you may retry this call if the result is still needed"),
940 },
941 })
942 }
943 if len(syntheticParts) == 0 {
944 return fantasy.Message{}, false
945 }
946 return fantasy.Message{
947 Role: fantasy.MessageRoleTool,
948 Content: syntheticParts,
949 }, true
950}
951
952func (a *sessionAgent) getSessionMessages(ctx context.Context, session session.Session) ([]message.Message, error) {
953 msgs, err := a.messages.List(ctx, session.ID)
954 if err != nil {
955 return nil, fmt.Errorf("failed to list messages: %w", err)
956 }
957
958 if session.SummaryMessageID != "" {
959 summaryMsgIndex := -1
960 for i, msg := range msgs {
961 if msg.ID == session.SummaryMessageID {
962 summaryMsgIndex = i
963 break
964 }
965 }
966 if summaryMsgIndex != -1 {
967 msgs = msgs[summaryMsgIndex:]
968 msgs[0].Role = message.User
969 }
970 }
971 return msgs, nil
972}
973
974// generateTitle generates a session titled based on the initial prompt.
975func (a *sessionAgent) generateTitle(ctx context.Context, sessionID string, userPrompt string) {
976 if userPrompt == "" {
977 return
978 }
979
980 smallModel := a.smallModel.Get()
981 largeModel := a.largeModel.Get()
982 systemPromptPrefix := a.systemPromptPrefix.Get()
983
984 var maxOutputTokens int64 = 40
985 if smallModel.CatwalkCfg.CanReason {
986 maxOutputTokens = smallModel.CatwalkCfg.DefaultMaxTokens
987 }
988
989 newAgent := func(m fantasy.LanguageModel, p []byte, tok int64) fantasy.Agent {
990 return fantasy.NewAgent(m,
991 fantasy.WithSystemPrompt(string(p)+"\n /no_think"),
992 fantasy.WithMaxOutputTokens(tok),
993 fantasy.WithUserAgent(userAgent),
994 )
995 }
996
997 streamCall := fantasy.AgentStreamCall{
998 Prompt: fmt.Sprintf("Generate a concise title for the following content:\n\n%s\n <think>\n\n</think>", userPrompt),
999 PrepareStep: func(callCtx context.Context, opts fantasy.PrepareStepFunctionOptions) (_ context.Context, prepared fantasy.PrepareStepResult, err error) {
1000 prepared.Messages = opts.Messages
1001 if systemPromptPrefix != "" {
1002 prepared.Messages = append([]fantasy.Message{
1003 fantasy.NewSystemMessage(systemPromptPrefix),
1004 }, prepared.Messages...)
1005 }
1006 return callCtx, prepared, nil
1007 },
1008 }
1009
1010 // Use the small model to generate the title.
1011 model := smallModel
1012 agent := newAgent(model.Model, titlePrompt, maxOutputTokens)
1013 resp, err := agent.Stream(ctx, streamCall)
1014 if err == nil {
1015 // We successfully generated a title with the small model.
1016 slog.Debug("Generated title with small model")
1017 } else {
1018 // It didn't work. Let's try with the big model.
1019 slog.Error("Error generating title with small model; trying big model", "err", err)
1020 model = largeModel
1021 agent = newAgent(model.Model, titlePrompt, maxOutputTokens)
1022 resp, err = agent.Stream(ctx, streamCall)
1023 if err == nil {
1024 slog.Debug("Generated title with large model")
1025 } else {
1026 // Welp, the large model didn't work either. Use the default
1027 // session name and return.
1028 slog.Error("Error generating title with large model", "err", err)
1029 saveErr := a.sessions.Rename(ctx, sessionID, DefaultSessionName)
1030 if saveErr != nil {
1031 slog.Error("Failed to save session title", "error", saveErr)
1032 }
1033 return
1034 }
1035 }
1036
1037 if resp == nil {
1038 // Actually, we didn't get a response so we can't. Use the default
1039 // session name and return.
1040 slog.Error("Response is nil; can't generate title")
1041 saveErr := a.sessions.Rename(ctx, sessionID, DefaultSessionName)
1042 if saveErr != nil {
1043 slog.Error("Failed to save session title", "error", saveErr)
1044 }
1045 return
1046 }
1047
1048 // Clean up title.
1049 var title string
1050 title = strings.ReplaceAll(resp.Response.Content.Text(), "\n", " ")
1051
1052 // Remove thinking tags if present.
1053 title = thinkTagRegex.ReplaceAllString(title, "")
1054 title = orphanThinkTagRegex.ReplaceAllString(title, "")
1055
1056 title = strings.TrimSpace(title)
1057 title = cmp.Or(title, DefaultSessionName)
1058
1059 // Calculate usage and cost.
1060 var openrouterCost *float64
1061 for _, step := range resp.Steps {
1062 stepCost := a.openrouterCost(step.ProviderMetadata)
1063 if stepCost != nil {
1064 newCost := *stepCost
1065 if openrouterCost != nil {
1066 newCost += *openrouterCost
1067 }
1068 openrouterCost = &newCost
1069 }
1070 }
1071
1072 modelConfig := model.CatwalkCfg
1073 cost := modelConfig.CostPer1MInCached/1e6*float64(resp.TotalUsage.CacheCreationTokens) +
1074 modelConfig.CostPer1MOutCached/1e6*float64(resp.TotalUsage.CacheReadTokens) +
1075 modelConfig.CostPer1MIn/1e6*float64(resp.TotalUsage.InputTokens) +
1076 modelConfig.CostPer1MOut/1e6*float64(resp.TotalUsage.OutputTokens)
1077
1078 // Use override cost if available (e.g., from OpenRouter).
1079 if openrouterCost != nil {
1080 cost = *openrouterCost
1081 }
1082
1083 // Skip cost accumulation
1084 if model.FlatRate {
1085 cost = 0
1086 }
1087
1088 promptTokens := resp.TotalUsage.InputTokens + resp.TotalUsage.CacheCreationTokens
1089 completionTokens := resp.TotalUsage.OutputTokens
1090
1091 // Atomically update only title and usage fields to avoid overriding other
1092 // concurrent session updates.
1093 saveErr := a.sessions.UpdateTitleAndUsage(ctx, sessionID, title, promptTokens, completionTokens, cost)
1094 if saveErr != nil {
1095 slog.Error("Failed to save session title and usage", "error", saveErr)
1096 return
1097 }
1098}
1099
1100func (a *sessionAgent) openrouterCost(metadata fantasy.ProviderMetadata) *float64 {
1101 openrouterMetadata, ok := metadata[openrouter.Name]
1102 if !ok {
1103 return nil
1104 }
1105
1106 opts, ok := openrouterMetadata.(*openrouter.ProviderMetadata)
1107 if !ok {
1108 return nil
1109 }
1110 return &opts.Usage.Cost
1111}
1112
1113func (a *sessionAgent) updateSessionUsage(model Model, session *session.Session, usage fantasy.Usage, overrideCost *float64) {
1114 modelConfig := model.CatwalkCfg
1115 cost := modelConfig.CostPer1MInCached/1e6*float64(usage.CacheCreationTokens) +
1116 modelConfig.CostPer1MOutCached/1e6*float64(usage.CacheReadTokens) +
1117 modelConfig.CostPer1MIn/1e6*float64(usage.InputTokens) +
1118 modelConfig.CostPer1MOut/1e6*float64(usage.OutputTokens)
1119
1120 a.eventTokensUsed(session.ID, model, usage, cost)
1121
1122 // Use override cost if available (e.g., from OpenRouter).
1123 if overrideCost != nil {
1124 cost = *overrideCost
1125 }
1126
1127 // Skip cost accumulation
1128 if model.FlatRate {
1129 cost = 0
1130 }
1131
1132 session.Cost += cost
1133 session.CompletionTokens = usage.OutputTokens
1134 session.PromptTokens = usage.InputTokens + usage.CacheReadTokens
1135}
1136
1137func (a *sessionAgent) Cancel(sessionID string) {
1138 // Cancel regular requests. Don't use Take() here - we need the entry to
1139 // remain in activeRequests so IsBusy() returns true until the goroutine
1140 // fully completes (including error handling that may access the DB).
1141 // The defer in processRequest will clean up the entry.
1142 if cancel, ok := a.activeRequests.Get(sessionID); ok && cancel != nil {
1143 slog.Debug("Request cancellation initiated", "session_id", sessionID)
1144 cancel()
1145 }
1146
1147 // Also check for summarize requests.
1148 if cancel, ok := a.activeRequests.Get(sessionID + "-summarize"); ok && cancel != nil {
1149 slog.Debug("Summarize cancellation initiated", "session_id", sessionID)
1150 cancel()
1151 }
1152
1153 if a.QueuedPrompts(sessionID) > 0 {
1154 slog.Debug("Clearing queued prompts", "session_id", sessionID)
1155 a.messageQueue.Del(sessionID)
1156 }
1157}
1158
1159func (a *sessionAgent) ClearQueue(sessionID string) {
1160 if a.QueuedPrompts(sessionID) > 0 {
1161 slog.Debug("Clearing queued prompts", "session_id", sessionID)
1162 a.messageQueue.Del(sessionID)
1163 }
1164}
1165
1166func (a *sessionAgent) CancelAll() {
1167 if !a.IsBusy() {
1168 return
1169 }
1170 for key := range a.activeRequests.Seq2() {
1171 a.Cancel(key) // key is sessionID
1172 }
1173
1174 timeout := time.After(5 * time.Second)
1175 for a.IsBusy() {
1176 select {
1177 case <-timeout:
1178 return
1179 default:
1180 time.Sleep(200 * time.Millisecond)
1181 }
1182 }
1183}
1184
1185func (a *sessionAgent) IsBusy() bool {
1186 var busy bool
1187 for cancelFunc := range a.activeRequests.Seq() {
1188 if cancelFunc != nil {
1189 busy = true
1190 break
1191 }
1192 }
1193 return busy
1194}
1195
1196func (a *sessionAgent) IsSessionBusy(sessionID string) bool {
1197 _, busy := a.activeRequests.Get(sessionID)
1198 return busy
1199}
1200
1201func (a *sessionAgent) QueuedPrompts(sessionID string) int {
1202 l, ok := a.messageQueue.Get(sessionID)
1203 if !ok {
1204 return 0
1205 }
1206 return len(l)
1207}
1208
1209func (a *sessionAgent) QueuedPromptsList(sessionID string) []string {
1210 l, ok := a.messageQueue.Get(sessionID)
1211 if !ok {
1212 return nil
1213 }
1214 prompts := make([]string, len(l))
1215 for i, call := range l {
1216 prompts[i] = call.Prompt
1217 }
1218 return prompts
1219}
1220
1221func (a *sessionAgent) SetModels(large Model, small Model) {
1222 a.largeModel.Set(large)
1223 a.smallModel.Set(small)
1224}
1225
1226func (a *sessionAgent) SetTools(tools []fantasy.AgentTool) {
1227 a.tools.SetSlice(tools)
1228}
1229
1230func (a *sessionAgent) SetSystemPrompt(systemPrompt string) {
1231 a.systemPrompt.Set(systemPrompt)
1232}
1233
1234func (a *sessionAgent) Model() Model {
1235 return a.largeModel.Get()
1236}
1237
1238// convertToToolResult converts a fantasy tool result to a message tool result.
1239func (a *sessionAgent) convertToToolResult(result fantasy.ToolResultContent) message.ToolResult {
1240 baseResult := message.ToolResult{
1241 ToolCallID: result.ToolCallID,
1242 Name: result.ToolName,
1243 Metadata: result.ClientMetadata,
1244 }
1245
1246 switch result.Result.GetType() {
1247 case fantasy.ToolResultContentTypeText:
1248 if r, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](result.Result); ok {
1249 baseResult.Content = r.Text
1250 }
1251 case fantasy.ToolResultContentTypeError:
1252 if r, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentError](result.Result); ok {
1253 baseResult.Content = r.Error.Error()
1254 baseResult.IsError = true
1255 }
1256 case fantasy.ToolResultContentTypeMedia:
1257 if r, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](result.Result); ok {
1258 if !stringext.IsValidBase64(r.Data) {
1259 slog.Warn("Tool returned media with invalid base64 data, discarding image",
1260 "tool", result.ToolName,
1261 "tool_call_id", result.ToolCallID,
1262 )
1263 baseResult.Content = "Tool returned image data with invalid encoding"
1264 baseResult.IsError = true
1265 } else {
1266 content := r.Text
1267 if content == "" {
1268 content = fmt.Sprintf("Loaded %s content", r.MediaType)
1269 }
1270 baseResult.Content = content
1271 baseResult.Data = r.Data
1272 baseResult.MIMEType = r.MediaType
1273 }
1274 }
1275 }
1276
1277 return baseResult
1278}
1279
1280// workaroundProviderMediaLimitations converts media content in tool results to
1281// user messages for providers that don't natively support images in tool results.
1282//
1283// Problem: OpenAI, Google, OpenRouter, and other OpenAI-compatible providers
1284// don't support sending images/media in tool result messages - they only accept
1285// text in tool results. However, they DO support images in user messages.
1286//
1287// If we send media in tool results to these providers, the API returns an error.
1288//
1289// Solution: For these providers, we:
1290// 1. Replace the media in the tool result with a text placeholder
1291// 2. Inject a user message immediately after with the image as a file attachment
1292// 3. This maintains the tool execution flow while working around API limitations
1293//
1294// Anthropic and Bedrock support images natively in tool results, so we skip
1295// this workaround for them.
1296//
1297// Example transformation:
1298//
1299// BEFORE: [tool result: image data]
1300// AFTER: [tool result: "Image loaded - see attached"], [user: image attachment]
1301func (a *sessionAgent) workaroundProviderMediaLimitations(messages []fantasy.Message, largeModel Model) []fantasy.Message {
1302 providerSupportsMedia := largeModel.ModelCfg.Provider == string(catwalk.InferenceProviderAnthropic) ||
1303 largeModel.ModelCfg.Provider == string(catwalk.InferenceProviderBedrock)
1304
1305 if providerSupportsMedia {
1306 return messages
1307 }
1308
1309 convertedMessages := make([]fantasy.Message, 0, len(messages))
1310
1311 for _, msg := range messages {
1312 if msg.Role != fantasy.MessageRoleTool {
1313 convertedMessages = append(convertedMessages, msg)
1314 continue
1315 }
1316
1317 textParts := make([]fantasy.MessagePart, 0, len(msg.Content))
1318 var mediaFiles []fantasy.FilePart
1319
1320 for _, part := range msg.Content {
1321 toolResult, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part)
1322 if !ok {
1323 textParts = append(textParts, part)
1324 continue
1325 }
1326
1327 if media, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](toolResult.Output); ok {
1328 decoded, err := base64.StdEncoding.DecodeString(media.Data)
1329 if err != nil {
1330 slog.Warn("Failed to decode media data", "error", err)
1331 textParts = append(textParts, part)
1332 continue
1333 }
1334
1335 mediaFiles = append(mediaFiles, fantasy.FilePart{
1336 Data: decoded,
1337 MediaType: media.MediaType,
1338 Filename: fmt.Sprintf("tool-result-%s", toolResult.ToolCallID),
1339 })
1340
1341 textParts = append(textParts, fantasy.ToolResultPart{
1342 ToolCallID: toolResult.ToolCallID,
1343 Output: fantasy.ToolResultOutputContentText{
1344 Text: "[Image/media content loaded - see attached file]",
1345 },
1346 ProviderOptions: toolResult.ProviderOptions,
1347 })
1348 } else {
1349 textParts = append(textParts, part)
1350 }
1351 }
1352
1353 convertedMessages = append(convertedMessages, fantasy.Message{
1354 Role: fantasy.MessageRoleTool,
1355 Content: textParts,
1356 })
1357
1358 if len(mediaFiles) > 0 {
1359 convertedMessages = append(convertedMessages, fantasy.NewUserMessage(
1360 "Here is the media content from the tool result:",
1361 mediaFiles...,
1362 ))
1363 }
1364 }
1365
1366 return convertedMessages
1367}
1368
1369// buildSummaryPrompt constructs the prompt text for session summarization.
1370func buildSummaryPrompt(todos []session.Todo) string {
1371 var sb strings.Builder
1372 sb.WriteString("Provide a detailed summary of our conversation above.")
1373 if len(todos) > 0 {
1374 sb.WriteString("\n\n## Current Todo List\n\n")
1375 for _, t := range todos {
1376 fmt.Fprintf(&sb, "- [%s] %s\n", t.Status, t.Content)
1377 }
1378 sb.WriteString("\nInclude these tasks and their statuses in your summary. ")
1379 sb.WriteString("Instruct the resuming assistant to use the `todos` tool to continue tracking progress on these tasks.")
1380 }
1381 return sb.String()
1382}
1383
1384func providerRetryLogFields(err *fantasy.ProviderError, delay time.Duration) []any {
1385 fields := []any{
1386 "retry_delay", delay.String(),
1387 }
1388 if err == nil {
1389 return fields
1390 }
1391 fields = append(fields, "status_code", err.StatusCode)
1392 if err.Title != "" {
1393 fields = append(fields, "title", err.Title)
1394 }
1395 if err.Message != "" {
1396 fields = append(fields, "message", err.Message)
1397 }
1398 return fields
1399}