Merge branch 'public_main' into config-provider-changes

Kujtim Hoxha created

Change summary

internal/config/config.go       |   9 ++
internal/fsext/fileutil.go      |   2 
internal/llm/agent/agent.go     |  25 +++++-
internal/llm/agent/mcp-tools.go |   6 +
internal/logging/logger.go      | 133 ++++++++++++++++++++++++++++++++++
internal/logging/writer.go      |   1 
6 files changed, 167 insertions(+), 9 deletions(-)

Detailed changes

internal/config/config.go 🔗

@@ -223,6 +223,15 @@ func loadConfig(cwd string, debug bool) (*Config, error) {
 			}
 		}
 
+		messagesPath := fmt.Sprintf("%s/%s", cfg.Options.DataDirectory, "messages")
+
+		if _, err := os.Stat(messagesPath); os.IsNotExist(err) {
+			if err := os.MkdirAll(messagesPath, 0o756); err != nil {
+				return cfg, fmt.Errorf("failed to create directory: %w", err)
+			}
+		}
+		logging.MessageDir = messagesPath
+
 		sloggingFileWriter, err := os.OpenFile(loggingFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o666)
 		if err != nil {
 			return cfg, fmt.Errorf("failed to open log file: %w", err)

internal/fsext/fileutil.go 🔗

@@ -55,7 +55,7 @@ func GetRgSearchCmd(pattern, path, include string) *exec.Cmd {
 		return nil
 	}
 	// Use -n to show line numbers and include the matched line
-	args := []string{"-n", pattern}
+	args := []string{"-H", "-n", pattern}
 	if include != "" {
 		args = append(args, "--glob", include)
 	}

internal/llm/agent/agent.go 🔗

@@ -363,6 +363,7 @@ func (a *agent) Run(ctx context.Context, sessionID string, content string, attac
 }
 
 func (a *agent) processGeneration(ctx context.Context, sessionID, content string, attachmentParts []message.ContentPart) AgentEvent {
+	cfg := config.Get()
 	// List existing messages; if none, start title generation asynchronously.
 	msgs, err := a.messages.List(ctx, sessionID)
 	if err != nil {
@@ -421,7 +422,13 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string
 			}
 			return a.err(fmt.Errorf("failed to process events: %w", err))
 		}
-		logging.Info("Result", "message", agentMessage.FinishReason(), "toolResults", toolResults)
+		if cfg.Options.Debug {
+			seqId := (len(msgHistory) + 1) / 2
+			toolResultFilepath := logging.WriteToolResultsJson(sessionID, seqId, toolResults)
+			logging.Info("Result", "message", agentMessage.FinishReason(), "toolResults", "{}", "filepath", toolResultFilepath)
+		} else {
+			logging.Info("Result", "message", agentMessage.FinishReason(), "toolResults", toolResults)
+		}
 		if (agentMessage.FinishReason() == message.FinishReasonToolUse) && toolResults != nil {
 			// We are not done, we need to respond with the tool response
 			msgHistory = append(msgHistory, agentMessage, *toolResults)
@@ -445,6 +452,7 @@ func (a *agent) createUserMessage(ctx context.Context, sessionID, content string
 }
 
 func (a *agent) streamAndHandleEvents(ctx context.Context, sessionID string, msgHistory []message.Message) (message.Message, *message.Message, error) {
+	ctx = context.WithValue(ctx, tools.SessionIDContextKey, sessionID)
 	eventChan := a.provider.StreamResponse(ctx, msgHistory, a.tools)
 
 	assistantMsg, err := a.messages.Create(ctx, sessionID, message.CreateMessageParams{
@@ -459,7 +467,6 @@ func (a *agent) streamAndHandleEvents(ctx context.Context, sessionID string, msg
 
 	// Add the session and message ID into the context if needed by tools.
 	ctx = context.WithValue(ctx, tools.MessageIDContextKey, assistantMsg.ID)
-	ctx = context.WithValue(ctx, tools.SessionIDContextKey, sessionID)
 
 	// Process each event in the stream.
 	for event := range eventChan {
@@ -491,10 +498,17 @@ func (a *agent) streamAndHandleEvents(ctx context.Context, sessionID string, msg
 		default:
 			// Continue processing
 			var tool tools.BaseTool
-			for _, availableTools := range a.tools {
-				if availableTools.Info().Name == toolCall.Name {
-					tool = availableTools
+			for _, availableTool := range a.tools {
+				if availableTool.Info().Name == toolCall.Name {
+					tool = availableTool
+					break
 				}
+				// Monkey patch for Copilot Sonnet-4 tool repetition obfuscation
+				// if strings.HasPrefix(toolCall.Name, availableTool.Info().Name) &&
+				// 	strings.HasPrefix(toolCall.Name, availableTool.Info().Name+availableTool.Info().Name) {
+				// 	tool = availableTool
+				// 	break
+				// }
 			}
 
 			// Tool not found
@@ -665,6 +679,7 @@ func (a *agent) Summarize(ctx context.Context, sessionID string) error {
 			a.Publish(pubsub.CreatedEvent, event)
 			return
 		}
+		summarizeCtx = context.WithValue(summarizeCtx, tools.SessionIDContextKey, sessionID)
 
 		if len(msgs) == 0 {
 			event = AgentEvent{

internal/llm/agent/mcp-tools.go 🔗

@@ -37,11 +37,15 @@ func (b *mcpTool) Name() string {
 }
 
 func (b *mcpTool) Info() tools.ToolInfo {
+	required := b.tool.InputSchema.Required
+	if required == nil {
+		required = make([]string, 0)
+	}
 	return tools.ToolInfo{
 		Name:        fmt.Sprintf("%s_%s", b.mcpName, b.tool.Name),
 		Description: b.tool.Description,
 		Parameters:  b.tool.InputSchema.Properties,
-		Required:    b.tool.InputSchema.Required,
+		Required:    required,
 	}
 }
 

internal/logging/logger.go 🔗

@@ -4,16 +4,33 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	// "path/filepath"
+	"encoding/json"
+	"runtime"
 	"runtime/debug"
+	"sync"
 	"time"
 )
 
+func getCaller() string {
+	var caller string
+	if _, file, line, ok := runtime.Caller(2); ok {
+		// caller = fmt.Sprintf("%s:%d", filepath.Base(file), line)
+		caller = fmt.Sprintf("%s:%d", file, line)
+	} else {
+		caller = "unknown"
+	}
+	return caller
+}
 func Info(msg string, args ...any) {
-	slog.Info(msg, args...)
+	source := getCaller()
+	slog.Info(msg, append([]any{"source", source}, args...)...)
 }
 
 func Debug(msg string, args ...any) {
-	slog.Debug(msg, args...)
+	// slog.Debug(msg, args...)
+	source := getCaller()
+	slog.Debug(msg, append([]any{"source", source}, args...)...)
 }
 
 func Warn(msg string, args ...any) {
@@ -76,3 +93,115 @@ func RecoverPanic(name string, cleanup func()) {
 		}
 	}
 }
+
+// Message Logging for Debug
+var MessageDir string
+
+func GetSessionPrefix(sessionId string) string {
+	return sessionId[:8]
+}
+
+var sessionLogMutex sync.Mutex
+
+func AppendToSessionLogFile(sessionId string, filename string, content string) string {
+	if MessageDir == "" || sessionId == "" {
+		return ""
+	}
+	sessionPrefix := GetSessionPrefix(sessionId)
+
+	sessionLogMutex.Lock()
+	defer sessionLogMutex.Unlock()
+
+	sessionPath := fmt.Sprintf("%s/%s", MessageDir, sessionPrefix)
+	if _, err := os.Stat(sessionPath); os.IsNotExist(err) {
+		if err := os.MkdirAll(sessionPath, 0o766); err != nil {
+			Error("Failed to create session directory", "dirpath", sessionPath, "error", err)
+			return ""
+		}
+	}
+
+	filePath := fmt.Sprintf("%s/%s", sessionPath, filename)
+
+	f, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		Error("Failed to open session log file", "filepath", filePath, "error", err)
+		return ""
+	}
+	defer f.Close()
+
+	// Append chunk to file
+	_, err = f.WriteString(content)
+	if err != nil {
+		Error("Failed to write chunk to session log file", "filepath", filePath, "error", err)
+		return ""
+	}
+	return filePath
+}
+
+func WriteRequestMessageJson(sessionId string, requestSeqId int, message any) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	msgJson, err := json.Marshal(message)
+	if err != nil {
+		Error("Failed to marshal message", "session_id", sessionId, "request_seq_id", requestSeqId, "error", err)
+		return ""
+	}
+	return WriteRequestMessage(sessionId, requestSeqId, string(msgJson))
+}
+
+func WriteRequestMessage(sessionId string, requestSeqId int, message string) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	filename := fmt.Sprintf("%d_request.json", requestSeqId)
+
+	return AppendToSessionLogFile(sessionId, filename, message)
+}
+
+func AppendToStreamSessionLogJson(sessionId string, requestSeqId int, jsonableChunk any) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	chunkJson, err := json.Marshal(jsonableChunk)
+	if err != nil {
+		Error("Failed to marshal message", "session_id", sessionId, "request_seq_id", requestSeqId, "error", err)
+		return ""
+	}
+	return AppendToStreamSessionLog(sessionId, requestSeqId, string(chunkJson))
+}
+
+func AppendToStreamSessionLog(sessionId string, requestSeqId int, chunk string) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	filename := fmt.Sprintf("%d_response_stream.log", requestSeqId)
+	return AppendToSessionLogFile(sessionId, filename, chunk)
+}
+
+func WriteChatResponseJson(sessionId string, requestSeqId int, response any) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	responseJson, err := json.Marshal(response)
+	if err != nil {
+		Error("Failed to marshal response", "session_id", sessionId, "request_seq_id", requestSeqId, "error", err)
+		return ""
+	}
+	filename := fmt.Sprintf("%d_response.json", requestSeqId)
+
+	return AppendToSessionLogFile(sessionId, filename, string(responseJson))
+}
+
+func WriteToolResultsJson(sessionId string, requestSeqId int, toolResults any) string {
+	if MessageDir == "" || sessionId == "" || requestSeqId <= 0 {
+		return ""
+	}
+	toolResultsJson, err := json.Marshal(toolResults)
+	if err != nil {
+		Error("Failed to marshal tool results", "session_id", sessionId, "request_seq_id", requestSeqId, "error", err)
+		return ""
+	}
+	filename := fmt.Sprintf("%d_tool_results.json", requestSeqId)
+	return AppendToSessionLogFile(sessionId, filename, string(toolResultsJson))
+}

internal/logging/writer.go 🔗

@@ -45,6 +45,7 @@ type writer struct{}
 
 func (w *writer) Write(p []byte) (int, error) {
 	d := logfmt.NewDecoder(bytes.NewReader(p))
+
 	for d.ScanRecord() {
 		msg := LogMessage{
 			ID:   fmt.Sprintf("%d", time.Now().UnixNano()),