From fd2094fa19828d34614f7d420adcd87bff597edc Mon Sep 17 00:00:00 2001 From: Mikayla Maki Date: Tue, 9 Dec 2025 23:46:04 -0800 Subject: [PATCH] Add inline prompt rating (#44230) TODO: - [x] Add inline prompt rating buttons - [ ] Hook this into our other systems Release Notes: - N/A --- Cargo.lock | 1 + assets/keymaps/default-linux.json | 5 +- assets/keymaps/default-macos.json | 4 +- assets/keymaps/default-windows.json | 4 +- crates/agent_ui/Cargo.toml | 1 + crates/agent_ui/src/agent_model_selector.rs | 4 + crates/agent_ui/src/buffer_codegen.rs | 20 ++ crates/agent_ui/src/inline_prompt_editor.rs | 263 ++++++++++++++++++-- crates/agent_ui/src/terminal_codegen.rs | 17 +- crates/zed/src/zed.rs | 1 + 10 files changed, 292 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49b9d6069ccdfadf2d5145808fd7b758f9b389bb..7ae8d55a2484970b0ae1ad0631acadf22d106e46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,6 +401,7 @@ dependencies = [ "unindent", "url", "util", + "uuid", "watch", "workspace", "zed_actions", diff --git a/assets/keymaps/default-linux.json b/assets/keymaps/default-linux.json index 54a4f331c0b0c59eca79065fe42c1a8ecbf646b7..3838edb7a1fbea49ee0c5e1a978f9e8a9b919320 100644 --- a/assets/keymaps/default-linux.json +++ b/assets/keymaps/default-linux.json @@ -811,7 +811,10 @@ "context": "PromptEditor", "bindings": { "ctrl-[": "agent::CyclePreviousInlineAssist", - "ctrl-]": "agent::CycleNextInlineAssist" + "ctrl-]": "agent::CycleNextInlineAssist", + "ctrl-shift-enter": "inline_assistant::ThumbsUpResult", + "ctrl-shift-backspace": "inline_assistant::ThumbsDownResult" + } }, { diff --git a/assets/keymaps/default-macos.json b/assets/keymaps/default-macos.json index 060151c647e42370f5aa0be5d2fa186774c2574d..9edfaa03f8d7c9609d7b642ee7ddf61973f75e76 100644 --- a/assets/keymaps/default-macos.json +++ b/assets/keymaps/default-macos.json @@ -878,7 +878,9 @@ "bindings": { "cmd-alt-/": "agent::ToggleModelSelector", "ctrl-[": "agent::CyclePreviousInlineAssist", - "ctrl-]": "agent::CycleNextInlineAssist" + "ctrl-]": "agent::CycleNextInlineAssist", + "cmd-shift-enter": "inline_assistant::ThumbsUpResult", + "cmd-shift-backspace": "inline_assistant::ThumbsDownResult" } }, { diff --git a/assets/keymaps/default-windows.json b/assets/keymaps/default-windows.json index d749ac56886860b0e80de27f942082639df0447b..5842fe7729c74ad3f226055382cbac7f0b6d2f8f 100644 --- a/assets/keymaps/default-windows.json +++ b/assets/keymaps/default-windows.json @@ -816,7 +816,9 @@ "use_key_equivalents": true, "bindings": { "ctrl-[": "agent::CyclePreviousInlineAssist", - "ctrl-]": "agent::CycleNextInlineAssist" + "ctrl-]": "agent::CycleNextInlineAssist", + "ctrl-shift-enter": "inline_assistant::ThumbsUpResult", + "ctrl-shift-delete": "inline_assistant::ThumbsDownResult" } }, { diff --git a/crates/agent_ui/Cargo.toml b/crates/agent_ui/Cargo.toml index 048ffab9b72bdecce3754320bf34f1702f021554..2af0ce6fbd2b636d19d9cb8e544851514800313c 100644 --- a/crates/agent_ui/Cargo.toml +++ b/crates/agent_ui/Cargo.toml @@ -95,6 +95,7 @@ ui.workspace = true ui_input.workspace = true url.workspace = true util.workspace = true +uuid.workspace = true watch.workspace = true workspace.workspace = true zed_actions.workspace = true diff --git a/crates/agent_ui/src/agent_model_selector.rs b/crates/agent_ui/src/agent_model_selector.rs index 3840e40cf4d22db9d52e74ef0489c06ca8a15f26..9c2634143099d2097b5c6492f81c56aa51f12491 100644 --- a/crates/agent_ui/src/agent_model_selector.rs +++ b/crates/agent_ui/src/agent_model_selector.rs @@ -63,6 +63,10 @@ impl AgentModelSelector { pub fn toggle(&self, window: &mut Window, cx: &mut Context) { self.menu_handle.toggle(window, cx); } + + pub fn active_model(&self, cx: &App) -> Option { + self.selector.read(cx).delegate.active_model(cx) + } } impl Render for AgentModelSelector { diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rs index f7e7884310458e97421768882df57934a19b4430..1cd7bec7b5b2c24cfbcf01a20091e8a07608e73a 100644 --- a/crates/agent_ui/src/buffer_codegen.rs +++ b/crates/agent_ui/src/buffer_codegen.rs @@ -119,6 +119,10 @@ impl BufferCodegen { .push(cx.subscribe(&codegen, |_, _, event, cx| cx.emit(*event))); } + pub fn active_completion(&self, cx: &App) -> Option { + self.active_alternative().read(cx).current_completion() + } + pub fn active_alternative(&self) -> &Entity { &self.alternatives[self.active_alternative] } @@ -241,6 +245,10 @@ impl BufferCodegen { pub fn last_equal_ranges<'a>(&self, cx: &'a App) -> &'a [Range] { self.active_alternative().read(cx).last_equal_ranges() } + + pub fn selected_text<'a>(&self, cx: &'a App) -> Option<&'a str> { + self.active_alternative().read(cx).selected_text() + } } impl EventEmitter for BufferCodegen {} @@ -264,6 +272,7 @@ pub struct CodegenAlternative { line_operations: Vec, elapsed_time: Option, completion: Option, + selected_text: Option, pub message_id: Option, pub model_explanation: Option, } @@ -323,6 +332,7 @@ impl CodegenAlternative { range, elapsed_time: None, completion: None, + selected_text: None, model_explanation: None, _subscription: cx.subscribe(&buffer, Self::handle_buffer_event), } @@ -608,6 +618,8 @@ impl CodegenAlternative { .text_for_range(self.range.start..self.range.end) .collect::(); + self.selected_text = Some(selected_text.to_string()); + let selection_start = self.range.start.to_point(&snapshot); // Start with the indentation of the first line in the selection @@ -868,6 +880,14 @@ impl CodegenAlternative { cx.notify(); } + pub fn current_completion(&self) -> Option { + self.completion.clone() + } + + pub fn selected_text(&self) -> Option<&str> { + self.selected_text.as_deref() + } + pub fn stop(&mut self, cx: &mut Context) { self.last_equal_ranges.clear(); if self.diff.is_empty() { diff --git a/crates/agent_ui/src/inline_prompt_editor.rs b/crates/agent_ui/src/inline_prompt_editor.rs index b9852ea727c7974e3564fadc652f132076c01f09..4856d4024c94856e8dee91c048fe6ce72e79a7b8 100644 --- a/crates/agent_ui/src/inline_prompt_editor.rs +++ b/crates/agent_ui/src/inline_prompt_editor.rs @@ -8,10 +8,11 @@ use editor::{ ContextMenuOptions, Editor, EditorElement, EditorEvent, EditorMode, EditorStyle, MultiBuffer, actions::{MoveDown, MoveUp}, }; +use feature_flags::{FeatureFlag, FeatureFlagAppExt}; use fs::Fs; use gpui::{ - AnyElement, App, Context, Entity, EventEmitter, FocusHandle, Focusable, Subscription, - TextStyle, TextStyleRefinement, WeakEntity, Window, + AnyElement, App, ClipboardItem, Context, Entity, EventEmitter, FocusHandle, Focusable, + Subscription, TextStyle, TextStyleRefinement, WeakEntity, Window, actions, }; use language_model::{LanguageModel, LanguageModelRegistry}; use markdown::{HeadingLevelStyles, Markdown, MarkdownElement, MarkdownStyle}; @@ -19,14 +20,16 @@ use parking_lot::Mutex; use project::Project; use prompt_store::PromptStore; use settings::Settings; -use std::cmp; use std::ops::Range; use std::rc::Rc; use std::sync::Arc; +use std::{cmp, mem}; use theme::ThemeSettings; use ui::utils::WithRemSize; use ui::{IconButtonShape, KeyBinding, PopoverMenuHandle, Tooltip, prelude::*}; -use workspace::Workspace; +use uuid::Uuid; +use workspace::notifications::NotificationId; +use workspace::{Toast, Workspace}; use zed_actions::agent::ToggleModelSelector; use crate::agent_model_selector::AgentModelSelector; @@ -39,6 +42,58 @@ use crate::mention_set::{MentionSet, crease_for_mention}; use crate::terminal_codegen::TerminalCodegen; use crate::{CycleNextInlineAssist, CyclePreviousInlineAssist, ModelUsageContext}; +actions!(inline_assistant, [ThumbsUpResult, ThumbsDownResult]); + +pub struct InlineAssistRatingFeatureFlag; + +impl FeatureFlag for InlineAssistRatingFeatureFlag { + const NAME: &'static str = "inline-assist-rating"; + + fn enabled_for_staff() -> bool { + false + } +} + +enum RatingState { + Pending, + GeneratedCompletion(Option), + Rated(Uuid), +} + +impl RatingState { + fn is_pending(&self) -> bool { + matches!(self, RatingState::Pending) + } + + fn rating_id(&self) -> Option { + match self { + RatingState::Pending => None, + RatingState::GeneratedCompletion(_) => None, + RatingState::Rated(id) => Some(*id), + } + } + + fn rate(&mut self) -> (Uuid, Option) { + let id = Uuid::new_v4(); + let old_state = mem::replace(self, RatingState::Rated(id)); + let completion = match old_state { + RatingState::Pending => None, + RatingState::GeneratedCompletion(completion) => completion, + RatingState::Rated(_) => None, + }; + + (id, completion) + } + + fn reset(&mut self) { + *self = RatingState::Pending; + } + + fn generated_completion(&mut self, generated_completion: Option) { + *self = RatingState::GeneratedCompletion(generated_completion); + } +} + pub struct PromptEditor { pub editor: Entity, mode: PromptEditorMode, @@ -54,6 +109,7 @@ pub struct PromptEditor { _codegen_subscription: Subscription, editor_subscriptions: Vec, show_rate_limit_notice: bool, + rated: RatingState, _phantom: std::marker::PhantomData, } @@ -153,6 +209,8 @@ impl Render for PromptEditor { .on_action(cx.listener(Self::cancel)) .on_action(cx.listener(Self::move_up)) .on_action(cx.listener(Self::move_down)) + .on_action(cx.listener(Self::thumbs_up)) + .on_action(cx.listener(Self::thumbs_down)) .capture_action(cx.listener(Self::cycle_prev)) .capture_action(cx.listener(Self::cycle_next)) .child( @@ -429,6 +487,7 @@ impl PromptEditor { } self.edited_since_done = true; + self.rated.reset(); cx.notify(); } EditorEvent::Blurred => { @@ -516,6 +575,121 @@ impl PromptEditor { } } + fn thumbs_up(&mut self, _: &ThumbsUpResult, _window: &mut Window, cx: &mut Context) { + if self.rated.is_pending() { + self.toast("Still generating...", None, cx); + return; + } + + if let Some(rating_id) = self.rated.rating_id() { + self.toast("Already rated this completion", Some(rating_id), cx); + return; + } + + let (rating_id, completion) = self.rated.rate(); + + let selected_text = match &self.mode { + PromptEditorMode::Buffer { codegen, .. } => { + codegen.read(cx).selected_text(cx).map(|s| s.to_string()) + } + PromptEditorMode::Terminal { .. } => None, + }; + + let model_info = self.model_selector.read(cx).active_model(cx); + let model_id = { + let Some(configured_model) = model_info else { + self.toast("No configured model", None, cx); + return; + }; + + configured_model.model.telemetry_id() + }; + + let prompt = self.editor.read(cx).text(cx); + + telemetry::event!( + "Inline Assistant Rated", + rating = "positive", + model = model_id, + prompt = prompt, + completion = completion, + selected_text = selected_text, + rating_id = rating_id.to_string() + ); + + cx.notify(); + } + + fn thumbs_down(&mut self, _: &ThumbsDownResult, _window: &mut Window, cx: &mut Context) { + if self.rated.is_pending() { + self.toast("Still generating...", None, cx); + return; + } + if let Some(rating_id) = self.rated.rating_id() { + self.toast("Already rated this completion", Some(rating_id), cx); + return; + } + + let (rating_id, completion) = self.rated.rate(); + + let selected_text = match &self.mode { + PromptEditorMode::Buffer { codegen, .. } => { + codegen.read(cx).selected_text(cx).map(|s| s.to_string()) + } + PromptEditorMode::Terminal { .. } => None, + }; + + let model_info = self.model_selector.read(cx).active_model(cx); + let model_telemetry_id = { + let Some(configured_model) = model_info else { + self.toast("No configured model", None, cx); + return; + }; + + configured_model.model.telemetry_id() + }; + + let prompt = self.editor.read(cx).text(cx); + + telemetry::event!( + "Inline Assistant Rated", + rating = "negative", + model = model_telemetry_id, + prompt = prompt, + completion = completion, + selected_text = selected_text, + rating_id = rating_id.to_string() + ); + + cx.notify(); + } + + fn toast(&mut self, msg: &str, uuid: Option, cx: &mut Context<'_, PromptEditor>) { + self.workspace + .update(cx, |workspace, cx| { + enum InlinePromptRating {} + workspace.show_toast( + { + let mut toast = Toast::new( + NotificationId::unique::(), + msg.to_string(), + ) + .autohide(); + + if let Some(uuid) = uuid { + toast = toast.on_click("Click to copy rating ID", move |_, cx| { + cx.write_to_clipboard(ClipboardItem::new_string(uuid.to_string())); + }); + }; + + toast + }, + cx, + ); + }) + .ok(); + } + fn move_up(&mut self, _: &MoveUp, window: &mut Window, cx: &mut Context) { if let Some(ix) = self.prompt_history_ix { if ix > 0 { @@ -621,6 +795,9 @@ impl PromptEditor { .into_any_element(), ] } else { + let show_rating_buttons = cx.has_flag::(); + let rated = self.rated.rating_id().is_some(); + let accept = IconButton::new("accept", IconName::Check) .icon_color(Color::Info) .shape(IconButtonShape::Square) @@ -632,25 +809,59 @@ impl PromptEditor { })) .into_any_element(); - match &self.mode { - PromptEditorMode::Terminal { .. } => vec![ - accept, - IconButton::new("confirm", IconName::PlayFilled) - .icon_color(Color::Info) + let mut buttons = Vec::new(); + + if show_rating_buttons { + buttons.push( + IconButton::new("thumbs-down", IconName::ThumbsDown) + .icon_color(if rated { Color::Muted } else { Color::Default }) .shape(IconButtonShape::Square) - .tooltip(|_window, cx| { - Tooltip::for_action( - "Execute Generated Command", - &menu::SecondaryConfirm, - cx, - ) - }) - .on_click(cx.listener(|_, _, _, cx| { - cx.emit(PromptEditorEvent::ConfirmRequested { execute: true }); + .disabled(rated) + .tooltip(Tooltip::text("Bad result")) + .on_click(cx.listener(|this, _, window, cx| { + this.thumbs_down(&ThumbsDownResult, window, cx); })) .into_any_element(), - ], - PromptEditorMode::Buffer { .. } => vec![accept], + ); + + buttons.push( + IconButton::new("thumbs-up", IconName::ThumbsUp) + .icon_color(if rated { Color::Muted } else { Color::Default }) + .shape(IconButtonShape::Square) + .disabled(rated) + .tooltip(Tooltip::text("Good result")) + .on_click(cx.listener(|this, _, window, cx| { + this.thumbs_up(&ThumbsUpResult, window, cx); + })) + .into_any_element(), + ); + } + + buttons.push(accept); + + match &self.mode { + PromptEditorMode::Terminal { .. } => { + buttons.push( + IconButton::new("confirm", IconName::PlayFilled) + .icon_color(Color::Info) + .shape(IconButtonShape::Square) + .tooltip(|_window, cx| { + Tooltip::for_action( + "Execute Generated Command", + &menu::SecondaryConfirm, + cx, + ) + }) + .on_click(cx.listener(|_, _, _, cx| { + cx.emit(PromptEditorEvent::ConfirmRequested { + execute: true, + }); + })) + .into_any_element(), + ); + buttons + } + PromptEditorMode::Buffer { .. } => buttons, } } } @@ -979,6 +1190,7 @@ impl PromptEditor { editor_subscriptions: Vec::new(), show_rate_limit_notice: false, mode, + rated: RatingState::Pending, _phantom: Default::default(), }; @@ -989,7 +1201,7 @@ impl PromptEditor { fn handle_codegen_changed( &mut self, - _: Entity, + codegen: Entity, cx: &mut Context>, ) { match self.codegen_status(cx) { @@ -998,10 +1210,13 @@ impl PromptEditor { .update(cx, |editor, _| editor.set_read_only(false)); } CodegenStatus::Pending => { + self.rated.reset(); self.editor .update(cx, |editor, _| editor.set_read_only(true)); } CodegenStatus::Done => { + let completion = codegen.read(cx).active_completion(cx); + self.rated.generated_completion(completion); self.edited_since_done = false; self.editor .update(cx, |editor, _| editor.set_read_only(false)); @@ -1122,6 +1337,7 @@ impl PromptEditor { editor_subscriptions: Vec::new(), mode, show_rate_limit_notice: false, + rated: RatingState::Pending, _phantom: Default::default(), }; this.count_lines(cx); @@ -1154,17 +1370,20 @@ impl PromptEditor { } } - fn handle_codegen_changed(&mut self, _: Entity, cx: &mut Context) { + fn handle_codegen_changed(&mut self, codegen: Entity, cx: &mut Context) { match &self.codegen().read(cx).status { CodegenStatus::Idle => { self.editor .update(cx, |editor, _| editor.set_read_only(false)); } CodegenStatus::Pending => { + self.rated = RatingState::Pending; self.editor .update(cx, |editor, _| editor.set_read_only(true)); } CodegenStatus::Done | CodegenStatus::Error(_) => { + self.rated + .generated_completion(codegen.read(cx).completion()); self.edited_since_done = false; self.editor .update(cx, |editor, _| editor.set_read_only(false)); diff --git a/crates/agent_ui/src/terminal_codegen.rs b/crates/agent_ui/src/terminal_codegen.rs index 5a4a9d560a16e858dcaedf706f2067a24bc12c5f..cc99471f7f3037cb94ff23979036bd6c2026e2f0 100644 --- a/crates/agent_ui/src/terminal_codegen.rs +++ b/crates/agent_ui/src/terminal_codegen.rs @@ -135,6 +135,12 @@ impl TerminalCodegen { cx.notify(); } + pub fn completion(&self) -> Option { + self.transaction + .as_ref() + .map(|transaction| transaction.completion.clone()) + } + pub fn stop(&mut self, cx: &mut Context) { self.status = CodegenStatus::Done; self.generation = Task::ready(()); @@ -167,27 +173,32 @@ pub const CLEAR_INPUT: &str = "\x03"; const CARRIAGE_RETURN: &str = "\x0d"; struct TerminalTransaction { + completion: String, terminal: Entity, } impl TerminalTransaction { pub fn start(terminal: Entity) -> Self { - Self { terminal } + Self { + completion: String::new(), + terminal, + } } pub fn push(&mut self, hunk: String, cx: &mut App) { // Ensure that the assistant cannot accidentally execute commands that are streamed into the terminal let input = Self::sanitize_input(hunk); + self.completion.push_str(&input); self.terminal .update(cx, |terminal, _| terminal.input(input.into_bytes())); } - pub fn undo(&self, cx: &mut App) { + pub fn undo(self, cx: &mut App) { self.terminal .update(cx, |terminal, _| terminal.input(CLEAR_INPUT.as_bytes())); } - pub fn complete(&self, cx: &mut App) { + pub fn complete(self, cx: &mut App) { self.terminal .update(cx, |terminal, _| terminal.input(CARRIAGE_RETURN.as_bytes())); } diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index 1361fcdba788752099c8e5b37b51e751fccf4dfd..71653124b1c4af993d9878b2b689d07f4f2acd02 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -4745,6 +4745,7 @@ mod tests { "git_panel", "go_to_line", "icon_theme_selector", + "inline_assistant", "journal", "keymap_editor", "keystroke_input",