From 671500de1b821072967a05613b33462f20a116cc Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 17 Nov 2025 18:58:22 +0100 Subject: [PATCH] agent_ui: Fix images copied from win explorer not being pastable (#42858) Closes https://github.com/zed-industries/zed/issues/41505 A bit adhoc but it gets the job done for now Release Notes: - Fixed images copied from windows explorer not being pastable in the agent panel --- Cargo.lock | 2 + crates/agent_ui/Cargo.toml | 2 + crates/agent_ui/src/acp/message_editor.rs | 186 +++++++++++------- crates/agent_ui/src/text_thread_editor.rs | 36 +++- crates/gpui/src/interactive.rs | 2 +- crates/gpui/src/platform.rs | 21 +- crates/gpui/src/platform/mac/platform.rs | 1 + crates/gpui/src/platform/windows/clipboard.rs | 59 +++--- 8 files changed, 207 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69c75d7ce0f9184342fbf202a149c6feb1d6a982..8a68187705d129beee2384246a81a488a09cb6e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,7 @@ dependencies = [ "assistant_slash_command", "assistant_slash_commands", "assistant_text_thread", + "async-fs", "audio", "buffer_diff", "chrono", @@ -343,6 +344,7 @@ dependencies = [ "gpui", "html_to_markdown", "http_client", + "image", "indoc", "itertools 0.14.0", "jsonschema", diff --git a/crates/agent_ui/Cargo.toml b/crates/agent_ui/Cargo.toml index 724b53a017911edbd6e9dd88c410daf794889d4e..6447b41335ece093718036091baf30c505ad76fd 100644 --- a/crates/agent_ui/Cargo.toml +++ b/crates/agent_ui/Cargo.toml @@ -98,6 +98,8 @@ util.workspace = true watch.workspace = true workspace.workspace = true zed_actions.workspace = true +image.workspace = true +async-fs.workspace = true [dev-dependencies] acp_thread = { workspace = true, features = ["test-support"] } diff --git a/crates/agent_ui/src/acp/message_editor.rs b/crates/agent_ui/src/acp/message_editor.rs index 9835dc929bd86085b481cbdb5e2ee667591c6e73..84896e7a74ab8f9514f0550bae1d28433650ce8d 100644 --- a/crates/agent_ui/src/acp/message_editor.rs +++ b/crates/agent_ui/src/acp/message_editor.rs @@ -28,6 +28,7 @@ use gpui::{ EventEmitter, FocusHandle, Focusable, Image, ImageFormat, Img, KeyContext, SharedString, Subscription, Task, TextStyle, WeakEntity, pulsating_between, }; +use itertools::Either; use language::{Buffer, Language, language_settings::InlayHintKind}; use language_model::LanguageModelImage; use postage::stream::Stream as _; @@ -912,74 +913,112 @@ impl MessageEditor { if !self.prompt_capabilities.borrow().image { return; } - - let images = cx - .read_from_clipboard() - .map(|item| { - item.into_entries() - .filter_map(|entry| { - if let ClipboardEntry::Image(image) = entry { - Some(image) - } else { - None - } - }) - .collect::>() - }) - .unwrap_or_default(); - - if images.is_empty() { + let Some(clipboard) = cx.read_from_clipboard() else { return; - } + }; cx.stop_propagation(); + cx.spawn_in(window, async move |this, cx| { + use itertools::Itertools; + let (mut images, paths) = clipboard + .into_entries() + .filter_map(|entry| match entry { + ClipboardEntry::Image(image) => Some(Either::Left(image)), + ClipboardEntry::ExternalPaths(paths) => Some(Either::Right(paths)), + _ => None, + }) + .partition_map::, Vec<_>, _, _, _>(std::convert::identity); + + if !paths.is_empty() { + images.extend( + cx.background_spawn(async move { + let mut images = vec![]; + for path in paths.into_iter().flat_map(|paths| paths.paths().to_owned()) { + let Ok(content) = async_fs::read(path).await else { + continue; + }; + let Ok(format) = image::guess_format(&content) else { + continue; + }; + images.push(gpui::Image::from_bytes( + match format { + image::ImageFormat::Png => gpui::ImageFormat::Png, + image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg, + image::ImageFormat::WebP => gpui::ImageFormat::Webp, + image::ImageFormat::Gif => gpui::ImageFormat::Gif, + image::ImageFormat::Bmp => gpui::ImageFormat::Bmp, + image::ImageFormat::Tiff => gpui::ImageFormat::Tiff, + image::ImageFormat::Ico => gpui::ImageFormat::Ico, + _ => continue, + }, + content, + )); + } + images + }) + .await, + ); + } - let replacement_text = MentionUri::PastedImage.as_link().to_string(); - for image in images { - let (excerpt_id, text_anchor, multibuffer_anchor) = - self.editor.update(cx, |message_editor, cx| { - let snapshot = message_editor.snapshot(window, cx); - let (excerpt_id, _, buffer_snapshot) = - snapshot.buffer_snapshot().as_singleton().unwrap(); - - let text_anchor = buffer_snapshot.anchor_before(buffer_snapshot.len()); - let multibuffer_anchor = snapshot - .buffer_snapshot() - .anchor_in_excerpt(*excerpt_id, text_anchor); - message_editor.edit( - [( - multi_buffer::Anchor::max()..multi_buffer::Anchor::max(), - format!("{replacement_text} "), - )], - cx, - ); - (*excerpt_id, text_anchor, multibuffer_anchor) - }); + if images.is_empty() { + return; + } - let content_len = replacement_text.len(); - let Some(start_anchor) = multibuffer_anchor else { - continue; + let replacement_text = MentionUri::PastedImage.as_link().to_string(); + let Ok(editor) = this.update(cx, |this, _| this.editor.clone()) else { + return; }; - let end_anchor = self.editor.update(cx, |editor, cx| { - let snapshot = editor.buffer().read(cx).snapshot(cx); - snapshot.anchor_before(start_anchor.to_offset(&snapshot) + content_len) - }); - let image = Arc::new(image); - let Some((crease_id, tx)) = insert_crease_for_mention( - excerpt_id, - text_anchor, - content_len, - MentionUri::PastedImage.name().into(), - IconName::Image.path().into(), - Some(Task::ready(Ok(image.clone())).shared()), - self.editor.clone(), - window, - cx, - ) else { - continue; - }; - let task = cx - .spawn_in(window, { - async move |_, cx| { + for image in images { + let Ok((excerpt_id, text_anchor, multibuffer_anchor)) = + editor.update_in(cx, |message_editor, window, cx| { + let snapshot = message_editor.snapshot(window, cx); + let (excerpt_id, _, buffer_snapshot) = + snapshot.buffer_snapshot().as_singleton().unwrap(); + + let text_anchor = buffer_snapshot.anchor_before(buffer_snapshot.len()); + let multibuffer_anchor = snapshot + .buffer_snapshot() + .anchor_in_excerpt(*excerpt_id, text_anchor); + message_editor.edit( + [( + multi_buffer::Anchor::max()..multi_buffer::Anchor::max(), + format!("{replacement_text} "), + )], + cx, + ); + (*excerpt_id, text_anchor, multibuffer_anchor) + }) + else { + break; + }; + + let content_len = replacement_text.len(); + let Some(start_anchor) = multibuffer_anchor else { + continue; + }; + let Ok(end_anchor) = editor.update(cx, |editor, cx| { + let snapshot = editor.buffer().read(cx).snapshot(cx); + snapshot.anchor_before(start_anchor.to_offset(&snapshot) + content_len) + }) else { + continue; + }; + let image = Arc::new(image); + let Ok(Some((crease_id, tx))) = cx.update(|window, cx| { + insert_crease_for_mention( + excerpt_id, + text_anchor, + content_len, + MentionUri::PastedImage.name().into(), + IconName::Image.path().into(), + Some(Task::ready(Ok(image.clone())).shared()), + editor.clone(), + window, + cx, + ) + }) else { + continue; + }; + let task = cx + .spawn(async move |cx| { let format = image.format; let image = cx .update(|_, cx| LanguageModelImage::from_image(image, cx)) @@ -994,15 +1033,16 @@ impl MessageEditor { } else { Err("Failed to convert image".into()) } - } - }) - .shared(); + }) + .shared(); - self.mention_set - .mentions - .insert(crease_id, (MentionUri::PastedImage, task.clone())); + this.update(cx, |this, _| { + this.mention_set + .mentions + .insert(crease_id, (MentionUri::PastedImage, task.clone())) + }) + .ok(); - cx.spawn_in(window, async move |this, cx| { if task.await.notify_async_err(cx).is_none() { this.update(cx, |this, cx| { this.editor.update(cx, |editor, cx| { @@ -1012,9 +1052,9 @@ impl MessageEditor { }) .ok(); } - }) - .detach(); - } + } + }) + .detach(); } pub fn insert_dragged_files( diff --git a/crates/agent_ui/src/text_thread_editor.rs b/crates/agent_ui/src/text_thread_editor.rs index e7f16b8886c719cf60763f651fe9abb9fe33d828..a46bf530217050a1f01ca777ee5e2af108989fbf 100644 --- a/crates/agent_ui/src/text_thread_editor.rs +++ b/crates/agent_ui/src/text_thread_editor.rs @@ -1679,7 +1679,7 @@ impl TextThreadEditor { ) { cx.stop_propagation(); - let images = if let Some(item) = cx.read_from_clipboard() { + let mut images = if let Some(item) = cx.read_from_clipboard() { item.into_entries() .filter_map(|entry| { if let ClipboardEntry::Image(image) = entry { @@ -1693,6 +1693,40 @@ impl TextThreadEditor { Vec::new() }; + if let Some(paths) = cx.read_from_clipboard() { + for path in paths + .into_entries() + .filter_map(|entry| { + if let ClipboardEntry::ExternalPaths(paths) = entry { + Some(paths.paths().to_owned()) + } else { + None + } + }) + .flatten() + { + let Ok(content) = std::fs::read(path) else { + continue; + }; + let Ok(format) = image::guess_format(&content) else { + continue; + }; + images.push(gpui::Image::from_bytes( + match format { + image::ImageFormat::Png => gpui::ImageFormat::Png, + image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg, + image::ImageFormat::WebP => gpui::ImageFormat::Webp, + image::ImageFormat::Gif => gpui::ImageFormat::Gif, + image::ImageFormat::Bmp => gpui::ImageFormat::Bmp, + image::ImageFormat::Tiff => gpui::ImageFormat::Tiff, + image::ImageFormat::Ico => gpui::ImageFormat::Ico, + _ => continue, + }, + content, + )); + } + } + let metadata = if let Some(item) = cx.read_from_clipboard() { item.entries().first().and_then(|entry| { if let ClipboardEntry::String(text) = entry { diff --git a/crates/gpui/src/interactive.rs b/crates/gpui/src/interactive.rs index 9e18b7990d9a30f2fd5e50010766d5f14a33e4e2..03acf81addaad1ae9800ef476a2dc7d13e690cf7 100644 --- a/crates/gpui/src/interactive.rs +++ b/crates/gpui/src/interactive.rs @@ -509,7 +509,7 @@ impl Deref for MouseExitEvent { } /// A collection of paths from the platform, such as from a file drop. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct ExternalPaths(pub(crate) SmallVec<[PathBuf; 2]>); impl ExternalPaths { diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index e50f407dc313038032c433aa0243d3c7791c5c1f..7168d0179424028e7f823d39df0f6f51f45095ac 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -1573,6 +1573,8 @@ pub enum ClipboardEntry { String(ClipboardString), /// An image entry Image(Image), + /// A file entry + ExternalPaths(crate::ExternalPaths), } impl ClipboardItem { @@ -1613,16 +1615,29 @@ impl ClipboardItem { /// Returns None if there were no ClipboardString entries. pub fn text(&self) -> Option { let mut answer = String::new(); - let mut any_entries = false; for entry in self.entries.iter() { if let ClipboardEntry::String(ClipboardString { text, metadata: _ }) = entry { answer.push_str(text); - any_entries = true; } } - if any_entries { Some(answer) } else { None } + if answer.is_empty() { + for entry in self.entries.iter() { + if let ClipboardEntry::ExternalPaths(paths) = entry { + for path in &paths.0 { + use std::fmt::Write as _; + _ = write!(answer, "{}", path.display()); + } + } + } + } + + if !answer.is_empty() { + Some(answer) + } else { + None + } } /// If this item is one ClipboardEntry::String, returns its metadata. diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 21dab9d3b617d505b80cb2d48b579916a9eb1238..cc6390cdb887a6c08fbb4520c9ab9fac4b50f9cf 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -1046,6 +1046,7 @@ impl Platform for MacPlatform { ClipboardEntry::Image(image) => { self.write_image_to_clipboard(image); } + ClipboardEntry::ExternalPaths(_) => {} }, None => { // Writing an empty list of entries just clears the clipboard. diff --git a/crates/gpui/src/platform/windows/clipboard.rs b/crates/gpui/src/platform/windows/clipboard.rs index 90d97a84c0bedcc241f7432a7f14f09d46018b49..2a5e8dcbbe2426674f7eae173231e2919590ad49 100644 --- a/crates/gpui/src/platform/windows/clipboard.rs +++ b/crates/gpui/src/platform/windows/clipboard.rs @@ -1,7 +1,7 @@ use std::sync::LazyLock; use anyhow::Result; -use collections::{FxHashMap, FxHashSet}; +use collections::FxHashMap; use itertools::Itertools; use windows::Win32::{ Foundation::{HANDLE, HGLOBAL}, @@ -18,7 +18,9 @@ use windows::Win32::{ }; use windows_core::PCWSTR; -use crate::{ClipboardEntry, ClipboardItem, ClipboardString, Image, ImageFormat, hash}; +use crate::{ + ClipboardEntry, ClipboardItem, ClipboardString, ExternalPaths, Image, ImageFormat, hash, +}; // https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-dragqueryfilew const DRAGDROP_GET_FILES_COUNT: u32 = 0xFFFFFFFF; @@ -48,16 +50,6 @@ static FORMATS_MAP: LazyLock> = LazyLock::ne formats_map.insert(CF_HDROP.0 as u32, ClipboardFormatType::Files); formats_map }); -static FORMATS_SET: LazyLock> = LazyLock::new(|| { - let mut formats_map = FxHashSet::default(); - formats_map.insert(CF_UNICODETEXT.0 as u32); - formats_map.insert(*CLIPBOARD_PNG_FORMAT); - formats_map.insert(*CLIPBOARD_GIF_FORMAT); - formats_map.insert(*CLIPBOARD_JPG_FORMAT); - formats_map.insert(*CLIPBOARD_SVG_FORMAT); - formats_map.insert(CF_HDROP.0 as u32); - formats_map -}); static IMAGE_FORMATS_MAP: LazyLock> = LazyLock::new(|| { let mut formats_map = FxHashMap::default(); formats_map.insert(*CLIPBOARD_PNG_FORMAT, ImageFormat::Png); @@ -138,6 +130,11 @@ fn register_clipboard_format(format: PCWSTR) -> u32 { std::io::Error::last_os_error() ); } + log::debug!( + "Registered clipboard format {} as {}", + unsafe { format.display() }, + ret + ); ret } @@ -159,6 +156,7 @@ fn write_to_clipboard_inner(item: ClipboardItem) -> Result<()> { ClipboardEntry::Image(image) => { write_image_to_clipboard(image)?; } + ClipboardEntry::ExternalPaths(_) => {} }, None => { // Writing an empty list of entries just clears the clipboard. @@ -249,19 +247,33 @@ fn with_best_match_format(f: F) -> Option where F: Fn(u32) -> Option, { + let mut text = None; + let mut image = None; + let mut files = None; let count = unsafe { CountClipboardFormats() }; let mut clipboard_format = 0; for _ in 0..count { clipboard_format = unsafe { EnumClipboardFormats(clipboard_format) }; - let Some(item_format) = FORMATS_SET.get(&clipboard_format) else { + let Some(item_format) = FORMATS_MAP.get(&clipboard_format) else { continue; }; - if let Some(entry) = f(*item_format) { - return Some(ClipboardItem { - entries: vec![entry], - }); + let bucket = match item_format { + ClipboardFormatType::Text if text.is_none() => &mut text, + ClipboardFormatType::Image if image.is_none() => &mut image, + ClipboardFormatType::Files if files.is_none() => &mut files, + _ => continue, + }; + if let Some(entry) = f(clipboard_format) { + *bucket = Some(entry); } } + + if let Some(entry) = [image, files, text].into_iter().flatten().next() { + return Some(ClipboardItem { + entries: vec![entry], + }); + } + // log the formats that we don't support yet. { clipboard_format = 0; @@ -346,18 +358,17 @@ fn read_image_for_type(format_number: u32, format: ImageFormat) -> Option Option { - let text = with_clipboard_data(CF_HDROP.0 as u32, |data_ptr, _size| { + let filenames = with_clipboard_data(CF_HDROP.0 as u32, |data_ptr, _size| { let hdrop = HDROP(data_ptr); - let mut filenames = String::new(); + let mut filenames = Vec::new(); with_file_names(hdrop, |file_name| { - filenames.push_str(&file_name); + filenames.push(std::path::PathBuf::from(file_name)); }); filenames })?; - Some(ClipboardEntry::String(ClipboardString { - text, - metadata: None, - })) + Some(ClipboardEntry::ExternalPaths(ExternalPaths( + filenames.into(), + ))) } fn with_clipboard_data(format: u32, f: F) -> Option