Lukas Wirth created
Closes https://github.com/zed-industries/zed/issues/41505
A bit adhoc but it gets the job done for now
Release Notes:
- Fixed images copied from windows explorer not being pastable in the
agent panelCargo.lock | 2
crates/agent_ui/Cargo.toml | 2
crates/agent_ui/src/acp/message_editor.rs | 186 ++++++++++++--------
crates/agent_ui/src/text_thread_editor.rs | 36 +++
crates/gpui/src/interactive.rs | 2
crates/gpui/src/platform.rs | 21 ++
crates/gpui/src/platform/mac/platform.rs | 1
crates/gpui/src/platform/windows/clipboard.rs | 59 +++--
8 files changed, 207 insertions(+), 102 deletions(-)
@@ -322,6 +322,7 @@ dependencies = [
"assistant_slash_command",
"assistant_slash_commands",
"assistant_text_thread",
+ "async-fs",
"audio",
"buffer_diff",
"chrono",
@@ -343,6 +344,7 @@ dependencies = [
"gpui",
"html_to_markdown",
"http_client",
+ "image",
"indoc",
"itertools 0.14.0",
"jsonschema",
@@ -98,6 +98,8 @@ util.workspace = true
watch.workspace = true
workspace.workspace = true
zed_actions.workspace = true
+image.workspace = true
+async-fs.workspace = true
[dev-dependencies]
acp_thread = { workspace = true, features = ["test-support"] }
@@ -28,6 +28,7 @@ use gpui::{
EventEmitter, FocusHandle, Focusable, Image, ImageFormat, Img, KeyContext, SharedString,
Subscription, Task, TextStyle, WeakEntity, pulsating_between,
};
+use itertools::Either;
use language::{Buffer, Language, language_settings::InlayHintKind};
use language_model::LanguageModelImage;
use postage::stream::Stream as _;
@@ -912,74 +913,112 @@ impl MessageEditor {
if !self.prompt_capabilities.borrow().image {
return;
}
-
- let images = cx
- .read_from_clipboard()
- .map(|item| {
- item.into_entries()
- .filter_map(|entry| {
- if let ClipboardEntry::Image(image) = entry {
- Some(image)
- } else {
- None
- }
- })
- .collect::<Vec<_>>()
- })
- .unwrap_or_default();
-
- if images.is_empty() {
+ let Some(clipboard) = cx.read_from_clipboard() else {
return;
- }
+ };
cx.stop_propagation();
+ cx.spawn_in(window, async move |this, cx| {
+ use itertools::Itertools;
+ let (mut images, paths) = clipboard
+ .into_entries()
+ .filter_map(|entry| match entry {
+ ClipboardEntry::Image(image) => Some(Either::Left(image)),
+ ClipboardEntry::ExternalPaths(paths) => Some(Either::Right(paths)),
+ _ => None,
+ })
+ .partition_map::<Vec<_>, Vec<_>, _, _, _>(std::convert::identity);
+
+ if !paths.is_empty() {
+ images.extend(
+ cx.background_spawn(async move {
+ let mut images = vec![];
+ for path in paths.into_iter().flat_map(|paths| paths.paths().to_owned()) {
+ let Ok(content) = async_fs::read(path).await else {
+ continue;
+ };
+ let Ok(format) = image::guess_format(&content) else {
+ continue;
+ };
+ images.push(gpui::Image::from_bytes(
+ match format {
+ image::ImageFormat::Png => gpui::ImageFormat::Png,
+ image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg,
+ image::ImageFormat::WebP => gpui::ImageFormat::Webp,
+ image::ImageFormat::Gif => gpui::ImageFormat::Gif,
+ image::ImageFormat::Bmp => gpui::ImageFormat::Bmp,
+ image::ImageFormat::Tiff => gpui::ImageFormat::Tiff,
+ image::ImageFormat::Ico => gpui::ImageFormat::Ico,
+ _ => continue,
+ },
+ content,
+ ));
+ }
+ images
+ })
+ .await,
+ );
+ }
- let replacement_text = MentionUri::PastedImage.as_link().to_string();
- for image in images {
- let (excerpt_id, text_anchor, multibuffer_anchor) =
- self.editor.update(cx, |message_editor, cx| {
- let snapshot = message_editor.snapshot(window, cx);
- let (excerpt_id, _, buffer_snapshot) =
- snapshot.buffer_snapshot().as_singleton().unwrap();
-
- let text_anchor = buffer_snapshot.anchor_before(buffer_snapshot.len());
- let multibuffer_anchor = snapshot
- .buffer_snapshot()
- .anchor_in_excerpt(*excerpt_id, text_anchor);
- message_editor.edit(
- [(
- multi_buffer::Anchor::max()..multi_buffer::Anchor::max(),
- format!("{replacement_text} "),
- )],
- cx,
- );
- (*excerpt_id, text_anchor, multibuffer_anchor)
- });
+ if images.is_empty() {
+ return;
+ }
- let content_len = replacement_text.len();
- let Some(start_anchor) = multibuffer_anchor else {
- continue;
+ let replacement_text = MentionUri::PastedImage.as_link().to_string();
+ let Ok(editor) = this.update(cx, |this, _| this.editor.clone()) else {
+ return;
};
- let end_anchor = self.editor.update(cx, |editor, cx| {
- let snapshot = editor.buffer().read(cx).snapshot(cx);
- snapshot.anchor_before(start_anchor.to_offset(&snapshot) + content_len)
- });
- let image = Arc::new(image);
- let Some((crease_id, tx)) = insert_crease_for_mention(
- excerpt_id,
- text_anchor,
- content_len,
- MentionUri::PastedImage.name().into(),
- IconName::Image.path().into(),
- Some(Task::ready(Ok(image.clone())).shared()),
- self.editor.clone(),
- window,
- cx,
- ) else {
- continue;
- };
- let task = cx
- .spawn_in(window, {
- async move |_, cx| {
+ for image in images {
+ let Ok((excerpt_id, text_anchor, multibuffer_anchor)) =
+ editor.update_in(cx, |message_editor, window, cx| {
+ let snapshot = message_editor.snapshot(window, cx);
+ let (excerpt_id, _, buffer_snapshot) =
+ snapshot.buffer_snapshot().as_singleton().unwrap();
+
+ let text_anchor = buffer_snapshot.anchor_before(buffer_snapshot.len());
+ let multibuffer_anchor = snapshot
+ .buffer_snapshot()
+ .anchor_in_excerpt(*excerpt_id, text_anchor);
+ message_editor.edit(
+ [(
+ multi_buffer::Anchor::max()..multi_buffer::Anchor::max(),
+ format!("{replacement_text} "),
+ )],
+ cx,
+ );
+ (*excerpt_id, text_anchor, multibuffer_anchor)
+ })
+ else {
+ break;
+ };
+
+ let content_len = replacement_text.len();
+ let Some(start_anchor) = multibuffer_anchor else {
+ continue;
+ };
+ let Ok(end_anchor) = editor.update(cx, |editor, cx| {
+ let snapshot = editor.buffer().read(cx).snapshot(cx);
+ snapshot.anchor_before(start_anchor.to_offset(&snapshot) + content_len)
+ }) else {
+ continue;
+ };
+ let image = Arc::new(image);
+ let Ok(Some((crease_id, tx))) = cx.update(|window, cx| {
+ insert_crease_for_mention(
+ excerpt_id,
+ text_anchor,
+ content_len,
+ MentionUri::PastedImage.name().into(),
+ IconName::Image.path().into(),
+ Some(Task::ready(Ok(image.clone())).shared()),
+ editor.clone(),
+ window,
+ cx,
+ )
+ }) else {
+ continue;
+ };
+ let task = cx
+ .spawn(async move |cx| {
let format = image.format;
let image = cx
.update(|_, cx| LanguageModelImage::from_image(image, cx))
@@ -994,15 +1033,16 @@ impl MessageEditor {
} else {
Err("Failed to convert image".into())
}
- }
- })
- .shared();
+ })
+ .shared();
- self.mention_set
- .mentions
- .insert(crease_id, (MentionUri::PastedImage, task.clone()));
+ this.update(cx, |this, _| {
+ this.mention_set
+ .mentions
+ .insert(crease_id, (MentionUri::PastedImage, task.clone()))
+ })
+ .ok();
- cx.spawn_in(window, async move |this, cx| {
if task.await.notify_async_err(cx).is_none() {
this.update(cx, |this, cx| {
this.editor.update(cx, |editor, cx| {
@@ -1012,9 +1052,9 @@ impl MessageEditor {
})
.ok();
}
- })
- .detach();
- }
+ }
+ })
+ .detach();
}
pub fn insert_dragged_files(
@@ -1679,7 +1679,7 @@ impl TextThreadEditor {
) {
cx.stop_propagation();
- let images = if let Some(item) = cx.read_from_clipboard() {
+ let mut images = if let Some(item) = cx.read_from_clipboard() {
item.into_entries()
.filter_map(|entry| {
if let ClipboardEntry::Image(image) = entry {
@@ -1693,6 +1693,40 @@ impl TextThreadEditor {
Vec::new()
};
+ if let Some(paths) = cx.read_from_clipboard() {
+ for path in paths
+ .into_entries()
+ .filter_map(|entry| {
+ if let ClipboardEntry::ExternalPaths(paths) = entry {
+ Some(paths.paths().to_owned())
+ } else {
+ None
+ }
+ })
+ .flatten()
+ {
+ let Ok(content) = std::fs::read(path) else {
+ continue;
+ };
+ let Ok(format) = image::guess_format(&content) else {
+ continue;
+ };
+ images.push(gpui::Image::from_bytes(
+ match format {
+ image::ImageFormat::Png => gpui::ImageFormat::Png,
+ image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg,
+ image::ImageFormat::WebP => gpui::ImageFormat::Webp,
+ image::ImageFormat::Gif => gpui::ImageFormat::Gif,
+ image::ImageFormat::Bmp => gpui::ImageFormat::Bmp,
+ image::ImageFormat::Tiff => gpui::ImageFormat::Tiff,
+ image::ImageFormat::Ico => gpui::ImageFormat::Ico,
+ _ => continue,
+ },
+ content,
+ ));
+ }
+ }
+
let metadata = if let Some(item) = cx.read_from_clipboard() {
item.entries().first().and_then(|entry| {
if let ClipboardEntry::String(text) = entry {
@@ -509,7 +509,7 @@ impl Deref for MouseExitEvent {
}
/// A collection of paths from the platform, such as from a file drop.
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Eq, PartialEq)]
pub struct ExternalPaths(pub(crate) SmallVec<[PathBuf; 2]>);
impl ExternalPaths {
@@ -1573,6 +1573,8 @@ pub enum ClipboardEntry {
String(ClipboardString),
/// An image entry
Image(Image),
+ /// A file entry
+ ExternalPaths(crate::ExternalPaths),
}
impl ClipboardItem {
@@ -1613,16 +1615,29 @@ impl ClipboardItem {
/// Returns None if there were no ClipboardString entries.
pub fn text(&self) -> Option<String> {
let mut answer = String::new();
- let mut any_entries = false;
for entry in self.entries.iter() {
if let ClipboardEntry::String(ClipboardString { text, metadata: _ }) = entry {
answer.push_str(text);
- any_entries = true;
}
}
- if any_entries { Some(answer) } else { None }
+ if answer.is_empty() {
+ for entry in self.entries.iter() {
+ if let ClipboardEntry::ExternalPaths(paths) = entry {
+ for path in &paths.0 {
+ use std::fmt::Write as _;
+ _ = write!(answer, "{}", path.display());
+ }
+ }
+ }
+ }
+
+ if !answer.is_empty() {
+ Some(answer)
+ } else {
+ None
+ }
}
/// If this item is one ClipboardEntry::String, returns its metadata.
@@ -1046,6 +1046,7 @@ impl Platform for MacPlatform {
ClipboardEntry::Image(image) => {
self.write_image_to_clipboard(image);
}
+ ClipboardEntry::ExternalPaths(_) => {}
},
None => {
// Writing an empty list of entries just clears the clipboard.
@@ -1,7 +1,7 @@
use std::sync::LazyLock;
use anyhow::Result;
-use collections::{FxHashMap, FxHashSet};
+use collections::FxHashMap;
use itertools::Itertools;
use windows::Win32::{
Foundation::{HANDLE, HGLOBAL},
@@ -18,7 +18,9 @@ use windows::Win32::{
};
use windows_core::PCWSTR;
-use crate::{ClipboardEntry, ClipboardItem, ClipboardString, Image, ImageFormat, hash};
+use crate::{
+ ClipboardEntry, ClipboardItem, ClipboardString, ExternalPaths, Image, ImageFormat, hash,
+};
// https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-dragqueryfilew
const DRAGDROP_GET_FILES_COUNT: u32 = 0xFFFFFFFF;
@@ -48,16 +50,6 @@ static FORMATS_MAP: LazyLock<FxHashMap<u32, ClipboardFormatType>> = LazyLock::ne
formats_map.insert(CF_HDROP.0 as u32, ClipboardFormatType::Files);
formats_map
});
-static FORMATS_SET: LazyLock<FxHashSet<u32>> = LazyLock::new(|| {
- let mut formats_map = FxHashSet::default();
- formats_map.insert(CF_UNICODETEXT.0 as u32);
- formats_map.insert(*CLIPBOARD_PNG_FORMAT);
- formats_map.insert(*CLIPBOARD_GIF_FORMAT);
- formats_map.insert(*CLIPBOARD_JPG_FORMAT);
- formats_map.insert(*CLIPBOARD_SVG_FORMAT);
- formats_map.insert(CF_HDROP.0 as u32);
- formats_map
-});
static IMAGE_FORMATS_MAP: LazyLock<FxHashMap<u32, ImageFormat>> = LazyLock::new(|| {
let mut formats_map = FxHashMap::default();
formats_map.insert(*CLIPBOARD_PNG_FORMAT, ImageFormat::Png);
@@ -138,6 +130,11 @@ fn register_clipboard_format(format: PCWSTR) -> u32 {
std::io::Error::last_os_error()
);
}
+ log::debug!(
+ "Registered clipboard format {} as {}",
+ unsafe { format.display() },
+ ret
+ );
ret
}
@@ -159,6 +156,7 @@ fn write_to_clipboard_inner(item: ClipboardItem) -> Result<()> {
ClipboardEntry::Image(image) => {
write_image_to_clipboard(image)?;
}
+ ClipboardEntry::ExternalPaths(_) => {}
},
None => {
// Writing an empty list of entries just clears the clipboard.
@@ -249,19 +247,33 @@ fn with_best_match_format<F>(f: F) -> Option<ClipboardItem>
where
F: Fn(u32) -> Option<ClipboardEntry>,
{
+ let mut text = None;
+ let mut image = None;
+ let mut files = None;
let count = unsafe { CountClipboardFormats() };
let mut clipboard_format = 0;
for _ in 0..count {
clipboard_format = unsafe { EnumClipboardFormats(clipboard_format) };
- let Some(item_format) = FORMATS_SET.get(&clipboard_format) else {
+ let Some(item_format) = FORMATS_MAP.get(&clipboard_format) else {
continue;
};
- if let Some(entry) = f(*item_format) {
- return Some(ClipboardItem {
- entries: vec![entry],
- });
+ let bucket = match item_format {
+ ClipboardFormatType::Text if text.is_none() => &mut text,
+ ClipboardFormatType::Image if image.is_none() => &mut image,
+ ClipboardFormatType::Files if files.is_none() => &mut files,
+ _ => continue,
+ };
+ if let Some(entry) = f(clipboard_format) {
+ *bucket = Some(entry);
}
}
+
+ if let Some(entry) = [image, files, text].into_iter().flatten().next() {
+ return Some(ClipboardItem {
+ entries: vec![entry],
+ });
+ }
+
// log the formats that we don't support yet.
{
clipboard_format = 0;
@@ -346,18 +358,17 @@ fn read_image_for_type(format_number: u32, format: ImageFormat) -> Option<Clipbo
}
fn read_files_from_clipboard() -> Option<ClipboardEntry> {
- let text = with_clipboard_data(CF_HDROP.0 as u32, |data_ptr, _size| {
+ let filenames = with_clipboard_data(CF_HDROP.0 as u32, |data_ptr, _size| {
let hdrop = HDROP(data_ptr);
- let mut filenames = String::new();
+ let mut filenames = Vec::new();
with_file_names(hdrop, |file_name| {
- filenames.push_str(&file_name);
+ filenames.push(std::path::PathBuf::from(file_name));
});
filenames
})?;
- Some(ClipboardEntry::String(ClipboardString {
- text,
- metadata: None,
- }))
+ Some(ClipboardEntry::ExternalPaths(ExternalPaths(
+ filenames.into(),
+ )))
}
fn with_clipboard_data<F, R>(format: u32, f: F) -> Option<R>