Cargo.lock 🔗
@@ -8517,6 +8517,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"assets",
+ "base64 0.22.1",
"env_logger 0.11.8",
"gpui",
"language",
Implement rendering of images with data urls in markdown (#30322)
Max Brunsfeld and Mikayla Maki created
Fixes #28266
Release Notes:
- Added support for rendering images with data URLs in markdown. This
can show up in hover documentation provided by language servers.
Co-authored-by: Mikayla Maki <mikayla.c.maki@gmail.com>Change summary
Cargo.lock | 1
crates/agent/src/context.rs | 4
crates/gpui/src/platform.rs | 29 ++++++-
crates/gpui/src/window.rs | 4
crates/markdown/Cargo.toml | 1
crates/markdown/examples/markdown.rs | 7 +
crates/markdown/src/markdown.rs | 107 ++++++++++++++++++++++++-----
crates/project/src/image_store.rs | 12 +-
crates/repl/src/outputs/image.rs | 6 -
9 files changed, 129 insertions(+), 42 deletions(-)
Detailed changes
crates/agent/src/context.rs 🔗
@@ -754,11 +754,11 @@ pub enum ImageStatus {
impl ImageContext {
pub fn eq_for_key(&self, other: &Self) -> bool {
- self.original_image.id == other.original_image.id
+ self.original_image.id() == other.original_image.id()
}
pub fn hash_for_key<H: Hasher>(&self, state: &mut H) {
- self.original_image.id.hash(state);
+ self.original_image.id().hash(state);
}
pub fn image(&self) -> Option<LanguageModelImage> {
crates/gpui/src/platform.rs 🔗
@@ -36,7 +36,7 @@ use crate::{
ForegroundExecutor, GlyphId, GpuSpecs, ImageSource, Keymap, LineLayout, Pixels, PlatformInput,
Point, RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, ScaledPixels, Scene,
ShapedGlyph, ShapedRun, SharedString, Size, SvgRenderer, SvgSize, Task, TaskLabel, Window,
- point, px, size,
+ hash, point, px, size,
};
use anyhow::Result;
use async_task::Runnable;
@@ -1499,6 +1499,20 @@ impl ImageFormat {
ImageFormat::Tiff => "image/tiff",
}
}
+
+ /// Returns the ImageFormat for the given mime type
+ pub fn from_mime_type(mime_type: &str) -> Option<Self> {
+ match mime_type {
+ "image/png" => Some(Self::Png),
+ "image/jpeg" | "image/jpg" => Some(Self::Jpeg),
+ "image/webp" => Some(Self::Webp),
+ "image/gif" => Some(Self::Gif),
+ "image/svg+xml" => Some(Self::Svg),
+ "image/bmp" => Some(Self::Bmp),
+ "image/tiff" | "image/tif" => Some(Self::Tiff),
+ _ => None,
+ }
+ }
}
/// An image, with a format and certain bytes
@@ -1509,7 +1523,7 @@ pub struct Image {
/// The raw image bytes
pub bytes: Vec<u8>,
/// The unique ID for the image
- pub id: u64,
+ id: u64,
}
impl Hash for Image {
@@ -1521,10 +1535,15 @@ impl Hash for Image {
impl Image {
/// An empty image containing no data
pub fn empty() -> Self {
+ Self::from_bytes(ImageFormat::Png, Vec::new())
+ }
+
+ /// Create an image from a format and bytes
+ pub fn from_bytes(format: ImageFormat, bytes: Vec<u8>) -> Self {
Self {
- format: ImageFormat::Png,
- bytes: Vec::new(),
- id: 0,
+ id: hash(&bytes),
+ format,
+ bytes,
}
}
crates/gpui/src/window.rs 🔗
@@ -2100,14 +2100,14 @@ impl Window {
let (task, is_first) = cx.fetch_asset::<A>(source);
task.clone().now_or_never().or_else(|| {
if is_first {
- let entity = self.current_view();
+ let entity_id = self.current_view();
self.spawn(cx, {
let task = task.clone();
async move |cx| {
task.await;
cx.on_next_frame(move |_, cx| {
- cx.notify(entity);
+ cx.notify(entity_id);
});
}
})
crates/markdown/Cargo.toml 🔗
@@ -20,6 +20,7 @@ test-support = [
[dependencies]
anyhow.workspace = true
+base64.workspace = true
gpui.workspace = true
language.workspace = true
linkify.workspace = true
crates/markdown/examples/markdown.rs 🔗
@@ -22,6 +22,15 @@ function a(b: T) {
```
Remember, markdown processors may have slight differences and extensions, so always refer to the specific documentation or guides relevant to your platform or editor for the best practices and additional features.
+
+## Images
+
+ item one
+
+ item two
+
crates/markdown/src/markdown.rs 🔗
@@ -1,9 +1,12 @@
pub mod parser;
mod path_range;
+use base64::Engine as _;
+use log::Level;
pub use path_range::{LineCol, PathWithRange};
use std::borrow::Cow;
+use std::collections::HashMap;
use std::collections::HashSet;
use std::iter;
use std::mem;
@@ -15,10 +18,10 @@ use std::time::Duration;
use gpui::{
AnyElement, App, BorderStyle, Bounds, ClipboardItem, CursorStyle, DispatchPhase, Edges, Entity,
- FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, KeyContext,
- Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent, Point, Stateful,
- StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun, TextStyle,
- TextStyleRefinement, actions, point, quad,
+ FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, Image,
+ ImageFormat, KeyContext, Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent,
+ Point, Stateful, StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun,
+ TextStyle, TextStyleRefinement, actions, img, point, quad,
};
use language::{Language, LanguageRegistry, Rope};
use parser::CodeBlockMetadata;
@@ -93,6 +96,7 @@ pub struct Markdown {
pressed_link: Option<RenderedLink>,
autoscroll_request: Option<usize>,
parsed_markdown: ParsedMarkdown,
+ images_by_source_offset: HashMap<usize, Arc<Image>>,
should_reparse: bool,
pending_parse: Option<Task<Option<()>>>,
focus_handle: FocusHandle,
@@ -149,6 +153,7 @@ impl Markdown {
pressed_link: None,
autoscroll_request: None,
should_reparse: false,
+ images_by_source_offset: Default::default(),
parsed_markdown: ParsedMarkdown::default(),
pending_parse: None,
focus_handle,
@@ -172,6 +177,7 @@ impl Markdown {
autoscroll_request: None,
should_reparse: false,
parsed_markdown: ParsedMarkdown::default(),
+ images_by_source_offset: Default::default(),
pending_parse: None,
focus_handle,
language_registry: None,
@@ -269,19 +275,23 @@ impl Markdown {
}
let source = self.source.clone();
- let parse_text_only = self.options.parse_links_only;
+ let should_parse_links_only = self.options.parse_links_only;
let language_registry = self.language_registry.clone();
let fallback = self.fallback_code_block_language.clone();
let parsed = cx.background_spawn(async move {
- if parse_text_only {
- return anyhow::Ok(ParsedMarkdown {
- events: Arc::from(parse_links_only(source.as_ref())),
- source,
- languages_by_name: TreeMap::default(),
- languages_by_path: TreeMap::default(),
- });
+ if should_parse_links_only {
+ return anyhow::Ok((
+ ParsedMarkdown {
+ events: Arc::from(parse_links_only(source.as_ref())),
+ source,
+ languages_by_name: TreeMap::default(),
+ languages_by_path: TreeMap::default(),
+ },
+ Default::default(),
+ ));
}
let (events, language_names, paths) = parse_markdown(&source);
+ let mut images_by_source_offset = HashMap::default();
let mut languages_by_name = TreeMap::default();
let mut languages_by_path = TreeMap::default();
if let Some(registry) = language_registry.as_ref() {
@@ -304,20 +314,52 @@ impl Markdown {
}
}
}
- anyhow::Ok(ParsedMarkdown {
- source,
- events: Arc::from(events),
- languages_by_name,
- languages_by_path,
- })
+
+ for (range, event) in &events {
+ if let MarkdownEvent::Start(MarkdownTag::Image { dest_url, .. }) = event {
+ if let Some(data_url) = dest_url.strip_prefix("data:") {
+ let Some((mime_info, data)) = data_url.split_once(',') else {
+ continue;
+ };
+ let Some((mime_type, encoding)) = mime_info.split_once(';') else {
+ continue;
+ };
+ let Some(format) = ImageFormat::from_mime_type(mime_type) else {
+ continue;
+ };
+ let is_base64 = encoding == "base64";
+ if is_base64 {
+ if let Some(bytes) = base64::prelude::BASE64_STANDARD
+ .decode(data)
+ .log_with_level(Level::Debug)
+ {
+ let image = Arc::new(Image::from_bytes(format, bytes));
+ images_by_source_offset.insert(range.start, image);
+ }
+ }
+ }
+ }
+ }
+
+ anyhow::Ok((
+ ParsedMarkdown {
+ source,
+ events: Arc::from(events),
+ languages_by_name,
+ languages_by_path,
+ },
+ images_by_source_offset,
+ ))
});
self.should_reparse = false;
self.pending_parse = Some(cx.spawn(async move |this, cx| {
async move {
- let parsed = parsed.await?;
+ let (parsed, images_by_source_offset) = parsed.await?;
+
this.update(cx, |this, cx| {
this.parsed_markdown = parsed;
+ this.images_by_source_offset = images_by_source_offset;
this.pending_parse.take();
if this.should_reparse {
this.parse(cx);
@@ -680,7 +722,9 @@ impl Element for MarkdownElement {
self.style.base_text_style.clone(),
self.style.syntax.clone(),
);
- let parsed_markdown = &self.markdown.read(cx).parsed_markdown;
+ let markdown = self.markdown.read(cx);
+ let parsed_markdown = &markdown.parsed_markdown;
+ let images = &markdown.images_by_source_offset;
let markdown_end = if let Some(last) = parsed_markdown.events.last() {
last.0.end
} else {
@@ -688,11 +732,29 @@ impl Element for MarkdownElement {
};
let mut current_code_block_metadata = None;
-
+ let mut current_img_block_range: Option<Range<usize>> = None;
for (range, event) in parsed_markdown.events.iter() {
+ // Skip alt text for images that rendered
+ if let Some(current_img_block_range) = ¤t_img_block_range {
+ if current_img_block_range.end > range.end {
+ continue;
+ }
+ }
+
match event {
MarkdownEvent::Start(tag) => {
match tag {
+ MarkdownTag::Image { .. } => {
+ if let Some(image) = images.get(&range.start) {
+ current_img_block_range = Some(range.clone());
+ builder.modify_current_div(|el| {
+ el.items_center()
+ .flex()
+ .flex_row()
+ .child(img(image.clone()))
+ });
+ }
+ }
MarkdownTag::Paragraph => {
builder.push_div(
div().when(!self.style.height_is_multiple_of_line_height, |el| {
@@ -940,6 +1002,9 @@ impl Element for MarkdownElement {
}
}
MarkdownEvent::End(tag) => match tag {
+ MarkdownTagEnd::Image => {
+ current_img_block_range.take();
+ }
MarkdownTagEnd::Paragraph => {
builder.pop_div();
}
crates/project/src/image_store.rs 🔗
@@ -6,8 +6,7 @@ use anyhow::{Context as _, Result, anyhow};
use collections::{HashMap, HashSet, hash_map};
use futures::{StreamExt, channel::oneshot};
use gpui::{
- App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, hash,
- prelude::*,
+ App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, prelude::*,
};
pub use image::ImageFormat;
use image::{ExtendedColorType, GenericImageView, ImageReader};
@@ -701,9 +700,8 @@ impl LocalImageStore {
fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
let format = image::guess_format(&content)?;
- Ok(Arc::new(gpui::Image {
- id: hash(&content),
- format: match format {
+ Ok(Arc::new(gpui::Image::from_bytes(
+ match format {
image::ImageFormat::Png => gpui::ImageFormat::Png,
image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg,
image::ImageFormat::WebP => gpui::ImageFormat::Webp,
@@ -712,8 +710,8 @@ fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
image::ImageFormat::Tiff => gpui::ImageFormat::Tiff,
_ => Err(anyhow::anyhow!("Image format not supported"))?,
},
- bytes: content,
- }))
+ content,
+ )))
}
impl ImageStoreImpl for Entity<RemoteImageStore> {
crates/repl/src/outputs/image.rs 🔗
@@ -57,11 +57,7 @@ impl ImageView {
};
// Convert back to a GPUI image for use with the clipboard
- let clipboard_image = Arc::new(Image {
- format,
- bytes,
- id: gpui_image_data.id.0 as u64,
- });
+ let clipboard_image = Arc::new(Image::from_bytes(format, bytes));
Ok(ImageView {
clipboard_image,