diff --git a/crates/call2/src/shared_screen.rs b/crates/call2/src/shared_screen.rs index 7b7cd7d9df33045759ae9bda2f16e016f959c9d6..14b997af3b4f7221ff6a2acded9af516d7773310 100644 --- a/crates/call2/src/shared_screen.rs +++ b/crates/call2/src/shared_screen.rs @@ -3,8 +3,8 @@ use anyhow::Result; use client::{proto::PeerId, User}; use futures::StreamExt; use gpui::{ - div, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement, Render, - SharedString, Task, View, ViewContext, VisualContext, WindowContext, + div, img, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement, + Render, SharedString, Styled, Task, View, ViewContext, VisualContext, WindowContext, }; use std::sync::{Arc, Weak}; use workspace::{item::Item, ItemNavHistory, WorkspaceId}; @@ -68,15 +68,11 @@ impl Render for SharedScreen { type Element = Div; fn render(&mut self, _: &mut ViewContext) -> Self::Element { let frame = self.frame.clone(); - let frame_id = self.current_frame_id; - self.current_frame_id = self.current_frame_id.wrapping_add(1); - div().children(frame.map(|_| { - ui::Label::new(frame_id.to_string()).color(ui::Color::Error) - // img().data(Arc::new(ImageData::new(image::ImageBuffer::new( - // frame.width() as u32, - // frame.height() as u32, - // )))) - })) + // let frame_id = self.current_frame_id; + // self.current_frame_id = self.current_frame_id.wrapping_add(1); + div() + .size_full() + .children(frame.map(|frame| img().size_full().surface(frame.image()))) } } // impl View for SharedScreen { diff --git a/crates/gpui2/build.rs b/crates/gpui2/build.rs index 6e8a0868b969c7a85627fd974f0f1bde793eb587..24e493cb812d6e7478b266cc621eea7cbc77b051 100644 --- a/crates/gpui2/build.rs +++ b/crates/gpui2/build.rs @@ -65,6 +65,8 @@ fn generate_shader_bindings() -> PathBuf { "MonochromeSprite".into(), "PolychromeSprite".into(), "PathSprite".into(), + "SurfaceInputIndex".into(), + "SurfaceBounds".into(), ]); config.no_includes = true; config.enumeration.prefix_with_name = true; diff --git a/crates/gpui2/src/elements/img.rs b/crates/gpui2/src/elements/img.rs index 2aece17b4707f4c9fa473e7d7388e9ffdc8093f8..1c1fbd0961fb0a86a1c407d803245d11ce1c72d6 100644 --- a/crates/gpui2/src/elements/img.rs +++ b/crates/gpui2/src/elements/img.rs @@ -5,6 +5,7 @@ use crate::{ IntoElement, LayoutId, Pixels, SharedString, StyleRefinement, Styled, WindowContext, }; use futures::FutureExt; +use media::core_video::CVImageBuffer; use util::ResultExt; #[derive(Clone, Debug)] @@ -12,6 +13,7 @@ pub enum ImageSource { /// Image content will be loaded from provided URI at render time. Uri(SharedString), Data(Arc), + Surface(CVImageBuffer), } impl From for ImageSource { @@ -26,6 +28,12 @@ impl From> for ImageSource { } } +impl From for ImageSource { + fn from(value: CVImageBuffer) -> Self { + Self::Surface(value) + } +} + pub struct Img { interactivity: Interactivity, source: Option, @@ -45,11 +53,17 @@ impl Img { self.source = Some(ImageSource::from(uri.into())); self } + pub fn data(mut self, data: Arc) -> Self { self.source = Some(ImageSource::from(data)); self } + pub fn surface(mut self, data: CVImageBuffer) -> Self { + self.source = Some(ImageSource::from(data)); + self + } + pub fn source(mut self, source: impl Into) -> Self { self.source = Some(source.into()); self @@ -85,36 +99,41 @@ impl Element for Img { element_state, cx, |style, _scroll_offset, cx| { - let corner_radii = style.corner_radii; - - if let Some(source) = self.source { - let image = match source { - ImageSource::Uri(uri) => { - let image_future = cx.image_cache.get(uri.clone()); - if let Some(data) = image_future - .clone() - .now_or_never() - .and_then(|result| result.ok()) - { - data - } else { - cx.spawn(|mut cx| async move { - if image_future.await.ok().is_some() { - cx.on_next_frame(|cx| cx.notify()); - } - }) - .detach(); - return; + let corner_radii = style.corner_radii.to_pixels(bounds.size, cx.rem_size()); + cx.with_z_index(1, |cx| { + if let Some(source) = self.source { + match source { + ImageSource::Uri(uri) => { + let image_future = cx.image_cache.get(uri.clone()); + if let Some(data) = image_future + .clone() + .now_or_never() + .and_then(|result| result.ok()) + { + cx.paint_image(bounds, corner_radii, data, self.grayscale) + .log_err(); + } else { + cx.spawn(|mut cx| async move { + if image_future.await.ok().is_some() { + cx.on_next_frame(|cx| cx.notify()); + } + }) + .detach(); + } + } + + ImageSource::Data(image) => { + cx.paint_image(bounds, corner_radii, image, self.grayscale) + .log_err(); + } + + ImageSource::Surface(surface) => { + // TODO: Add support for corner_radii and grayscale. + cx.paint_surface(bounds, surface); } - } - ImageSource::Data(image) => image, - }; - let corner_radii = corner_radii.to_pixels(bounds.size, cx.rem_size()); - cx.with_z_index(1, |cx| { - cx.paint_image(bounds, corner_radii, image, self.grayscale) - .log_err() - }); - } + }; + } + }); }, ) } diff --git a/crates/gpui2/src/platform/mac/metal_renderer.rs b/crates/gpui2/src/platform/mac/metal_renderer.rs index 0631c75de5e222e43f780f29fb9844f5e96c099c..19afb503324907a0b84bcac7722fc71ead05451d 100644 --- a/crates/gpui2/src/platform/mac/metal_renderer.rs +++ b/crates/gpui2/src/platform/mac/metal_renderer.rs @@ -1,7 +1,7 @@ use crate::{ point, size, AtlasTextureId, AtlasTextureKind, AtlasTile, Bounds, ContentMask, DevicePixels, Hsla, MetalAtlas, MonochromeSprite, Path, PathId, PathVertex, PolychromeSprite, PrimitiveBatch, - Quad, ScaledPixels, Scene, Shadow, Size, Underline, + Quad, ScaledPixels, Scene, Shadow, Size, Surface, Underline, }; use cocoa::{ base::{NO, YES}, @@ -9,6 +9,9 @@ use cocoa::{ quartzcore::AutoresizingMask, }; use collections::HashMap; +use core_foundation::base::TCFType; +use foreign_types::ForeignType; +use media::core_video::CVMetalTextureCache; use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; use objc::{self, msg_send, sel, sel_impl}; use smallvec::SmallVec; @@ -27,9 +30,11 @@ pub(crate) struct MetalRenderer { underlines_pipeline_state: metal::RenderPipelineState, monochrome_sprites_pipeline_state: metal::RenderPipelineState, polychrome_sprites_pipeline_state: metal::RenderPipelineState, + surfaces_pipeline_state: metal::RenderPipelineState, unit_vertices: metal::Buffer, instances: metal::Buffer, sprite_atlas: Arc, + core_video_texture_cache: CVMetalTextureCache, } impl MetalRenderer { @@ -143,6 +148,14 @@ impl MetalRenderer { "polychrome_sprite_fragment", MTLPixelFormat::BGRA8Unorm, ); + let surfaces_pipeline_state = build_pipeline_state( + &device, + &library, + "surfaces", + "surface_vertex", + "surface_fragment", + MTLPixelFormat::BGRA8Unorm, + ); let command_queue = device.new_command_queue(); let sprite_atlas = Arc::new(MetalAtlas::new(device.clone())); @@ -157,9 +170,11 @@ impl MetalRenderer { underlines_pipeline_state, monochrome_sprites_pipeline_state, polychrome_sprites_pipeline_state, + surfaces_pipeline_state, unit_vertices, instances, sprite_atlas, + core_video_texture_cache: CVMetalTextureCache::new(device.as_ptr()).unwrap(), } } @@ -268,6 +283,14 @@ impl MetalRenderer { command_encoder, ); } + PrimitiveBatch::Surfaces(surfaces) => { + self.draw_surfaces( + surfaces, + &mut instance_offset, + viewport_size, + command_encoder, + ); + } } } @@ -793,6 +816,102 @@ impl MetalRenderer { ); *offset = next_offset; } + + fn draw_surfaces( + &mut self, + surfaces: &[Surface], + offset: &mut usize, + viewport_size: Size, + command_encoder: &metal::RenderCommandEncoderRef, + ) { + command_encoder.set_render_pipeline_state(&self.surfaces_pipeline_state); + command_encoder.set_vertex_buffer( + SurfaceInputIndex::Vertices as u64, + Some(&self.unit_vertices), + 0, + ); + command_encoder.set_vertex_bytes( + SurfaceInputIndex::ViewportSize as u64, + mem::size_of_val(&viewport_size) as u64, + &viewport_size as *const Size as *const _, + ); + + for surface in surfaces { + let texture_size = size( + DevicePixels::from(surface.image_buffer.width() as i32), + DevicePixels::from(surface.image_buffer.height() as i32), + ); + + assert_eq!( + surface.image_buffer.pixel_format_type(), + media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ); + + let y_texture = self + .core_video_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + MTLPixelFormat::R8Unorm, + surface.image_buffer.plane_width(0), + surface.image_buffer.plane_height(0), + 0, + ) + .unwrap(); + let cb_cr_texture = self + .core_video_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + MTLPixelFormat::RG8Unorm, + surface.image_buffer.plane_width(1), + surface.image_buffer.plane_height(1), + 1, + ) + .unwrap(); + + align_offset(offset); + let next_offset = *offset + mem::size_of::(); + assert!( + next_offset <= INSTANCE_BUFFER_SIZE, + "instance buffer exhausted" + ); + + command_encoder.set_vertex_buffer( + SurfaceInputIndex::Surfaces as u64, + Some(&self.instances), + *offset as u64, + ); + command_encoder.set_vertex_bytes( + SurfaceInputIndex::TextureSize as u64, + mem::size_of_val(&texture_size) as u64, + &texture_size as *const Size as *const _, + ); + command_encoder.set_fragment_texture( + SurfaceInputIndex::YTexture as u64, + Some(y_texture.as_texture_ref()), + ); + command_encoder.set_fragment_texture( + SurfaceInputIndex::CbCrTexture as u64, + Some(cb_cr_texture.as_texture_ref()), + ); + + unsafe { + let buffer_contents = + (self.instances.contents() as *mut u8).add(*offset) as *mut SurfaceBounds; + ptr::write( + buffer_contents, + SurfaceBounds { + bounds: surface.bounds, + content_mask: surface.content_mask.clone(), + }, + ); + } + + command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6); + *offset = next_offset; + } + } } fn build_pipeline_state( @@ -898,6 +1017,16 @@ enum SpriteInputIndex { AtlasTexture = 4, } +#[repr(C)] +enum SurfaceInputIndex { + Vertices = 0, + Surfaces = 1, + ViewportSize = 2, + TextureSize = 3, + YTexture = 4, + CbCrTexture = 5, +} + #[repr(C)] enum PathRasterizationInputIndex { Vertices = 0, @@ -911,3 +1040,10 @@ pub struct PathSprite { pub color: Hsla, pub tile: AtlasTile, } + +#[derive(Clone, Debug, Eq, PartialEq)] +#[repr(C)] +pub struct SurfaceBounds { + pub bounds: Bounds, + pub content_mask: ContentMask, +} diff --git a/crates/gpui2/src/platform/mac/shaders.metal b/crates/gpui2/src/platform/mac/shaders.metal index 4def1c33b85a430d051376322dca47de93b9e70c..aba01b9d5b059da1c1df55c0a01120d8be10775b 100644 --- a/crates/gpui2/src/platform/mac/shaders.metal +++ b/crates/gpui2/src/platform/mac/shaders.metal @@ -469,6 +469,58 @@ fragment float4 path_sprite_fragment( return color; } +struct SurfaceVertexOutput { + float4 position [[position]]; + float2 texture_position; + float clip_distance [[clip_distance]][4]; +}; + +struct SurfaceFragmentInput { + float4 position [[position]]; + float2 texture_position; +}; + +vertex SurfaceVertexOutput surface_vertex( + uint unit_vertex_id [[vertex_id]], uint surface_id [[instance_id]], + constant float2 *unit_vertices [[buffer(SurfaceInputIndex_Vertices)]], + constant SurfaceBounds *surfaces [[buffer(SurfaceInputIndex_Surfaces)]], + constant Size_DevicePixels *viewport_size + [[buffer(SurfaceInputIndex_ViewportSize)]], + constant Size_DevicePixels *texture_size + [[buffer(SurfaceInputIndex_TextureSize)]]) { + float2 unit_vertex = unit_vertices[unit_vertex_id]; + SurfaceBounds surface = surfaces[surface_id]; + float4 device_position = + to_device_position(unit_vertex, surface.bounds, viewport_size); + float4 clip_distance = distance_from_clip_rect(unit_vertex, surface.bounds, + surface.content_mask.bounds); + // We are going to copy the whole texture, so the texture position corresponds + // to the current vertex of the unit triangle. + float2 texture_position = unit_vertex; + return SurfaceVertexOutput{ + device_position, + texture_position, + {clip_distance.x, clip_distance.y, clip_distance.z, clip_distance.w}}; +} + +fragment float4 surface_fragment(SurfaceFragmentInput input [[stage_in]], + texture2d y_texture + [[texture(SurfaceInputIndex_YTexture)]], + texture2d cb_cr_texture + [[texture(SurfaceInputIndex_CbCrTexture)]]) { + constexpr sampler texture_sampler(mag_filter::linear, min_filter::linear); + const float4x4 ycbcrToRGBTransform = + float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), + float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), + float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), + float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f)); + float4 ycbcr = float4( + y_texture.sample(texture_sampler, input.texture_position).r, + cb_cr_texture.sample(texture_sampler, input.texture_position).rg, 1.0); + + return ycbcrToRGBTransform * ycbcr; +} + float4 hsla_to_rgba(Hsla hsla) { float h = hsla.h * 6.0; // Now, it's an angle but scaled in [0, 6) range float s = hsla.s; diff --git a/crates/gpui2/src/scene.rs b/crates/gpui2/src/scene.rs index 87e89adfa0e15d18512d2a1a748ea5a212a48489..549260560236ffb43179caf6cb8fce5340c10b7a 100644 --- a/crates/gpui2/src/scene.rs +++ b/crates/gpui2/src/scene.rs @@ -25,6 +25,7 @@ pub(crate) struct SceneBuilder { underlines: Vec, monochrome_sprites: Vec, polychrome_sprites: Vec, + surfaces: Vec, } impl Default for SceneBuilder { @@ -38,6 +39,7 @@ impl Default for SceneBuilder { underlines: Vec::new(), monochrome_sprites: Vec::new(), polychrome_sprites: Vec::new(), + surfaces: Vec::new(), } } } @@ -120,6 +122,7 @@ impl SceneBuilder { (PrimitiveKind::PolychromeSprite, ix) => { self.polychrome_sprites[ix].order = draw_order as DrawOrder } + (PrimitiveKind::Surface, ix) => self.surfaces[ix].order = draw_order as DrawOrder, } } @@ -129,6 +132,7 @@ impl SceneBuilder { self.underlines.sort_unstable(); self.monochrome_sprites.sort_unstable(); self.polychrome_sprites.sort_unstable(); + self.surfaces.sort_unstable(); Scene { shadows: mem::take(&mut self.shadows), @@ -137,6 +141,7 @@ impl SceneBuilder { underlines: mem::take(&mut self.underlines), monochrome_sprites: mem::take(&mut self.monochrome_sprites), polychrome_sprites: mem::take(&mut self.polychrome_sprites), + surfaces: mem::take(&mut self.surfaces), } } @@ -185,6 +190,10 @@ impl SceneBuilder { sprite.order = layer_id; self.polychrome_sprites.push(sprite); } + Primitive::Surface(mut surface) => { + surface.order = layer_id; + self.surfaces.push(surface); + } } } } @@ -196,6 +205,7 @@ pub(crate) struct Scene { pub underlines: Vec, pub monochrome_sprites: Vec, pub polychrome_sprites: Vec, + pub surfaces: Vec, } impl Scene { @@ -224,6 +234,9 @@ impl Scene { polychrome_sprites: &self.polychrome_sprites, polychrome_sprites_start: 0, polychrome_sprites_iter: self.polychrome_sprites.iter().peekable(), + surfaces: &self.surfaces, + surfaces_start: 0, + surfaces_iter: self.surfaces.iter().peekable(), } } } @@ -247,6 +260,9 @@ struct BatchIterator<'a> { polychrome_sprites: &'a [PolychromeSprite], polychrome_sprites_start: usize, polychrome_sprites_iter: Peekable>, + surfaces: &'a [Surface], + surfaces_start: usize, + surfaces_iter: Peekable>, } impl<'a> Iterator for BatchIterator<'a> { @@ -272,6 +288,10 @@ impl<'a> Iterator for BatchIterator<'a> { self.polychrome_sprites_iter.peek().map(|s| s.order), PrimitiveKind::PolychromeSprite, ), + ( + self.surfaces_iter.peek().map(|s| s.order), + PrimitiveKind::Surface, + ), ]; orders_and_kinds.sort_by_key(|(order, kind)| (order.unwrap_or(u32::MAX), *kind)); @@ -378,6 +398,21 @@ impl<'a> Iterator for BatchIterator<'a> { sprites: &self.polychrome_sprites[sprites_start..sprites_end], }) } + PrimitiveKind::Surface => { + let surfaces_start = self.surfaces_start; + let mut surfaces_end = surfaces_start; + while self + .surfaces_iter + .next_if(|surface| surface.order <= max_order) + .is_some() + { + surfaces_end += 1; + } + self.surfaces_start = surfaces_end; + Some(PrimitiveBatch::Surfaces( + &self.surfaces[surfaces_start..surfaces_end], + )) + } } } } @@ -391,6 +426,7 @@ pub enum PrimitiveKind { Underline, MonochromeSprite, PolychromeSprite, + Surface, } pub enum Primitive { @@ -400,6 +436,7 @@ pub enum Primitive { Underline(Underline), MonochromeSprite(MonochromeSprite), PolychromeSprite(PolychromeSprite), + Surface(Surface), } impl Primitive { @@ -411,6 +448,7 @@ impl Primitive { Primitive::Underline(underline) => &underline.bounds, Primitive::MonochromeSprite(sprite) => &sprite.bounds, Primitive::PolychromeSprite(sprite) => &sprite.bounds, + Primitive::Surface(surface) => &surface.bounds, } } @@ -422,6 +460,7 @@ impl Primitive { Primitive::Underline(underline) => &underline.content_mask, Primitive::MonochromeSprite(sprite) => &sprite.content_mask, Primitive::PolychromeSprite(sprite) => &sprite.content_mask, + Primitive::Surface(surface) => &surface.content_mask, } } } @@ -440,6 +479,7 @@ pub(crate) enum PrimitiveBatch<'a> { texture_id: AtlasTextureId, sprites: &'a [PolychromeSprite], }, + Surfaces(&'a [Surface]), } #[derive(Default, Debug, Clone, Eq, PartialEq)] @@ -593,6 +633,32 @@ impl From for Primitive { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Surface { + pub order: u32, + pub bounds: Bounds, + pub content_mask: ContentMask, + pub image_buffer: media::core_video::CVImageBuffer, +} + +impl Ord for Surface { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.order.cmp(&other.order) + } +} + +impl PartialOrd for Surface { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl From for Primitive { + fn from(surface: Surface) -> Self { + Primitive::Surface(surface) + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub(crate) struct PathId(pub(crate) usize); diff --git a/crates/gpui2/src/window.rs b/crates/gpui2/src/window.rs index 20561c544368b6b9c41124194c07ada33145c968..1f98860f09c40eed3a427225dc9aa22fb7494986 100644 --- a/crates/gpui2/src/window.rs +++ b/crates/gpui2/src/window.rs @@ -8,8 +8,8 @@ use crate::{ MouseUpEvent, Path, Pixels, PlatformAtlas, PlatformDisplay, PlatformInputHandler, PlatformWindow, Point, PolychromeSprite, PromptLevel, Quad, Render, RenderGlyphParams, RenderImageParams, RenderSvgParams, ScaledPixels, SceneBuilder, Shadow, SharedString, Size, - Style, SubscriberSet, Subscription, TaffyLayoutEngine, Task, Underline, UnderlineStyle, View, - VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS, + Style, SubscriberSet, Subscription, Surface, TaffyLayoutEngine, Task, Underline, + UnderlineStyle, View, VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS, }; use anyhow::{anyhow, Context as _, Result}; use collections::HashMap; @@ -18,6 +18,7 @@ use futures::{ channel::{mpsc, oneshot}, StreamExt, }; +use media::core_video::CVImageBuffer; use parking_lot::RwLock; use slotmap::SlotMap; use smallvec::SmallVec; @@ -1090,6 +1091,23 @@ impl<'a> WindowContext<'a> { Ok(()) } + /// Paint a surface into the scene for the current frame at the current z-index. + pub fn paint_surface(&mut self, bounds: Bounds, image_buffer: CVImageBuffer) { + let scale_factor = self.scale_factor(); + let bounds = bounds.scale(scale_factor); + let content_mask = self.content_mask().scale(scale_factor); + let window = &mut *self.window; + window.current_frame.scene_builder.insert( + &window.current_frame.z_index_stack, + Surface { + order: 0, + bounds, + content_mask, + image_buffer, + }, + ); + } + /// Draw pixels to the display for this window based on the contents of its scene. pub(crate) fn draw(&mut self) { let root_view = self.window.root_view.take().unwrap();