Detailed changes
@@ -3,8 +3,8 @@ use anyhow::Result;
use client::{proto::PeerId, User};
use futures::StreamExt;
use gpui::{
- div, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement, Render,
- SharedString, Task, View, ViewContext, VisualContext, WindowContext,
+ div, img, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement,
+ Render, SharedString, Styled, Task, View, ViewContext, VisualContext, WindowContext,
};
use std::sync::{Arc, Weak};
use workspace::{item::Item, ItemNavHistory, WorkspaceId};
@@ -68,15 +68,11 @@ impl Render for SharedScreen {
type Element = Div;
fn render(&mut self, _: &mut ViewContext<Self>) -> Self::Element {
let frame = self.frame.clone();
- let frame_id = self.current_frame_id;
- self.current_frame_id = self.current_frame_id.wrapping_add(1);
- div().children(frame.map(|_| {
- ui::Label::new(frame_id.to_string()).color(ui::Color::Error)
- // img().data(Arc::new(ImageData::new(image::ImageBuffer::new(
- // frame.width() as u32,
- // frame.height() as u32,
- // ))))
- }))
+ // let frame_id = self.current_frame_id;
+ // self.current_frame_id = self.current_frame_id.wrapping_add(1);
+ div()
+ .size_full()
+ .children(frame.map(|frame| img().size_full().surface(frame.image())))
}
}
// impl View for SharedScreen {
@@ -65,6 +65,8 @@ fn generate_shader_bindings() -> PathBuf {
"MonochromeSprite".into(),
"PolychromeSprite".into(),
"PathSprite".into(),
+ "SurfaceInputIndex".into(),
+ "SurfaceBounds".into(),
]);
config.no_includes = true;
config.enumeration.prefix_with_name = true;
@@ -5,6 +5,7 @@ use crate::{
IntoElement, LayoutId, Pixels, SharedString, StyleRefinement, Styled, WindowContext,
};
use futures::FutureExt;
+use media::core_video::CVImageBuffer;
use util::ResultExt;
#[derive(Clone, Debug)]
@@ -12,6 +13,7 @@ pub enum ImageSource {
/// Image content will be loaded from provided URI at render time.
Uri(SharedString),
Data(Arc<ImageData>),
+ Surface(CVImageBuffer),
}
impl From<SharedString> for ImageSource {
@@ -26,6 +28,12 @@ impl From<Arc<ImageData>> for ImageSource {
}
}
+impl From<CVImageBuffer> for ImageSource {
+ fn from(value: CVImageBuffer) -> Self {
+ Self::Surface(value)
+ }
+}
+
pub struct Img {
interactivity: Interactivity,
source: Option<ImageSource>,
@@ -45,11 +53,17 @@ impl Img {
self.source = Some(ImageSource::from(uri.into()));
self
}
+
pub fn data(mut self, data: Arc<ImageData>) -> Self {
self.source = Some(ImageSource::from(data));
self
}
+ pub fn surface(mut self, data: CVImageBuffer) -> Self {
+ self.source = Some(ImageSource::from(data));
+ self
+ }
+
pub fn source(mut self, source: impl Into<ImageSource>) -> Self {
self.source = Some(source.into());
self
@@ -85,36 +99,41 @@ impl Element for Img {
element_state,
cx,
|style, _scroll_offset, cx| {
- let corner_radii = style.corner_radii;
-
- if let Some(source) = self.source {
- let image = match source {
- ImageSource::Uri(uri) => {
- let image_future = cx.image_cache.get(uri.clone());
- if let Some(data) = image_future
- .clone()
- .now_or_never()
- .and_then(|result| result.ok())
- {
- data
- } else {
- cx.spawn(|mut cx| async move {
- if image_future.await.ok().is_some() {
- cx.on_next_frame(|cx| cx.notify());
- }
- })
- .detach();
- return;
+ let corner_radii = style.corner_radii.to_pixels(bounds.size, cx.rem_size());
+ cx.with_z_index(1, |cx| {
+ if let Some(source) = self.source {
+ match source {
+ ImageSource::Uri(uri) => {
+ let image_future = cx.image_cache.get(uri.clone());
+ if let Some(data) = image_future
+ .clone()
+ .now_or_never()
+ .and_then(|result| result.ok())
+ {
+ cx.paint_image(bounds, corner_radii, data, self.grayscale)
+ .log_err();
+ } else {
+ cx.spawn(|mut cx| async move {
+ if image_future.await.ok().is_some() {
+ cx.on_next_frame(|cx| cx.notify());
+ }
+ })
+ .detach();
+ }
+ }
+
+ ImageSource::Data(image) => {
+ cx.paint_image(bounds, corner_radii, image, self.grayscale)
+ .log_err();
+ }
+
+ ImageSource::Surface(surface) => {
+ // TODO: Add support for corner_radii and grayscale.
+ cx.paint_surface(bounds, surface);
}
- }
- ImageSource::Data(image) => image,
- };
- let corner_radii = corner_radii.to_pixels(bounds.size, cx.rem_size());
- cx.with_z_index(1, |cx| {
- cx.paint_image(bounds, corner_radii, image, self.grayscale)
- .log_err()
- });
- }
+ };
+ }
+ });
},
)
}
@@ -1,7 +1,7 @@
use crate::{
point, size, AtlasTextureId, AtlasTextureKind, AtlasTile, Bounds, ContentMask, DevicePixels,
Hsla, MetalAtlas, MonochromeSprite, Path, PathId, PathVertex, PolychromeSprite, PrimitiveBatch,
- Quad, ScaledPixels, Scene, Shadow, Size, Underline,
+ Quad, ScaledPixels, Scene, Shadow, Size, Surface, Underline,
};
use cocoa::{
base::{NO, YES},
@@ -9,6 +9,9 @@ use cocoa::{
quartzcore::AutoresizingMask,
};
use collections::HashMap;
+use core_foundation::base::TCFType;
+use foreign_types::ForeignType;
+use media::core_video::CVMetalTextureCache;
use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange};
use objc::{self, msg_send, sel, sel_impl};
use smallvec::SmallVec;
@@ -27,9 +30,11 @@ pub(crate) struct MetalRenderer {
underlines_pipeline_state: metal::RenderPipelineState,
monochrome_sprites_pipeline_state: metal::RenderPipelineState,
polychrome_sprites_pipeline_state: metal::RenderPipelineState,
+ surfaces_pipeline_state: metal::RenderPipelineState,
unit_vertices: metal::Buffer,
instances: metal::Buffer,
sprite_atlas: Arc<MetalAtlas>,
+ core_video_texture_cache: CVMetalTextureCache,
}
impl MetalRenderer {
@@ -143,6 +148,14 @@ impl MetalRenderer {
"polychrome_sprite_fragment",
MTLPixelFormat::BGRA8Unorm,
);
+ let surfaces_pipeline_state = build_pipeline_state(
+ &device,
+ &library,
+ "surfaces",
+ "surface_vertex",
+ "surface_fragment",
+ MTLPixelFormat::BGRA8Unorm,
+ );
let command_queue = device.new_command_queue();
let sprite_atlas = Arc::new(MetalAtlas::new(device.clone()));
@@ -157,9 +170,11 @@ impl MetalRenderer {
underlines_pipeline_state,
monochrome_sprites_pipeline_state,
polychrome_sprites_pipeline_state,
+ surfaces_pipeline_state,
unit_vertices,
instances,
sprite_atlas,
+ core_video_texture_cache: CVMetalTextureCache::new(device.as_ptr()).unwrap(),
}
}
@@ -268,6 +283,14 @@ impl MetalRenderer {
command_encoder,
);
}
+ PrimitiveBatch::Surfaces(surfaces) => {
+ self.draw_surfaces(
+ surfaces,
+ &mut instance_offset,
+ viewport_size,
+ command_encoder,
+ );
+ }
}
}
@@ -793,6 +816,102 @@ impl MetalRenderer {
);
*offset = next_offset;
}
+
+ fn draw_surfaces(
+ &mut self,
+ surfaces: &[Surface],
+ offset: &mut usize,
+ viewport_size: Size<DevicePixels>,
+ command_encoder: &metal::RenderCommandEncoderRef,
+ ) {
+ command_encoder.set_render_pipeline_state(&self.surfaces_pipeline_state);
+ command_encoder.set_vertex_buffer(
+ SurfaceInputIndex::Vertices as u64,
+ Some(&self.unit_vertices),
+ 0,
+ );
+ command_encoder.set_vertex_bytes(
+ SurfaceInputIndex::ViewportSize as u64,
+ mem::size_of_val(&viewport_size) as u64,
+ &viewport_size as *const Size<DevicePixels> as *const _,
+ );
+
+ for surface in surfaces {
+ let texture_size = size(
+ DevicePixels::from(surface.image_buffer.width() as i32),
+ DevicePixels::from(surface.image_buffer.height() as i32),
+ );
+
+ assert_eq!(
+ surface.image_buffer.pixel_format_type(),
+ media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+ );
+
+ let y_texture = self
+ .core_video_texture_cache
+ .create_texture_from_image(
+ surface.image_buffer.as_concrete_TypeRef(),
+ ptr::null(),
+ MTLPixelFormat::R8Unorm,
+ surface.image_buffer.plane_width(0),
+ surface.image_buffer.plane_height(0),
+ 0,
+ )
+ .unwrap();
+ let cb_cr_texture = self
+ .core_video_texture_cache
+ .create_texture_from_image(
+ surface.image_buffer.as_concrete_TypeRef(),
+ ptr::null(),
+ MTLPixelFormat::RG8Unorm,
+ surface.image_buffer.plane_width(1),
+ surface.image_buffer.plane_height(1),
+ 1,
+ )
+ .unwrap();
+
+ align_offset(offset);
+ let next_offset = *offset + mem::size_of::<Surface>();
+ assert!(
+ next_offset <= INSTANCE_BUFFER_SIZE,
+ "instance buffer exhausted"
+ );
+
+ command_encoder.set_vertex_buffer(
+ SurfaceInputIndex::Surfaces as u64,
+ Some(&self.instances),
+ *offset as u64,
+ );
+ command_encoder.set_vertex_bytes(
+ SurfaceInputIndex::TextureSize as u64,
+ mem::size_of_val(&texture_size) as u64,
+ &texture_size as *const Size<DevicePixels> as *const _,
+ );
+ command_encoder.set_fragment_texture(
+ SurfaceInputIndex::YTexture as u64,
+ Some(y_texture.as_texture_ref()),
+ );
+ command_encoder.set_fragment_texture(
+ SurfaceInputIndex::CbCrTexture as u64,
+ Some(cb_cr_texture.as_texture_ref()),
+ );
+
+ unsafe {
+ let buffer_contents =
+ (self.instances.contents() as *mut u8).add(*offset) as *mut SurfaceBounds;
+ ptr::write(
+ buffer_contents,
+ SurfaceBounds {
+ bounds: surface.bounds,
+ content_mask: surface.content_mask.clone(),
+ },
+ );
+ }
+
+ command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6);
+ *offset = next_offset;
+ }
+ }
}
fn build_pipeline_state(
@@ -898,6 +1017,16 @@ enum SpriteInputIndex {
AtlasTexture = 4,
}
+#[repr(C)]
+enum SurfaceInputIndex {
+ Vertices = 0,
+ Surfaces = 1,
+ ViewportSize = 2,
+ TextureSize = 3,
+ YTexture = 4,
+ CbCrTexture = 5,
+}
+
#[repr(C)]
enum PathRasterizationInputIndex {
Vertices = 0,
@@ -911,3 +1040,10 @@ pub struct PathSprite {
pub color: Hsla,
pub tile: AtlasTile,
}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[repr(C)]
+pub struct SurfaceBounds {
+ pub bounds: Bounds<ScaledPixels>,
+ pub content_mask: ContentMask<ScaledPixels>,
+}
@@ -469,6 +469,58 @@ fragment float4 path_sprite_fragment(
return color;
}
+struct SurfaceVertexOutput {
+ float4 position [[position]];
+ float2 texture_position;
+ float clip_distance [[clip_distance]][4];
+};
+
+struct SurfaceFragmentInput {
+ float4 position [[position]];
+ float2 texture_position;
+};
+
+vertex SurfaceVertexOutput surface_vertex(
+ uint unit_vertex_id [[vertex_id]], uint surface_id [[instance_id]],
+ constant float2 *unit_vertices [[buffer(SurfaceInputIndex_Vertices)]],
+ constant SurfaceBounds *surfaces [[buffer(SurfaceInputIndex_Surfaces)]],
+ constant Size_DevicePixels *viewport_size
+ [[buffer(SurfaceInputIndex_ViewportSize)]],
+ constant Size_DevicePixels *texture_size
+ [[buffer(SurfaceInputIndex_TextureSize)]]) {
+ float2 unit_vertex = unit_vertices[unit_vertex_id];
+ SurfaceBounds surface = surfaces[surface_id];
+ float4 device_position =
+ to_device_position(unit_vertex, surface.bounds, viewport_size);
+ float4 clip_distance = distance_from_clip_rect(unit_vertex, surface.bounds,
+ surface.content_mask.bounds);
+ // We are going to copy the whole texture, so the texture position corresponds
+ // to the current vertex of the unit triangle.
+ float2 texture_position = unit_vertex;
+ return SurfaceVertexOutput{
+ device_position,
+ texture_position,
+ {clip_distance.x, clip_distance.y, clip_distance.z, clip_distance.w}};
+}
+
+fragment float4 surface_fragment(SurfaceFragmentInput input [[stage_in]],
+ texture2d<float> y_texture
+ [[texture(SurfaceInputIndex_YTexture)]],
+ texture2d<float> cb_cr_texture
+ [[texture(SurfaceInputIndex_CbCrTexture)]]) {
+ constexpr sampler texture_sampler(mag_filter::linear, min_filter::linear);
+ const float4x4 ycbcrToRGBTransform =
+ float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
+ float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
+ float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
+ float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f));
+ float4 ycbcr = float4(
+ y_texture.sample(texture_sampler, input.texture_position).r,
+ cb_cr_texture.sample(texture_sampler, input.texture_position).rg, 1.0);
+
+ return ycbcrToRGBTransform * ycbcr;
+}
+
float4 hsla_to_rgba(Hsla hsla) {
float h = hsla.h * 6.0; // Now, it's an angle but scaled in [0, 6) range
float s = hsla.s;
@@ -25,6 +25,7 @@ pub(crate) struct SceneBuilder {
underlines: Vec<Underline>,
monochrome_sprites: Vec<MonochromeSprite>,
polychrome_sprites: Vec<PolychromeSprite>,
+ surfaces: Vec<Surface>,
}
impl Default for SceneBuilder {
@@ -38,6 +39,7 @@ impl Default for SceneBuilder {
underlines: Vec::new(),
monochrome_sprites: Vec::new(),
polychrome_sprites: Vec::new(),
+ surfaces: Vec::new(),
}
}
}
@@ -120,6 +122,7 @@ impl SceneBuilder {
(PrimitiveKind::PolychromeSprite, ix) => {
self.polychrome_sprites[ix].order = draw_order as DrawOrder
}
+ (PrimitiveKind::Surface, ix) => self.surfaces[ix].order = draw_order as DrawOrder,
}
}
@@ -129,6 +132,7 @@ impl SceneBuilder {
self.underlines.sort_unstable();
self.monochrome_sprites.sort_unstable();
self.polychrome_sprites.sort_unstable();
+ self.surfaces.sort_unstable();
Scene {
shadows: mem::take(&mut self.shadows),
@@ -137,6 +141,7 @@ impl SceneBuilder {
underlines: mem::take(&mut self.underlines),
monochrome_sprites: mem::take(&mut self.monochrome_sprites),
polychrome_sprites: mem::take(&mut self.polychrome_sprites),
+ surfaces: mem::take(&mut self.surfaces),
}
}
@@ -185,6 +190,10 @@ impl SceneBuilder {
sprite.order = layer_id;
self.polychrome_sprites.push(sprite);
}
+ Primitive::Surface(mut surface) => {
+ surface.order = layer_id;
+ self.surfaces.push(surface);
+ }
}
}
}
@@ -196,6 +205,7 @@ pub(crate) struct Scene {
pub underlines: Vec<Underline>,
pub monochrome_sprites: Vec<MonochromeSprite>,
pub polychrome_sprites: Vec<PolychromeSprite>,
+ pub surfaces: Vec<Surface>,
}
impl Scene {
@@ -224,6 +234,9 @@ impl Scene {
polychrome_sprites: &self.polychrome_sprites,
polychrome_sprites_start: 0,
polychrome_sprites_iter: self.polychrome_sprites.iter().peekable(),
+ surfaces: &self.surfaces,
+ surfaces_start: 0,
+ surfaces_iter: self.surfaces.iter().peekable(),
}
}
}
@@ -247,6 +260,9 @@ struct BatchIterator<'a> {
polychrome_sprites: &'a [PolychromeSprite],
polychrome_sprites_start: usize,
polychrome_sprites_iter: Peekable<slice::Iter<'a, PolychromeSprite>>,
+ surfaces: &'a [Surface],
+ surfaces_start: usize,
+ surfaces_iter: Peekable<slice::Iter<'a, Surface>>,
}
impl<'a> Iterator for BatchIterator<'a> {
@@ -272,6 +288,10 @@ impl<'a> Iterator for BatchIterator<'a> {
self.polychrome_sprites_iter.peek().map(|s| s.order),
PrimitiveKind::PolychromeSprite,
),
+ (
+ self.surfaces_iter.peek().map(|s| s.order),
+ PrimitiveKind::Surface,
+ ),
];
orders_and_kinds.sort_by_key(|(order, kind)| (order.unwrap_or(u32::MAX), *kind));
@@ -378,6 +398,21 @@ impl<'a> Iterator for BatchIterator<'a> {
sprites: &self.polychrome_sprites[sprites_start..sprites_end],
})
}
+ PrimitiveKind::Surface => {
+ let surfaces_start = self.surfaces_start;
+ let mut surfaces_end = surfaces_start;
+ while self
+ .surfaces_iter
+ .next_if(|surface| surface.order <= max_order)
+ .is_some()
+ {
+ surfaces_end += 1;
+ }
+ self.surfaces_start = surfaces_end;
+ Some(PrimitiveBatch::Surfaces(
+ &self.surfaces[surfaces_start..surfaces_end],
+ ))
+ }
}
}
}
@@ -391,6 +426,7 @@ pub enum PrimitiveKind {
Underline,
MonochromeSprite,
PolychromeSprite,
+ Surface,
}
pub enum Primitive {
@@ -400,6 +436,7 @@ pub enum Primitive {
Underline(Underline),
MonochromeSprite(MonochromeSprite),
PolychromeSprite(PolychromeSprite),
+ Surface(Surface),
}
impl Primitive {
@@ -411,6 +448,7 @@ impl Primitive {
Primitive::Underline(underline) => &underline.bounds,
Primitive::MonochromeSprite(sprite) => &sprite.bounds,
Primitive::PolychromeSprite(sprite) => &sprite.bounds,
+ Primitive::Surface(surface) => &surface.bounds,
}
}
@@ -422,6 +460,7 @@ impl Primitive {
Primitive::Underline(underline) => &underline.content_mask,
Primitive::MonochromeSprite(sprite) => &sprite.content_mask,
Primitive::PolychromeSprite(sprite) => &sprite.content_mask,
+ Primitive::Surface(surface) => &surface.content_mask,
}
}
}
@@ -440,6 +479,7 @@ pub(crate) enum PrimitiveBatch<'a> {
texture_id: AtlasTextureId,
sprites: &'a [PolychromeSprite],
},
+ Surfaces(&'a [Surface]),
}
#[derive(Default, Debug, Clone, Eq, PartialEq)]
@@ -593,6 +633,32 @@ impl From<PolychromeSprite> for Primitive {
}
}
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Surface {
+ pub order: u32,
+ pub bounds: Bounds<ScaledPixels>,
+ pub content_mask: ContentMask<ScaledPixels>,
+ pub image_buffer: media::core_video::CVImageBuffer,
+}
+
+impl Ord for Surface {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.order.cmp(&other.order)
+ }
+}
+
+impl PartialOrd for Surface {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl From<Surface> for Primitive {
+ fn from(surface: Surface) -> Self {
+ Primitive::Surface(surface)
+ }
+}
+
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) struct PathId(pub(crate) usize);
@@ -8,8 +8,8 @@ use crate::{
MouseUpEvent, Path, Pixels, PlatformAtlas, PlatformDisplay, PlatformInputHandler,
PlatformWindow, Point, PolychromeSprite, PromptLevel, Quad, Render, RenderGlyphParams,
RenderImageParams, RenderSvgParams, ScaledPixels, SceneBuilder, Shadow, SharedString, Size,
- Style, SubscriberSet, Subscription, TaffyLayoutEngine, Task, Underline, UnderlineStyle, View,
- VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS,
+ Style, SubscriberSet, Subscription, Surface, TaffyLayoutEngine, Task, Underline,
+ UnderlineStyle, View, VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS,
};
use anyhow::{anyhow, Context as _, Result};
use collections::HashMap;
@@ -18,6 +18,7 @@ use futures::{
channel::{mpsc, oneshot},
StreamExt,
};
+use media::core_video::CVImageBuffer;
use parking_lot::RwLock;
use slotmap::SlotMap;
use smallvec::SmallVec;
@@ -1090,6 +1091,23 @@ impl<'a> WindowContext<'a> {
Ok(())
}
+ /// Paint a surface into the scene for the current frame at the current z-index.
+ pub fn paint_surface(&mut self, bounds: Bounds<Pixels>, image_buffer: CVImageBuffer) {
+ let scale_factor = self.scale_factor();
+ let bounds = bounds.scale(scale_factor);
+ let content_mask = self.content_mask().scale(scale_factor);
+ let window = &mut *self.window;
+ window.current_frame.scene_builder.insert(
+ &window.current_frame.z_index_stack,
+ Surface {
+ order: 0,
+ bounds,
+ content_mask,
+ image_buffer,
+ },
+ );
+ }
+
/// Draw pixels to the display for this window based on the contents of its scene.
pub(crate) fn draw(&mut self) {
let root_view = self.window.root_view.take().unwrap();