From c2afc2271b1dd420f482217cb0fa9dafcd17aab6 Mon Sep 17 00:00:00 2001 From: Michael Sloan Date: Fri, 4 Apr 2025 15:31:03 -0600 Subject: [PATCH] Use scap library to implement screensharing on X11 (#27807) While `scap` does have support for Wayland and Windows, but haven't seen screensharing work properly there yet. So for now just adding support for X11 screensharing. WIP branches for enabling wayland and windows support: * https://github.com/zed-industries/zed/tree/wayland-screenshare * https://github.com/zed-industries/zed/tree/windows-screenshare Release Notes: - Added support for screensharing on X11 (Linux) --------- Co-authored-by: Conrad Co-authored-by: Mikayla Co-authored-by: Junkui Zhang <364772080@qq.com> --- Cargo.lock | 152 ++++++++++ Cargo.toml | 11 +- crates/gpui/Cargo.toml | 14 +- crates/gpui/src/app.rs | 5 + crates/gpui/src/platform.rs | 12 +- crates/gpui/src/platform/linux.rs | 3 + .../src/platform/linux/headless/client.rs | 22 +- crates/gpui/src/platform/linux/platform.rs | 13 +- .../gpui/src/platform/linux/wayland/client.rs | 24 +- crates/gpui/src/platform/linux/x11/client.rs | 19 +- crates/gpui/src/platform/mac/platform.rs | 4 + .../gpui/src/platform/mac/screen_capture.rs | 14 +- .../gpui/src/platform/scap_screen_capture.rs | 282 ++++++++++++++++++ crates/gpui/src/platform/test/platform.rs | 18 +- crates/gpui/src/platform/windows/platform.rs | 4 + crates/livekit_client/Cargo.toml | 9 +- .../src/livekit_client/playback.rs | 46 ++- crates/title_bar/src/collab.rs | 5 +- crates/workspace/src/workspace.rs | 12 - script/linux | 4 + 20 files changed, 624 insertions(+), 49 deletions(-) create mode 100644 crates/gpui/src/platform/scap_screen_capture.rs diff --git a/Cargo.lock b/Cargo.lock index 3a1fbe5e179fa18a03960aace5d9003bd3706de3..0073e341a65d25e6aa95fb0785f387e4f8ffe72e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3454,6 +3454,19 @@ dependencies = [ "libc", ] +[[package]] +name = "core-graphics-helmer-fork" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32eb7c354ae9f6d437a6039099ce7ecd049337a8109b23d73e48e8ffba8e9cd5" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.9.4", + "core-graphics-types 0.1.3", + "foreign-types 0.5.0", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.1.3" @@ -4429,6 +4442,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "displaydoc" version = "0.2.5" @@ -6099,6 +6118,7 @@ dependencies = [ "refineable", "reqwest_client", "resvg", + "scap", "schemars", "seahash", "semantic_version", @@ -8125,6 +8145,7 @@ dependencies = [ "objc", "parking_lot", "postage", + "scap", "serde", "serde_json", "sha2", @@ -9136,6 +9157,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ "malloc_buf", + "objc_exception", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", ] [[package]] @@ -9340,6 +9373,24 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "object" version = "0.36.7" @@ -11146,6 +11197,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.32.0" @@ -12374,6 +12434,27 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scap" +version = "0.0.8" +source = "git+https://github.com/zed-industries/scap?rev=5715067104794aa356977c543e2f3e95c6183044#5715067104794aa356977c543e2f3e95c6183044" +dependencies = [ + "anyhow", + "cocoa 0.25.0", + "core-graphics-helmer-fork", + "log", + "objc", + "rand 0.8.5", + "screencapturekit", + "screencapturekit-sys", + "sysinfo", + "tao-core-video-sys", + "windows 0.61.1", + "windows-capture", + "x11", + "xcb", +] + [[package]] name = "schannel" version = "0.1.27" @@ -12440,6 +12521,29 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" +[[package]] +name = "screencapturekit" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5eeeb57ac94960cfe5ff4c402be6585ae4c8d29a2cf41b276048c2e849d64e" +dependencies = [ + "screencapturekit-sys", +] + +[[package]] +name = "screencapturekit-sys" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22411b57f7d49e7fe08025198813ee6fd65e1ee5eff4ebc7880c12c82bde4c60" +dependencies = [ + "block", + "dispatch", + "objc", + "objc-foundation", + "objc_id", + "once_cell", +] + [[package]] name = "scrypt" version = "0.11.0" @@ -14002,6 +14106,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bdb6fa0dfa67b38c1e66b7041ba9dcf23b99d8121907cd31c807a332f7a0bbb" +[[package]] +name = "tao-core-video-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271450eb289cb4d8d0720c6ce70c72c8c858c93dd61fc625881616752e6b98f6" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "objc", +] + [[package]] name = "tap" version = "1.0.1" @@ -16671,6 +16787,20 @@ dependencies = [ "windows-numerics", ] +[[package]] +name = "windows-capture" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6001b777f61cafce437201de46a019ed7f4afed3b669f02e5ce4e0759164cb3e" +dependencies = [ + "clap", + "ctrlc", + "parking_lot", + "rayon", + "thiserror 1.0.69", + "windows 0.58.0", +] + [[package]] name = "windows-collections" version = "0.2.0" @@ -17674,6 +17804,16 @@ dependencies = [ "tap", ] +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x11-clipboard" version = "0.9.3" @@ -17712,6 +17852,18 @@ dependencies = [ "libc", ] +[[package]] +name = "xcb" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e2f212bb1a92cd8caac8051b829a6582ede155ccb60b5d5908b81b100952be" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml 0.30.0", + "x11", +] + [[package]] name = "xcursor" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index bd4b9bbd2f56c8b1cf72a8e124d55d60d895d421..6693d57b4f4ecfc5ed6f3861a19343b57d8208b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,8 +400,12 @@ async-tungstenite = "0.28" async-watch = "0.3.1" async_zip = { version = "0.0.17", features = ["deflate", "deflate64"] } aws-config = { version = "1.5.16", features = ["behavior-version-latest"] } -aws-credential-types = { version = "1.2.1", features = ["hardcoded-credentials"] } -aws-sdk-bedrockruntime = { version = "1.73.0", features = ["behavior-version-latest"] } +aws-credential-types = { version = "1.2.1", features = [ + "hardcoded-credentials", +] } +aws-sdk-bedrockruntime = { version = "1.73.0", features = [ + "behavior-version-latest", +] } aws-smithy-runtime-api = { version = "1.7.3", features = ["http-1x", "client"] } aws-smithy-types = { version = "1.2.13", features = ["http-body-1-x"] } base64 = "0.22" @@ -508,6 +512,7 @@ rust-embed = { version = "8.4", features = ["include-exclude"] } rustc-hash = "2.1.0" rustls = { version = "0.23.22" } rustls-platform-verifier = "0.5.0" +scap = { git = "https://github.com/zed-industries/scap", rev = "5715067104794aa356977c543e2f3e95c6183044", default-features = false } schemars = { version = "0.8", features = ["impl_json_schema", "indexmap2"] } semver = "1.0" serde = { version = "1.0", features = ["derive", "rc"] } @@ -547,7 +552,7 @@ time = { version = "0.3", features = [ tiny_http = "0.8" toml = "0.8" tokio = { version = "1" } -tokio-tungstenite = { version = "0.26", features = ["__rustls-tls"]} +tokio-tungstenite = { version = "0.26", features = ["__rustls-tls"] } tower-http = "0.4.4" tree-sitter = { version = "0.25.3", features = ["wasm"] } tree-sitter-bash = "0.23" diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 2a7218f5845eea5d812488db0af50b017500609d..78a460a1f61d887de8b9ba4ab170951481eb88c4 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -49,6 +49,7 @@ wayland = [ "filedescriptor", "xkbcommon", "open", + "scap", ] x11 = [ "blade-graphics", @@ -65,6 +66,7 @@ x11 = [ "x11-clipboard", "filedescriptor", "open", + "scap" ] @@ -99,7 +101,11 @@ profiling.workspace = true rand = { optional = true, workspace = true } raw-window-handle = "0.6" refineable.workspace = true -resvg = { version = "0.45.0", default-features = false, features = ["text", "system-fonts", "memmap-fonts"] } +resvg = { version = "0.45.0", default-features = false, features = [ + "text", + "system-fonts", + "memmap-fonts", +] } usvg = { version = "0.45.0", default-features = false } schemars.workspace = true seahash = "4.1" @@ -159,6 +165,7 @@ cosmic-text = { version = "0.13.2", optional = true } font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "5474cfad4b719a72ec8ed2cb7327b2b01fd10568", features = [ "source-fontconfig-dlopen", ], optional = true } +scap = { workspace = true, optional = true } calloop = { version = "0.13.0" } filedescriptor = { version = "0.8.2", optional = true } @@ -193,7 +200,10 @@ x11rb = { version = "0.13.1", features = [ "resource_manager", "sync", ], optional = true } -xkbcommon = { version = "0.8.0", features = ["wayland", "x11"], optional = true } +xkbcommon = { version = "0.8.0", features = [ + "wayland", + "x11", +], optional = true } xim = { git = "https://github.com/XDeme1/xim-rs", rev = "d50d461764c2213655cd9cf65a0ea94c70d3c4fd", features = [ "x11rb-xcb", "x11rb-client", diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 95d16c0dfecbc9a1e03518522a9ed8f3499b6a7f..8d0b186a52f8882d5441270fad430b1dc2d97ec9 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -650,6 +650,11 @@ impl App { self.platform.primary_display() } + /// Returns whether `screen_capture_sources` may work. + pub fn is_screen_capture_supported(&self) -> bool { + self.platform.is_screen_capture_supported() + } + /// Returns a list of available screen capture sources. pub fn screen_capture_sources( &self, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index fdd4974be36e5b5b6584d04048fbc9859488a3f1..c118aa6249fe7f30ecf628e76d84325cde555a59 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -26,6 +26,12 @@ mod test; #[cfg(target_os = "windows")] mod windows; +#[cfg(all( + any(target_os = "linux", target_os = "freebsd"), + any(feature = "wayland", feature = "x11"), +))] +pub(crate) mod scap_screen_capture; + use crate::{ Action, AnyWindowHandle, App, AsyncWindowContext, BackgroundExecutor, Bounds, DEFAULT_WINDOW_SIZE, DevicePixels, DispatchEventResult, Font, FontId, FontMetrics, FontRun, @@ -158,6 +164,7 @@ pub(crate) trait Platform: 'static { None } + fn is_screen_capture_supported(&self) -> bool; fn screen_capture_sources( &self, ) -> oneshot::Receiver>>>; @@ -246,13 +253,14 @@ pub trait PlatformDisplay: Send + Sync + Debug { /// A source of on-screen video content that can be captured. pub trait ScreenCaptureSource { /// Returns the video resolution of this source. - fn resolution(&self) -> Result>; + fn resolution(&self) -> Result>; /// Start capture video from this source, invoking the given callback /// with each frame. fn stream( &self, - frame_callback: Box, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, ) -> oneshot::Receiver>>; } diff --git a/crates/gpui/src/platform/linux.rs b/crates/gpui/src/platform/linux.rs index 089b52cf1e7303aeef3ca1330a3c33b2b7150f7a..20bc2ddca9e363a5f373318b047c7ad97620290c 100644 --- a/crates/gpui/src/platform/linux.rs +++ b/crates/gpui/src/platform/linux.rs @@ -21,4 +21,7 @@ pub(crate) use wayland::*; #[cfg(feature = "x11")] pub(crate) use x11::*; +#[cfg(any(feature = "wayland", feature = "x11"))] +pub(crate) type PlatformScreenCaptureFrame = scap::frame::Frame; +#[cfg(not(any(feature = "wayland", feature = "x11")))] pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/linux/headless/client.rs b/crates/gpui/src/platform/linux/headless/client.rs index 71fdc26d9e0a9a30fa81af2310cfe8e6526ed3d9..46a3032fe093eb47df79fa71c2caf97fd24b236b 100644 --- a/crates/gpui/src/platform/linux/headless/client.rs +++ b/crates/gpui/src/platform/linux/headless/client.rs @@ -1,13 +1,16 @@ use std::cell::RefCell; use std::rc::Rc; +use anyhow::anyhow; use calloop::{EventLoop, LoopHandle}; - +use futures::channel::oneshot; use util::ResultExt; use crate::platform::linux::LinuxClient; use crate::platform::{LinuxCommon, PlatformWindow}; -use crate::{AnyWindowHandle, CursorStyle, DisplayId, PlatformDisplay, WindowParams}; +use crate::{ + AnyWindowHandle, CursorStyle, DisplayId, PlatformDisplay, ScreenCaptureSource, WindowParams, +}; pub struct HeadlessClientState { pub(crate) _loop_handle: LoopHandle<'static, HeadlessClient>, @@ -63,6 +66,21 @@ impl LinuxClient for HeadlessClient { None } + fn is_screen_capture_supported(&self) -> bool { + false + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!( + "Headless mode does not support screen capture." + ))) + .ok(); + rx + } + fn active_window(&self) -> Option { None } diff --git a/crates/gpui/src/platform/linux/platform.rs b/crates/gpui/src/platform/linux/platform.rs index cbf5861df291fe15c02c596926d4d88b1f76f35f..d02eea6dac4a7403e99ac78570f760a9a7a0b6af 100644 --- a/crates/gpui/src/platform/linux/platform.rs +++ b/crates/gpui/src/platform/linux/platform.rs @@ -28,6 +28,7 @@ use crate::{ Pixels, Platform, PlatformDisplay, PlatformTextSystem, PlatformWindow, Point, Result, ScreenCaptureSource, Task, WindowAppearance, WindowParams, px, }; + #[cfg(any(feature = "wayland", feature = "x11"))] pub(crate) const SCROLL_LINES: f32 = 3.0; @@ -50,6 +51,10 @@ pub trait LinuxClient { #[allow(unused)] fn display(&self, id: DisplayId) -> Option>; fn primary_display(&self) -> Option>; + fn is_screen_capture_supported(&self) -> bool; + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>>; fn open_window( &self, @@ -230,12 +235,14 @@ impl Platform for P { self.displays() } + fn is_screen_capture_supported(&self) -> bool { + self.is_screen_capture_supported() + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { - let (mut tx, rx) = oneshot::channel(); - tx.send(Err(anyhow!("screen capture not implemented"))).ok(); - rx + self.screen_capture_sources() } fn active_window(&self) -> Option { diff --git a/crates/gpui/src/platform/linux/wayland/client.rs b/crates/gpui/src/platform/linux/wayland/client.rs index 6906dc123e212208b58ab0e04a4c39b945a28b54..0255dd8776c6ca19599f4500a87308c177b7eba7 100644 --- a/crates/gpui/src/platform/linux/wayland/client.rs +++ b/crates/gpui/src/platform/linux/wayland/client.rs @@ -7,6 +7,7 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::anyhow; use calloop::{ EventLoop, LoopHandle, timer::{TimeoutAction, Timer}, @@ -14,7 +15,7 @@ use calloop::{ use calloop_wayland_source::WaylandSource; use collections::HashMap; use filedescriptor::Pipe; - +use futures::channel::oneshot; use http_client::Url; use smallvec::SmallVec; use util::ResultExt; @@ -85,7 +86,8 @@ use crate::{ FileDropEvent, ForegroundExecutor, KeyDownEvent, KeyUpEvent, Keystroke, LinuxCommon, Modifiers, ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseExitEvent, MouseMoveEvent, MouseUpEvent, NavigationDirection, Pixels, PlatformDisplay, PlatformInput, Point, SCROLL_LINES, - ScaledPixels, ScrollDelta, ScrollWheelEvent, Size, TouchPhase, WindowParams, point, px, size, + ScaledPixels, ScreenCaptureSource, ScrollDelta, ScrollWheelEvent, Size, TouchPhase, + WindowParams, point, px, size, }; /// Used to convert evdev scancode to xkb scancode @@ -633,6 +635,24 @@ impl LinuxClient for WaylandClient { None } + fn is_screen_capture_supported(&self) -> bool { + false + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + // TODO: Get screen capture working on wayland. Be sure to try window resizing as that may + // be tricky. + // + // start_scap_default_target_source() + let (sources_tx, sources_rx) = oneshot::channel(); + sources_tx + .send(Err(anyhow!("Wayland screen capture not yet implemented."))) + .ok(); + sources_rx + } + fn open_window( &self, handle: AnyWindowHandle, diff --git a/crates/gpui/src/platform/linux/x11/client.rs b/crates/gpui/src/platform/linux/x11/client.rs index 45c37221abcc708f811aa9ffc7821983753d9a23..3e375aac073a999b352dbfff5119969c6dd8e4e9 100644 --- a/crates/gpui/src/platform/linux/x11/client.rs +++ b/crates/gpui/src/platform/linux/x11/client.rs @@ -1,3 +1,4 @@ +use crate::platform::scap_screen_capture::scap_screen_sources; use core::str; use std::{ cell::RefCell, @@ -8,13 +9,13 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::Context as _; use calloop::{ EventLoop, LoopHandle, RegistrationToken, generic::{FdWrapper, Generic}, }; - -use anyhow::Context as _; use collections::HashMap; +use futures::channel::oneshot; use http_client::Url; use smallvec::SmallVec; use util::ResultExt; @@ -59,8 +60,8 @@ use crate::platform::{ use crate::{ AnyWindowHandle, Bounds, ClipboardItem, CursorStyle, DisplayId, FileDropEvent, Keystroke, Modifiers, ModifiersChangedEvent, MouseButton, Pixels, Platform, PlatformDisplay, - PlatformInput, Point, RequestFrameOptions, ScaledPixels, ScrollDelta, Size, TouchPhase, - WindowParams, X11Window, modifiers_from_xinput_info, point, px, + PlatformInput, Point, RequestFrameOptions, ScaledPixels, ScreenCaptureSource, ScrollDelta, + Size, TouchPhase, WindowParams, X11Window, modifiers_from_xinput_info, point, px, }; /// Value for DeviceId parameters which selects all devices. @@ -1327,6 +1328,16 @@ impl LinuxClient for X11Client { )) } + fn is_screen_capture_supported(&self) -> bool { + true + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + scap_screen_sources(&self.0.borrow().common.foreground_executor) + } + fn open_window( &self, handle: AnyWindowHandle, diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index c830522cbe2cd87354ad66504478a3fac437234d..0bda71369e48eea139930094f6ed21bbd87d9b84 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -552,6 +552,10 @@ impl Platform for MacPlatform { .collect() } + fn is_screen_capture_supported(&self) -> bool { + true + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/gpui/src/platform/mac/screen_capture.rs b/crates/gpui/src/platform/mac/screen_capture.rs index 900e9d2181dac554c9ecd9920616e562fa6236b6..8e9fc3d3f9f7e14a750544ada7febb8674d2cb78 100644 --- a/crates/gpui/src/platform/mac/screen_capture.rs +++ b/crates/gpui/src/platform/mac/screen_capture.rs @@ -1,7 +1,7 @@ use crate::{ - Pixels, Size, + DevicePixels, ForegroundExecutor, Size, platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream}, - px, size, + size, }; use anyhow::{Result, anyhow}; use block::ConcreteBlock; @@ -48,7 +48,7 @@ const FRAME_CALLBACK_IVAR: &str = "frame_callback"; const SCStreamOutputTypeScreen: NSInteger = 0; impl ScreenCaptureSource for MacScreenCaptureSource { - fn resolution(&self) -> Result> { + fn resolution(&self) -> Result> { unsafe { let display_id: CGDirectDisplayID = msg_send![self.sc_display, displayID]; let display_mode_ref = CGDisplayCopyDisplayMode(display_id); @@ -56,13 +56,17 @@ impl ScreenCaptureSource for MacScreenCaptureSource { let height = CGDisplayModeGetPixelHeight(display_mode_ref); CGDisplayModeRelease(display_mode_ref); - Ok(size(px(width as f32), px(height as f32))) + Ok(size( + DevicePixels(width as i32), + DevicePixels(height as i32), + )) } } fn stream( &self, - frame_callback: Box, + _foreground_executor: &ForegroundExecutor, + frame_callback: Box, ) -> oneshot::Receiver>> { unsafe { let stream: id = msg_send![class!(SCStream), alloc]; diff --git a/crates/gpui/src/platform/scap_screen_capture.rs b/crates/gpui/src/platform/scap_screen_capture.rs new file mode 100644 index 0000000000000000000000000000000000000000..c5e2267a37c794aeab70bc06d88d849b64be1c6f --- /dev/null +++ b/crates/gpui/src/platform/scap_screen_capture.rs @@ -0,0 +1,282 @@ +//! Screen capture for Linux and Windows +use crate::{ + DevicePixels, ForegroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, + Size, size, +}; +use anyhow::{Context as _, Result, anyhow}; +use futures::channel::oneshot; +use std::sync::Arc; +use std::sync::atomic::{self, AtomicBool}; + +/// Populates the receiver with the screens that can be captured. +/// +/// `scap_default_target_source` should be used instead on Wayland, since `scap_screen_sources` +/// won't return any results. +#[allow(dead_code)] +pub(crate) fn scap_screen_sources( + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (sources_tx, sources_rx) = oneshot::channel(); + get_screen_targets(sources_tx); + to_dyn_screen_capture_sources(sources_rx, foreground_executor) +} + +/// Starts screen capture for the default target, and populates the receiver with a single source +/// for it. The first frame of the screen capture is used to determine the size of the stream. +/// +/// On Wayland (Linux), prompts the user to select a target, and populates the receiver with a +/// single screen capture source for their selection. +#[allow(dead_code)] +pub(crate) fn start_scap_default_target_source( + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (sources_tx, sources_rx) = oneshot::channel(); + start_default_target_screen_capture(sources_tx); + to_dyn_screen_capture_sources(sources_rx, foreground_executor) +} + +struct ScapCaptureSource { + target: scap::Target, + size: Size, +} + +/// Populates the sender with the screens available for capture. +fn get_screen_targets(sources_tx: oneshot::Sender>>) { + // Due to use of blocking APIs, a new thread is used. + std::thread::spawn(|| { + let targets = match scap::get_all_targets() { + Ok(targets) => targets, + Err(err) => { + sources_tx.send(Err(err)).ok(); + return; + } + }; + let sources = targets + .iter() + .filter_map(|target| match target { + scap::Target::Display(display) => { + let size = Size { + width: DevicePixels(display.width as i32), + height: DevicePixels(display.height as i32), + }; + Some(ScapCaptureSource { + target: target.clone(), + size, + }) + } + scap::Target::Window(_) => None, + }) + .collect::>(); + sources_tx.send(Ok(sources)).ok(); + }); +} + +impl ScreenCaptureSource for ScapCaptureSource { + fn resolution(&self) -> Result> { + Ok(self.size) + } + + fn stream( + &self, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, + ) -> oneshot::Receiver>> { + let (stream_tx, stream_rx) = oneshot::channel(); + let target = self.target.clone(); + + // Due to use of blocking APIs, a dedicated thread is used. + std::thread::spawn(move || match new_scap_capturer(Some(target)) { + Ok(mut capturer) => { + capturer.start_capture(); + run_capture(capturer, frame_callback, stream_tx); + } + Err(e) => { + stream_tx.send(Err(e)).ok(); + } + }); + + to_dyn_screen_capture_stream(stream_rx, foreground_executor) + } +} + +struct ScapDefaultTargetCaptureSource { + // Sender populated by single call to `ScreenCaptureSource::stream`. + stream_call_tx: std::sync::mpsc::SyncSender<( + // Provides the result of `ScreenCaptureSource::stream`. + oneshot::Sender>, + // Callback for frames. + Box, + )>, + size: Size, +} + +/// Starts screen capture on the default capture target, and populates the sender with the source. +fn start_default_target_screen_capture( + sources_tx: oneshot::Sender>>, +) { + // Due to use of blocking APIs, a dedicated thread is used. + std::thread::spawn(|| { + let start_result = util::maybe!({ + let mut capturer = new_scap_capturer(None)?; + capturer.start_capture(); + let first_frame = capturer + .get_next_frame() + .context("Failed to get first frame of screenshare to get the size.")?; + let size = frame_size(&first_frame); + Ok((capturer, size)) + }); + + match start_result { + Err(e) => { + sources_tx.send(Err(e)).ok(); + } + Ok((capturer, size)) => { + let (stream_call_tx, stream_rx) = std::sync::mpsc::sync_channel(1); + sources_tx + .send(Ok(vec![ScapDefaultTargetCaptureSource { + stream_call_tx, + size, + }])) + .ok(); + let Ok((stream_tx, frame_callback)) = stream_rx.recv() else { + return; + }; + run_capture(capturer, frame_callback, stream_tx); + } + } + }); +} + +impl ScreenCaptureSource for ScapDefaultTargetCaptureSource { + fn resolution(&self) -> Result> { + Ok(self.size) + } + + fn stream( + &self, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, + ) -> oneshot::Receiver>> { + let (tx, rx) = oneshot::channel(); + match self.stream_call_tx.try_send((tx, frame_callback)) { + Ok(()) => {} + Err(std::sync::mpsc::TrySendError::Full((tx, _))) + | Err(std::sync::mpsc::TrySendError::Disconnected((tx, _))) => { + // Note: support could be added for being called again after end of prior stream. + tx.send(Err(anyhow!( + "Can't call ScapDefaultTargetCaptureSource::stream multiple times." + ))) + .ok(); + } + } + to_dyn_screen_capture_stream(rx, foreground_executor) + } +} + +fn new_scap_capturer(target: Option) -> Result { + scap::capturer::Capturer::build(scap::capturer::Options { + fps: 60, + show_cursor: true, + show_highlight: true, + // Note that the actual frame output type may differ. + output_type: scap::frame::FrameType::YUVFrame, + output_resolution: scap::capturer::Resolution::Captured, + crop_area: None, + target, + excluded_targets: None, + }) +} + +fn run_capture( + mut capturer: scap::capturer::Capturer, + frame_callback: Box, + stream_tx: oneshot::Sender>, +) { + let cancel_stream = Arc::new(AtomicBool::new(false)); + let stream_send_result = stream_tx.send(Ok(ScapStream { + cancel_stream: cancel_stream.clone(), + })); + if let Err(_) = stream_send_result { + return; + } + while !cancel_stream.load(std::sync::atomic::Ordering::SeqCst) { + match capturer.get_next_frame() { + Ok(frame) => frame_callback(ScreenCaptureFrame(frame)), + Err(err) => { + log::error!("Halting screen capture due to error: {err}"); + break; + } + } + } + capturer.stop_capture(); +} + +struct ScapStream { + cancel_stream: Arc, +} + +impl ScreenCaptureStream for ScapStream {} + +impl Drop for ScapStream { + fn drop(&mut self) { + self.cancel_stream.store(true, atomic::Ordering::SeqCst); + } +} + +fn frame_size(frame: &scap::frame::Frame) -> Size { + let (width, height) = match frame { + scap::frame::Frame::YUVFrame(frame) => (frame.width, frame.height), + scap::frame::Frame::RGB(frame) => (frame.width, frame.height), + scap::frame::Frame::RGBx(frame) => (frame.width, frame.height), + scap::frame::Frame::XBGR(frame) => (frame.width, frame.height), + scap::frame::Frame::BGRx(frame) => (frame.width, frame.height), + scap::frame::Frame::BGR0(frame) => (frame.width, frame.height), + scap::frame::Frame::BGRA(frame) => (frame.width, frame.height), + }; + size(DevicePixels(width), DevicePixels(height)) +} + +/// This is used by `get_screen_targets` and `start_default_target_screen_capture` to turn their +/// results into `Box`. They need to `Send` their capture source, and so +/// the capture source structs are used as `Box` is not `Send`. +fn to_dyn_screen_capture_sources( + sources_rx: oneshot::Receiver>>, + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (dyn_sources_tx, dyn_sources_rx) = oneshot::channel(); + foreground_executor + .spawn(async move { + match sources_rx.await { + Ok(Ok(results)) => dyn_sources_tx + .send(Ok(results + .into_iter() + .map(|source| Box::new(source) as Box) + .collect::>())) + .ok(), + Ok(Err(err)) => dyn_sources_tx.send(Err(err)).ok(), + Err(oneshot::Canceled) => None, + } + }) + .detach(); + dyn_sources_rx +} + +/// Same motivation as `to_dyn_screen_capture_sources` above. +fn to_dyn_screen_capture_stream( + sources_rx: oneshot::Receiver>, + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>> { + let (dyn_sources_tx, dyn_sources_rx) = oneshot::channel(); + foreground_executor + .spawn(async move { + match sources_rx.await { + Ok(Ok(stream)) => dyn_sources_tx + .send(Ok(Box::new(stream) as Box)) + .ok(), + Ok(Err(err)) => dyn_sources_tx.send(Err(err)).ok(), + Err(oneshot::Canceled) => None, + } + }) + .detach(); + dyn_sources_rx +} diff --git a/crates/gpui/src/platform/test/platform.rs b/crates/gpui/src/platform/test/platform.rs index e66465d2d2af90b8173ccc9943a76466dbec5d6d..90e3cf2fa6e0693cdaece70a41cbc4c50f46a7cf 100644 --- a/crates/gpui/src/platform/test/platform.rs +++ b/crates/gpui/src/platform/test/platform.rs @@ -1,7 +1,8 @@ use crate::{ - AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap, - Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource, - ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams, px, size, + AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DevicePixels, + ForegroundExecutor, Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, + ScreenCaptureSource, ScreenCaptureStream, Size, Task, TestDisplay, TestWindow, + WindowAppearance, WindowParams, size, }; use anyhow::Result; use collections::VecDeque; @@ -46,13 +47,14 @@ pub struct TestScreenCaptureSource {} pub struct TestScreenCaptureStream {} impl ScreenCaptureSource for TestScreenCaptureSource { - fn resolution(&self) -> Result> { - Ok(size(px(1.), px(1.))) + fn resolution(&self) -> Result> { + Ok(size(DevicePixels(1), DevicePixels(1))) } fn stream( &self, - _frame_callback: Box, + _foreground_executor: &ForegroundExecutor, + _frame_callback: Box, ) -> oneshot::Receiver>> { let (mut tx, rx) = oneshot::channel(); let stream = TestScreenCaptureStream {}; @@ -271,6 +273,10 @@ impl Platform for TestPlatform { Some(self.active_display.clone()) } + fn is_screen_capture_supported(&self) -> bool { + true + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/gpui/src/platform/windows/platform.rs b/crates/gpui/src/platform/windows/platform.rs index e991bc93c63ad5bd3bbaa51bc1df249f94122cc9..116b2253d105453b6f56baabf3523288d5026189 100644 --- a/crates/gpui/src/platform/windows/platform.rs +++ b/crates/gpui/src/platform/windows/platform.rs @@ -396,6 +396,10 @@ impl Platform for WindowsPlatform { WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc) } + fn is_screen_capture_supported(&self) -> bool { + false + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/livekit_client/Cargo.toml b/crates/livekit_client/Cargo.toml index dbd685ca034f10c7c3e79ad0fdf7e8d9f5fd480d..7c9e07cd5c8b3583b9c185922800d7ba7750ce71 100644 --- a/crates/livekit_client/Cargo.toml +++ b/crates/livekit_client/Cargo.toml @@ -25,7 +25,7 @@ async-trait.workspace = true collections.workspace = true cpal = "0.15" futures.workspace = true -gpui.workspace = true +gpui = { workspace = true, features = ["x11", "wayland"] } gpui_tokio.workspace = true http_client_tls.workspace = true image.workspace = true @@ -41,7 +41,12 @@ workspace-hack.workspace = true [target.'cfg(not(all(target_os = "windows", target_env = "gnu")))'.dependencies] libwebrtc = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks" } -livekit = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks", features = ["__rustls-tls"] } +livekit = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks", features = [ + "__rustls-tls" +] } + +[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] +scap.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true diff --git a/crates/livekit_client/src/livekit_client/playback.rs b/crates/livekit_client/src/livekit_client/playback.rs index f3df5b86ca81c87389c407a1b0077c113a0fad13..18144e694841de912b1dadd3516d57d34679d009 100644 --- a/crates/livekit_client/src/livekit_client/playback.rs +++ b/crates/livekit_client/src/livekit_client/playback.rs @@ -336,7 +336,7 @@ pub(crate) async fn capture_local_video_track( .await?; let capture_stream = capture_source - .stream({ + .stream(cx.foreground_executor(), { let track_source = track_source.clone(); Box::new(move |frame| { if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { @@ -620,7 +620,49 @@ fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option Option> { + use libwebrtc::native::yuv_helper::argb_to_nv12; + use livekit::webrtc::prelude::NV12Buffer; + match frame.0 { + scap::frame::Frame::BGRx(frame) => { + let mut buffer = NV12Buffer::new(frame.width as u32, frame.height as u32); + let (stride_y, stride_uv) = buffer.strides(); + let (data_y, data_uv) = buffer.data_mut(); + argb_to_nv12( + &frame.data, + frame.width as u32 * 4, + data_y, + stride_y, + data_uv, + stride_uv, + frame.width, + frame.height, + ); + Some(buffer) + } + scap::frame::Frame::YUVFrame(yuvframe) => { + let mut buffer = NV12Buffer::with_strides( + yuvframe.width as u32, + yuvframe.height as u32, + yuvframe.luminance_stride as u32, + yuvframe.chrominance_stride as u32, + ); + let (luminance, chrominance) = buffer.data_mut(); + luminance.copy_from_slice(yuvframe.luminance_bytes.as_slice()); + chrominance.copy_from_slice(yuvframe.chrominance_bytes.as_slice()); + Some(buffer) + } + _ => { + log::error!( + "Expected BGRx or YUV frame from scap screen capture but got some other format." + ); + None + } + } +} + +#[cfg(target_os = "windows")] fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { None as Option> } diff --git a/crates/title_bar/src/collab.rs b/crates/title_bar/src/collab.rs index 20dec0e6ea980f479a6bd8a62d2087592b578205..4a096ed034de0813dd87491ce78505b25bfa4f40 100644 --- a/crates/title_bar/src/collab.rs +++ b/crates/title_bar/src/collab.rs @@ -299,10 +299,7 @@ impl TitleBar { let is_screen_sharing = room.is_screen_sharing(); let can_use_microphone = room.can_use_microphone(); let can_share_projects = room.can_share_projects(); - let screen_sharing_supported = match self.platform_style { - PlatformStyle::Mac => true, - PlatformStyle::Linux | PlatformStyle::Windows => false, - }; + let screen_sharing_supported = cx.is_screen_capture_supported(); let mut children = Vec::new(); diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 813e9ce005037cb3ab17c33a1de1a30820edf4c2..397c3764bccd092f348093960995669df09b69f6 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -4423,18 +4423,6 @@ impl Workspace { None } - #[cfg(target_os = "windows")] - fn shared_screen_for_peer( - &self, - _peer_id: PeerId, - _pane: &Entity, - _window: &mut Window, - _cx: &mut App, - ) -> Option> { - None - } - - #[cfg(not(target_os = "windows"))] fn shared_screen_for_peer( &self, peer_id: PeerId, diff --git a/script/linux b/script/linux index 943c9d61b4677b58f2fb36f59082b73180ada54d..79e03df841114a7bf5660b555f61d3a65cb4b42b 100755 --- a/script/linux +++ b/script/linux @@ -28,6 +28,7 @@ if [[ -n $apt ]]; then libasound2-dev libfontconfig-dev libwayland-dev + libx11-xcb-dev libxkbcommon-x11-dev libssl-dev libzstd-dev @@ -76,6 +77,7 @@ if [[ -n $dnf ]] || [[ -n $yum ]]; then alsa-lib-devel fontconfig-devel wayland-devel + libxcb-devel libxkbcommon-x11-devel openssl-devel libzstd-devel @@ -144,6 +146,7 @@ if [[ -n $zyp ]]; then gzip jq libvulkan1 + libxcb-devel libxkbcommon-devel libxkbcommon-x11-devel libzstd-devel @@ -174,6 +177,7 @@ if [[ -n $pacman ]]; then fontconfig wayland libgit2 + libxcb libxkbcommon-x11 openbsd-netcat openssl