livekit_client: Do not encode video frames on the main thread (#46986)

Lukas Wirth created 1 week ago

This can block the mainthread for several milliseconds given large
enough screensizes

Release Notes:

- Fixed screensharing performance on windows and linux

Change summary

crates/livekit_client/src/lib.rs                     | 74 +++++++------
crates/livekit_client/src/livekit_client/playback.rs | 11 +
crates/livekit_client/src/mock_client.rs             |  1 
crates/livekit_client/src/remote_video_track_view.rs |  2 
4 files changed, 51 insertions(+), 37 deletions(-)

Detailed changes

crates/livekit_client/src/lib.rs

@@ -9,45 +9,55 @@ use rodio::DeviceTrait as _;
 mod record;
 pub use record::CaptureInput;
 
-#[cfg(not(any(
-    test,
-    feature = "test-support",
-    all(target_os = "windows", target_env = "gnu"),
-    target_os = "freebsd"
-)))]
+#[cfg(any(
+    rust_analyzer,
+    not(any(
+        test,
+        feature = "test-support",
+        all(target_os = "windows", target_env = "gnu"),
+        target_os = "freebsd"
+    ))
+))]
 mod livekit_client;
-#[cfg(not(any(
-    test,
-    feature = "test-support",
-    all(target_os = "windows", target_env = "gnu"),
-    target_os = "freebsd"
-)))]
+#[cfg(any(
+    rust_analyzer,
+    not(any(
+        test,
+        feature = "test-support",
+        all(target_os = "windows", target_env = "gnu"),
+        target_os = "freebsd"
+    ))
+))]
 pub use livekit_client::*;
 
-// If you need proper LSP in livekit_client you've got to comment
-// - the cfg blocks above
-// - the mods: mock_client & test and their conditional blocks
-// - the pub use mock_client::* and their conditional blocks
-
-#[cfg(any(
-    test,
-    feature = "test-support",
-    all(target_os = "windows", target_env = "gnu"),
-    target_os = "freebsd"
+#[cfg(all(
+    not(rust_analyzer),
+    any(
+        test,
+        feature = "test-support",
+        all(target_os = "windows", target_env = "gnu"),
+        target_os = "freebsd"
+    )
 ))]
 mod mock_client;
-#[cfg(any(
-    test,
-    feature = "test-support",
-    all(target_os = "windows", target_env = "gnu"),
-    target_os = "freebsd"
+#[cfg(all(
+    not(rust_analyzer),
+    any(
+        test,
+        feature = "test-support",
+        all(target_os = "windows", target_env = "gnu"),
+        target_os = "freebsd"
+    )
 ))]
 pub mod test;
-#[cfg(any(
-    test,
-    feature = "test-support",
-    all(target_os = "windows", target_env = "gnu"),
-    target_os = "freebsd"
+#[cfg(all(
+    not(rust_analyzer),
+    any(
+        test,
+        feature = "test-support",
+        all(target_os = "windows", target_env = "gnu"),
+        target_os = "freebsd"
+    )
 ))]
 pub use mock_client::*;
 

crates/livekit_client/src/livekit_client/playback.rs

@@ -551,7 +551,9 @@ impl libwebrtc::native::audio_mixer::AudioMixerSource for AudioMixerSource {
 
 pub fn play_remote_video_track(
     track: &crate::RemoteVideoTrack,
+    executor: &BackgroundExecutor,
 ) -> impl Stream<Item = RemoteVideoFrame> + use<> {
+    let executor = executor.clone();
     #[cfg(target_os = "macos")]
     {
         let mut pool = None;
@@ -563,7 +565,7 @@ pub fn play_remote_video_track(
                 pool = create_buffer_pool(frame.buffer.width(), frame.buffer.height()).log_err();
             }
             let pool = pool.clone();
-            async move {
+            executor.spawn(async move {
                 if frame.buffer.width() < 10 && frame.buffer.height() < 10 {
                     // when the remote stops sharing, we get an 8x8 black image.
                     // In a lil bit, the unpublish will come through and close the view,
@@ -572,13 +574,14 @@ pub fn play_remote_video_track(
                 }
 
                 video_frame_buffer_from_webrtc(pool?, frame.buffer)
-            }
+            })
         })
     }
     #[cfg(not(target_os = "macos"))]
     {
-        NativeVideoStream::new(track.0.rtc_track())
-            .filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) })
+        NativeVideoStream::new(track.0.rtc_track()).filter_map(move |frame| {
+            executor.spawn(async move { video_frame_buffer_from_webrtc(frame.buffer) })
+        })
     }
 }
 

crates/livekit_client/src/mock_client.rs

@@ -33,6 +33,7 @@ impl Into<gpui::SurfaceSource> for RemoteVideoFrame {
 }
 pub(crate) fn play_remote_video_track(
     _track: &crate::RemoteVideoTrack,
+    _: &gpui::BackgroundExecutor,
 ) -> impl futures::Stream<Item = RemoteVideoFrame> + use<> {
     futures::stream::pending()
 }

crates/livekit_client/src/remote_video_track_view.rs

@@ -22,7 +22,7 @@ pub enum RemoteVideoTrackViewEvent {
 impl RemoteVideoTrackView {
     pub fn new(track: RemoteVideoTrack, window: &mut Window, cx: &mut Context<Self>) -> Self {
         cx.focus_handle();
-        let frames = crate::play_remote_video_track(&track);
+        let frames = crate::play_remote_video_track(&track, cx.background_executor());
 
         #[cfg(not(target_os = "macos"))]
         {