From ad0f75f3869f81ec60edb77769d5ca0be8eec85b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 25 Aug 2022 13:48:10 -0600 Subject: [PATCH 01/89] Add capture example to GPUI Added a linker arg to the GPUI build script. Not sure if we'll want to bake this into GPUI or do it via another crate, but this is convenient for exploration for now. --- crates/gpui/Cargo.toml | 6 +++ crates/gpui/build.rs | 3 ++ crates/gpui/examples/capture.rs | 78 +++++++++++++++++++++++++++++++++ crates/gpui/script/capture | 4 ++ 4 files changed, 91 insertions(+) create mode 100644 crates/gpui/examples/capture.rs create mode 100755 crates/gpui/script/capture diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 72c65a4c337ff11fb828f324b5cef5995a6f5e72..255e85a5272c1fae46e4bc81f7e0e169ff065bd1 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -3,6 +3,7 @@ authors = ["Nathan Sobo "] edition = "2021" name = "gpui" version = "0.1.0" +description = "A GPU-accelerated UI framework" [lib] path = "src/gpui.rs" @@ -11,6 +12,11 @@ doctest = false [features] test-support = ["backtrace", "dhat", "env_logger", "collections/test-support"] +[package.metadata.bundle.example.capture] +name = "Capture" +identifier = "rs.gpui.examples.Capture" +description = "An example of screen capture" + [dependencies] collections = { path = "../collections" } gpui_macros = { path = "../gpui_macros" } diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 836d586c26bea742d7b31832d0b17b64d0295a05..1c6cc11de0057bf9d12d40bb4f900e2843e493ee 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -9,6 +9,9 @@ fn main() { compile_context_predicate_parser(); compile_metal_shaders(); generate_shader_bindings(); + + // Support screen capture + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); } fn generate_dispatch_bindings() { diff --git a/crates/gpui/examples/capture.rs b/crates/gpui/examples/capture.rs new file mode 100644 index 0000000000000000000000000000000000000000..27bb843ab983790cfb2e008b730a5be281043526 --- /dev/null +++ b/crates/gpui/examples/capture.rs @@ -0,0 +1,78 @@ +use std::{slice, str}; + +use block::ConcreteBlock; +use cocoa::{ + base::id, + foundation::{NSString, NSUInteger}, +}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use log::LevelFilter; +use objc::{class, msg_send, sel, sel_impl}; +use simplelog::SimpleLogger; + +#[allow(non_upper_case_globals)] +const NSUTF8StringEncoding: NSUInteger = 4; + +actions!(capture, [Quit]); + +fn main() { + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); + + gpui::App::new(()).unwrap().run(|cx| { + cx.platform().activate(true); + cx.add_global_action(quit); + + cx.add_bindings([Binding::new("cmd-q", Quit, None)]); + cx.set_menus(vec![Menu { + name: "Zed", + items: vec![MenuItem::Action { + name: "Quit", + action: Box::new(Quit), + }], + }]); + + unsafe { + let block = ConcreteBlock::new(move |content: id, error: id| { + println!("got response with shareable content"); + dbg!(content); + dbg!(error); + dbg!(string_from_objc(msg_send![error, localizedDescription])); + }); + + let _: id = msg_send![ + class!(SCShareableContent), + getShareableContentWithCompletionHandler: block + ]; + } + + // cx.add_window(Default::default(), |_| ScreenCaptureView); + }); +} + +struct ScreenCaptureView; + +impl gpui::Entity for ScreenCaptureView { + type Event = (); +} + +impl gpui::View for ScreenCaptureView { + fn ui_name() -> &'static str { + "View" + } + + fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { + Empty::new().boxed() + } +} + +pub unsafe fn string_from_objc(string: id) -> String { + let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; + let bytes = string.UTF8String() as *const u8; + str::from_utf8(slice::from_raw_parts(bytes, len)) + .unwrap() + .to_string() +} + +fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { + cx.platform().quit(); +} diff --git a/crates/gpui/script/capture b/crates/gpui/script/capture new file mode 100755 index 0000000000000000000000000000000000000000..753e5108e7f91093686c0b0e481b9b5380dcec4e --- /dev/null +++ b/crates/gpui/script/capture @@ -0,0 +1,4 @@ +#!/bin/bash + +cargo bundle --example capture +open ../../target/debug/examples/bundle/osx/Capture.app From 7918bf39f5f0efc4163a564872a4ae52c2b29434 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 26 Aug 2022 15:10:26 -0600 Subject: [PATCH 02/89] Make capture example its own crate --- Cargo.lock | 15 +++++++++++++ crates/capture/Cargo.toml | 21 +++++++++++++++++++ crates/capture/build.rs | 3 +++ crates/capture/script/capture | 5 +++++ .../capture.rs => capture/src/main.rs} | 10 +++++---- crates/gpui/Cargo.toml | 5 ----- crates/gpui/build.rs | 3 --- crates/gpui/script/capture | 4 ---- 8 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 crates/capture/Cargo.toml create mode 100644 crates/capture/build.rs create mode 100755 crates/capture/script/capture rename crates/{gpui/examples/capture.rs => capture/src/main.rs} (88%) delete mode 100755 crates/gpui/script/capture diff --git a/Cargo.lock b/Cargo.lock index 634ef452a369b142c983302bcb471c00395e1a19..e12e49184d914a49f057d8d8c6ffc039c883b5c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,6 +750,21 @@ dependencies = [ "winx", ] +[[package]] +name = "capture" +version = "0.1.0" +dependencies = [ + "block", + "cocoa", + "core-foundation", + "core-graphics", + "foreign-types", + "gpui", + "log", + "objc", + "simplelog", +] + [[package]] name = "castaway" version = "0.1.2" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4dc975353fc4e12106f0ba284b4fbf5dc6f74e0a --- /dev/null +++ b/crates/capture/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "capture" +version = "0.1.0" +edition = "2021" +description = "An example of screen capture" + +[package.metadata.bundle] +name = "Capture" +identifier = "dev.zed.Capture" + +[dependencies] +gpui = { path = "../gpui" } + +block = "0.1" +cocoa = "0.24" +core-foundation = "0.9.3" +core-graphics = "0.22.3" +foreign-types = "0.3" +log = { version = "0.4.16", features = ["kv_unstable_serde"] } +objc = "0.2" +simplelog = "0.9" diff --git a/crates/capture/build.rs b/crates/capture/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..5aa9032376979b9fbaf773a6b31cff089c964e8c --- /dev/null +++ b/crates/capture/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); +} diff --git a/crates/capture/script/capture b/crates/capture/script/capture new file mode 100755 index 0000000000000000000000000000000000000000..0e6cf1f409ad0f2aa0eae0a6aefc86c2fe203416 --- /dev/null +++ b/crates/capture/script/capture @@ -0,0 +1,5 @@ +#!/bin/bash + +cargo bundle +TTY=`tty` +open ../../target/debug/bundle/osx/Capture.app --stdout $TTY --stderr $TTY diff --git a/crates/gpui/examples/capture.rs b/crates/capture/src/main.rs similarity index 88% rename from crates/gpui/examples/capture.rs rename to crates/capture/src/main.rs index 27bb843ab983790cfb2e008b730a5be281043526..6bad5bb7f3f33f494f7517f6df4d9256375d0144 100644 --- a/crates/gpui/examples/capture.rs +++ b/crates/capture/src/main.rs @@ -33,10 +33,12 @@ fn main() { unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { - println!("got response with shareable content"); - dbg!(content); - dbg!(error); - dbg!(string_from_objc(msg_send![error, localizedDescription])); + println!( + "got response with shareable content {:?} {:?} {:?}", + content, + error, + string_from_objc(msg_send![error, localizedDescription]), + ) }); let _: id = msg_send![ diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 255e85a5272c1fae46e4bc81f7e0e169ff065bd1..08c32135fb5170644033232e20ae9859128e81b2 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -12,11 +12,6 @@ doctest = false [features] test-support = ["backtrace", "dhat", "env_logger", "collections/test-support"] -[package.metadata.bundle.example.capture] -name = "Capture" -identifier = "rs.gpui.examples.Capture" -description = "An example of screen capture" - [dependencies] collections = { path = "../collections" } gpui_macros = { path = "../gpui_macros" } diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 1c6cc11de0057bf9d12d40bb4f900e2843e493ee..836d586c26bea742d7b31832d0b17b64d0295a05 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -9,9 +9,6 @@ fn main() { compile_context_predicate_parser(); compile_metal_shaders(); generate_shader_bindings(); - - // Support screen capture - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); } fn generate_dispatch_bindings() { diff --git a/crates/gpui/script/capture b/crates/gpui/script/capture deleted file mode 100755 index 753e5108e7f91093686c0b0e481b9b5380dcec4e..0000000000000000000000000000000000000000 --- a/crates/gpui/script/capture +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -cargo bundle --example capture -open ../../target/debug/examples/bundle/osx/Capture.app From 45519cdd271b783193d0fe3449894d6ae5fe901b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 29 Aug 2022 18:00:51 +0200 Subject: [PATCH 03/89] WIP --- Cargo.lock | 1 + crates/capture/Cargo.toml | 3 ++ crates/capture/build.rs | 7 +++ crates/capture/src/dummy.m | 12 ++++++ crates/capture/src/main.rs | 75 ++++++++++++++++++++++++++++++--- crates/gpui/src/platform/mac.rs | 2 +- styles/package-lock.json | 1 + 7 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 crates/capture/src/dummy.m diff --git a/Cargo.lock b/Cargo.lock index e12e49184d914a49f057d8d8c6ffc039c883b5c4..f001ab934a803b43c1d30e51dcff626bfde7dcb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -755,6 +755,7 @@ name = "capture" version = "0.1.0" dependencies = [ "block", + "cc", "cocoa", "core-foundation", "core-graphics", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 4dc975353fc4e12106f0ba284b4fbf5dc6f74e0a..2b483a25caa5098b36c0f1a0c3c4d7aaf1edeebd 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -19,3 +19,6 @@ foreign-types = "0.3" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" simplelog = "0.9" + +[build-dependencies] +cc = "1.0" \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 5aa9032376979b9fbaf773a6b31cff089c964e8c..33baf392bbab06de29303ad9a5cb8da67505d722 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,3 +1,10 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=CoreMedia"); + println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); + + cc::Build::new() + .file("src/dummy.m") + .define("MACOSX_DEPLOYMENT_TARGET", "12.3") + .compile("dummy"); } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m new file mode 100644 index 0000000000000000000000000000000000000000..8edb379692a9db369d1095fe1386bd51e226df6d --- /dev/null +++ b/crates/capture/src/dummy.m @@ -0,0 +1,12 @@ +#import + +@interface MyClass : NSObject +@end + +@implementation MyClass +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { +} + +@end \ No newline at end of file diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 6bad5bb7f3f33f494f7517f6df4d9256375d0144..e790d622eec7c98c8c4a70753e66e1d73cf8a2e9 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,13 +1,14 @@ -use std::{slice, str}; +use std::{ffi::CStr, slice, str}; use block::ConcreteBlock; use cocoa::{ - base::id, - foundation::{NSString, NSUInteger}, + base::{id, nil}, + foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use core_graphics::display::CGDirectDisplayID; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::dispatch_get_main_queue}; use log::LevelFilter; -use objc::{class, msg_send, sel, sel_impl}; +use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; #[allow(non_upper_case_globals)] @@ -32,13 +33,57 @@ fn main() { }]); unsafe { + + let block = ConcreteBlock::new(move |content: id, error: id| { println!( "got response with shareable content {:?} {:?} {:?}", content, error, string_from_objc(msg_send![error, localizedDescription]), - ) + ); + + let displays: id = msg_send![content, displays]; + if let Some(display) = (0..displays.count()) + .map(|ix| displays.objectAtIndex(ix)) + .next() + { + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); + let capture_output_class = decl.register(); + + let output: id = msg_send![capture_output_class, alloc]; + let output: id = msg_send![output, init]; + + let excluded_windows: id = msg_send![class!(NSArray), array]; + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; + let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + // Configure the display content width and height. + let _: () = msg_send![config, setWidth: 800]; + let _: () = msg_send![config, setHeight: 600]; + let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; + let _: () = msg_send![config, setQueueDepth: 5]; + + let stream: id = msg_send![class!(SCStream), alloc]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; + let error: id = nil; + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + println!("Added stream output... error? {}", string_from_objc(msg_send![error, localizedDescription])); + + + let start_capture_completion = ConcreteBlock::new(move |error: id| { + println!("Started capturing... error? {}", string_from_objc(msg_send![error, localizedDescription])); + println!("recovery suggestion {}", string_from_objc(msg_send![error, localizedRecoverySuggestion])); + println!("failure reason {}", string_from_objc(msg_send![error, localizedFailureReason])); + + + }); + + assert!(!stream.is_null()); + let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; + } }); let _: id = msg_send![ @@ -75,6 +120,24 @@ pub unsafe fn string_from_objc(string: id) -> String { .to_string() } +extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { + println!("sample_output"); +} + + +extern "C" { + fn CMTimeMake(value: u64, timescale: i32) -> CMTime; +} + +#[repr(C)] +struct CMTime { + value: i64, + timescale: i32, + flags: u32, + epoch: i64, +} + + fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 8cf3f62874aac87bc6d1841fdfc79d7914a66b50..1d2f5fc8aeab255ff9bfb8880c5381ea2b707cc2 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,5 +1,5 @@ mod atlas; -mod dispatcher; +pub mod dispatcher; mod event; mod fonts; mod geometry; diff --git a/styles/package-lock.json b/styles/package-lock.json index 5499f1852cb4330467268dee6436b53589a90e9b..582f1c84968a5c1a25ddac5fd3c21ba907353c6d 100644 --- a/styles/package-lock.json +++ b/styles/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "styles", "version": "1.0.0", "license": "ISC", "dependencies": { From d91f26d0163dcf7d5a8d40ba5709a94960822977 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:15:49 -0600 Subject: [PATCH 04/89] Make SCStreamOutput protocol accessible in Rust --- crates/capture/build.rs | 2 ++ crates/capture/src/main.rs | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 33baf392bbab06de29303ad9a5cb8da67505d722..abbc019943e5f69d6035891103621ed558d9cd56 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -2,9 +2,11 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=CoreMedia"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); + println!("cargo:rustc-link-arg=-ObjC"); cc::Build::new() .file("src/dummy.m") .define("MACOSX_DEPLOYMENT_TARGET", "12.3") + .flag("-ObjC") .compile("dummy"); } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e790d622eec7c98c8c4a70753e66e1d73cf8a2e9..739813472965754d418d9bdd36c776979abb9449 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -33,16 +33,7 @@ fn main() { }]); unsafe { - - let block = ConcreteBlock::new(move |content: id, error: id| { - println!( - "got response with shareable content {:?} {:?} {:?}", - content, - error, - string_from_objc(msg_send![error, localizedDescription]), - ); - let displays: id = msg_send![content, displays]; if let Some(display) = (0..displays.count()) .map(|ix| displays.objectAtIndex(ix)) From 497232ee59376aff79e961975b5fdbfc7c2f60a7 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:32:11 -0600 Subject: [PATCH 05/89] Get capture starting without error --- crates/capture/src/main.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 739813472965754d418d9bdd36c776979abb9449..bfa0d036028776dc3aa7a302701ccd12afe8cb3c 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -34,11 +34,21 @@ fn main() { unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { + if !error.is_null() { + println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } + let displays: id = msg_send![content, displays]; + if let Some(display) = (0..displays.count()) .map(|ix| displays.objectAtIndex(ix)) .next() { + + let display_id: u32 = msg_send![display, displayID]; + println!("display id {:?}", display_id); + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); @@ -51,6 +61,7 @@ fn main() { let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + let config: id = msg_send![config, init]; // Configure the display content width and height. let _: () = msg_send![config, setWidth: 800]; let _: () = msg_send![config, setHeight: 600]; @@ -65,9 +76,10 @@ fn main() { let start_capture_completion = ConcreteBlock::new(move |error: id| { - println!("Started capturing... error? {}", string_from_objc(msg_send![error, localizedDescription])); - println!("recovery suggestion {}", string_from_objc(msg_send![error, localizedRecoverySuggestion])); - println!("failure reason {}", string_from_objc(msg_send![error, localizedFailureReason])); + if !error.is_null() { + println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } }); From 0df97dce028ac440adeb4748172698634a5dca53 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:40:30 -0600 Subject: [PATCH 06/89] WIP --- crates/capture/src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index bfa0d036028776dc3aa7a302701ccd12afe8cb3c..59d87b5225bd9d1683fb7dabe539f0597c27f27a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -72,8 +72,6 @@ fn main() { let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; - println!("Added stream output... error? {}", string_from_objc(msg_send![error, localizedDescription])); - let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -81,7 +79,7 @@ fn main() { return; } - + println!("starting capture"); }); assert!(!stream.is_null()); From 30a3c0fb46e05d544c0f725dbf30dfa3b8bb12a4 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 29 Aug 2022 19:59:58 +0200 Subject: [PATCH 07/89] WIP --- crates/capture/build.rs | 3 +-- crates/capture/src/main.rs | 7 ++++--- crates/gpui/build.rs | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index abbc019943e5f69d6035891103621ed558d9cd56..f45dc922c6f9c829133a85ada4774882a6e616e5 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -6,7 +6,6 @@ fn main() { cc::Build::new() .file("src/dummy.m") - .define("MACOSX_DEPLOYMENT_TARGET", "12.3") - .flag("-ObjC") + .flag("-mmacosx-version-min=12.3") .compile("dummy"); } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 59d87b5225bd9d1683fb7dabe539f0597c27f27a..a77ac61342dd3968dd486e085af53af453868ddb 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,4 +1,4 @@ -use std::{ffi::CStr, slice, str}; +use std::{ffi::CStr, slice, str, ptr}; use block::ConcreteBlock; use cocoa::{ @@ -6,7 +6,7 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; use core_graphics::display::CGDirectDisplayID; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::dispatch_get_main_queue}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::{dispatch_get_main_queue, dispatch_queue_create}}; use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; @@ -71,7 +71,8 @@ fn main() { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: queue error: &error]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 836d586c26bea742d7b31832d0b17b64d0295a05..967423d3f73297ecb9a48b11558c3e364a3173f8 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -19,6 +19,7 @@ fn generate_dispatch_bindings() { .header("src/platform/mac/dispatch.h") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") + .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() From 4440c9b18edada3afbf3a1dd353e8c339cba1cfd Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 16:07:33 -0600 Subject: [PATCH 08/89] Bind to capture's macOS C frameworks with bindgen --- Cargo.lock | 1 + crates/capture/Cargo.toml | 1 + crates/capture/build.rs | 23 ++++++++++++++++++++++- crates/capture/src/bindings.h | 2 ++ crates/capture/src/bindings.rs | 5 +++++ crates/capture/src/main.rs | 34 ++++++++++++++-------------------- 6 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 crates/capture/src/bindings.h create mode 100644 crates/capture/src/bindings.rs diff --git a/Cargo.lock b/Cargo.lock index f001ab934a803b43c1d30e51dcff626bfde7dcb6..7dff1a8bb834c5c5cb30facd056e84bc7b3d8d82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -754,6 +754,7 @@ dependencies = [ name = "capture" version = "0.1.0" dependencies = [ + "bindgen", "block", "cc", "cocoa", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 2b483a25caa5098b36c0f1a0c3c4d7aaf1edeebd..aade52f2fbc9a561bd9856e538e9edb5d80a21f9 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -21,4 +21,5 @@ objc = "0.2" simplelog = "0.9" [build-dependencies] +bindgen = "0.59.2" cc = "1.0" \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index f45dc922c6f9c829133a85ada4774882a6e616e5..4f8ada86706e61478c0da743eed9561ce7902156 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,9 +1,30 @@ +use std::{env, path::PathBuf}; + fn main() { - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=CoreMedia"); + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); println!("cargo:rustc-link-arg=-ObjC"); + let bindings = bindgen::Builder::default() + .header("src/bindings.h") + .clang_arg("-isysroot/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk") + .allowlist_function("CMTimeMake") + .allowlist_type("CMSampleBufferRef") + .allowlist_var("_dispatch_main_q") + .allowlist_function("dispatch_async_f") + .allowlist_function("dispatch_queue_create") + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .layout_tests(false) + .generate() + .expect("unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write dispatch bindings"); + cc::Build::new() .file("src/dummy.m") .flag("-mmacosx-version-min=12.3") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h new file mode 100644 index 0000000000000000000000000000000000000000..0df70f8e111c69a93c6d7475bbae96be1f94eba6 --- /dev/null +++ b/crates/capture/src/bindings.h @@ -0,0 +1,2 @@ +#import +#import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs new file mode 100644 index 0000000000000000000000000000000000000000..24de9f58554929bce09bbf28c7cc2b693d0081aa --- /dev/null +++ b/crates/capture/src/bindings.rs @@ -0,0 +1,5 @@ +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +pub fn dispatch_get_main_queue() -> dispatch_queue_t { + unsafe { &_dispatch_main_q as *const _ as dispatch_queue_t } +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index a77ac61342dd3968dd486e085af53af453868ddb..b41a6230925968976686c21dece66bf3df62625f 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,16 +1,19 @@ -use std::{ffi::CStr, slice, str, ptr}; +mod bindings; + +use std::{slice, str}; use block::ConcreteBlock; use cocoa::{ base::{id, nil}, foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; -use core_graphics::display::CGDirectDisplayID; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::{dispatch_get_main_queue, dispatch_queue_create}}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; +use crate::bindings::dispatch_get_main_queue; + #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -57,6 +60,10 @@ fn main() { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; + dbg!(conforms); + assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); + let excluded_windows: id = msg_send![class!(NSArray), array]; let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; @@ -65,14 +72,15 @@ fn main() { // Configure the display content width and height. let _: () = msg_send![config, setWidth: 800]; let _: () = msg_send![config, setHeight: 600]; - let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; + let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 5]; let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: queue error: &error]; + // let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -126,20 +134,6 @@ extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: println!("sample_output"); } - -extern "C" { - fn CMTimeMake(value: u64, timescale: i32) -> CMTime; -} - -#[repr(C)] -struct CMTime { - value: i64, - timescale: i32, - flags: u32, - epoch: i64, -} - - fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } From 99cb66dfb033275e9ccabd93e8d6764ed8dd5f0e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 16:25:37 -0600 Subject: [PATCH 09/89] Programmatically locate the system SDK path for bindgen --- crates/capture/build.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 4f8ada86706e61478c0da743eed9561ce7902156..d75b110b9b28ae0295c03f4096336d4e319448f2 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,4 +1,4 @@ -use std::{env, path::PathBuf}; +use std::{env, path::PathBuf, process::Command}; fn main() { println!("cargo:rustc-link-lib=framework=CoreMedia"); @@ -7,9 +7,19 @@ fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); println!("cargo:rustc-link-arg=-ObjC"); + let sdk_path = String::from_utf8( + Command::new("xcrun") + .args(&["--sdk", "macosx", "--show-sdk-path"]) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + let sdk_path = sdk_path.trim_end(); + let bindings = bindgen::Builder::default() .header("src/bindings.h") - .clang_arg("-isysroot/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk") + .clang_arg(format!("-isysroot{}", sdk_path)) .allowlist_function("CMTimeMake") .allowlist_type("CMSampleBufferRef") .allowlist_var("_dispatch_main_q") From 82ec2dc7ca8d668f832b076c3617eed1ce21a701 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 19:08:07 -0600 Subject: [PATCH 10/89] Try to use the dummy capture handler, still not working --- crates/capture/build.rs | 3 +++ crates/capture/src/bindings.h | 1 + crates/capture/src/bindings.rs | 4 +++- crates/capture/src/dummy.m | 16 +++++++++++--- crates/capture/src/main.rs | 39 ++++++++++++++++++++++++---------- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index d75b110b9b28ae0295c03f4096336d4e319448f2..6ab5a40e0f8df1a95e648ff96f7a6d51b5af3750 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -20,8 +20,10 @@ fn main() { let bindings = bindgen::Builder::default() .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) + .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") .allowlist_type("CMSampleBufferRef") + .allowlist_type("SCStreamOutputType") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") .allowlist_function("dispatch_queue_create") @@ -35,6 +37,7 @@ fn main() { .write_to_file(out_path.join("bindings.rs")) .expect("couldn't write dispatch bindings"); + println!("cargo:rerun-if-changed=src/dummy.m"); cc::Build::new() .file("src/dummy.m") .flag("-mmacosx-version-min=12.3") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h index 0df70f8e111c69a93c6d7475bbae96be1f94eba6..45ac30446bf515d084c95a9f327ed6b332952a39 100644 --- a/crates/capture/src/bindings.h +++ b/crates/capture/src/bindings.h @@ -1,2 +1,3 @@ #import +#import #import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index 24de9f58554929bce09bbf28c7cc2b693d0081aa..e8459106039f95bfe46d50fa01d89cf941c93652 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,5 +1,7 @@ +use objc::*; + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { &_dispatch_main_q as *const _ as dispatch_queue_t } + unsafe { std::mem::transmute(&_dispatch_main_q) } } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m index 8edb379692a9db369d1095fe1386bd51e226df6d..9e3a343ce5490d1facb7a987566c9fc6a9eb1456 100644 --- a/crates/capture/src/dummy.m +++ b/crates/capture/src/dummy.m @@ -1,12 +1,22 @@ #import -@interface MyClass : NSObject +@interface MyClass : NSObject @end @implementation MyClass + - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - ofType:(SCStreamOutputType)type { + ofType:(SCStreamOutputType)type { + printf("dummy capture handler called"); +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + printf("dummy did stop with error called"); } -@end \ No newline at end of file +int main() { + [[MyClass alloc] init]; +} + +@end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index b41a6230925968976686c21dece66bf3df62625f..70a8ca62b24dce463dc56b0598e04273cf4fd29d 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use std::{slice, str}; +use std::{slice, str, ptr::{self}}; use block::ConcreteBlock; use cocoa::{ @@ -12,7 +12,7 @@ use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; -use crate::bindings::dispatch_get_main_queue; +use crate::bindings::{dispatch_get_main_queue, dispatch_queue_create, NSObject, CMSampleBufferRef}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -52,18 +52,27 @@ fn main() { let display_id: u32 = msg_send![display, displayID]; println!("display id {:?}", display_id); - let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); - decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); - let capture_output_class = decl.register(); + // let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + // decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + // decl.add_protocol(Protocol::get("SCStreamDelegate").unwrap()); + // decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); + // decl.add_method(sel!(stream:didStopWithError:), did_stop_with_error as extern "C" fn(&Object, Sel, id, id)); + // let capture_output_class = decl.register(); - let output: id = msg_send![capture_output_class, alloc]; + // let output: id = msg_send![capture_output_class, alloc]; + // let output: id = msg_send![output, init]; + + let output: id = msg_send![class!(MyClass), alloc]; let output: id = msg_send![output, init]; + // Do we conform to the protocol? let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; dbg!(conforms); assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); + // Confirm we can send the protocol message to the object + let _: () = msg_send![output, stream:NSObject(ptr::null_mut()) didOutputSampleBuffer:NSObject(ptr::null_mut()) ofType:0]; + let excluded_windows: id = msg_send![class!(NSArray), array]; let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; @@ -76,11 +85,15 @@ fn main() { let _: () = msg_send![config, setQueueDepth: 5]; let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - // let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + let _: () = msg_send![stream, + addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen + sampleHandlerQueue: queue + error: &error + ]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -130,8 +143,12 @@ pub unsafe fn string_from_objc(string: id) -> String { .to_string() } +extern "C" fn did_stop_with_error(this: &Object, _: Sel, stream: id, error: id) { + println!("did_stop_with_error"); +} + extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { - println!("sample_output"); + println!("sample output"); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { From ef8a0dc175bf373a862467bd4af8008a0d5b7120 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 09:37:49 +0200 Subject: [PATCH 11/89] Fix bindgen warnings --- crates/capture/src/bindings.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index e8459106039f95bfe46d50fa01d89cf941c93652..afdef6d1dafb6b7572a18774961429fe21b1600a 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,3 +1,7 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); From 014246f56919c22b0eec55a5cd04aaaa317484ca Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 11:12:40 +0200 Subject: [PATCH 12/89] Provide all running applications to `SCContentFilter` to capture display --- crates/capture/build.rs | 4 +- crates/capture/src/bindings.rs | 4 - crates/capture/src/dummy.m | 15 ---- crates/capture/src/main.rs | 153 +++++++++++++++------------------ 4 files changed, 72 insertions(+), 104 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 6ab5a40e0f8df1a95e648ff96f7a6d51b5af3750..51ff36f705a1f969005c1bafc86fd560f446017f 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -17,15 +17,13 @@ fn main() { .unwrap(); let sdk_path = sdk_path.trim_end(); + println!("cargo:rerun-if-changed=src/bindings.h"); let bindings = bindgen::Builder::default() .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") - .allowlist_type("CMSampleBufferRef") .allowlist_type("SCStreamOutputType") - .allowlist_var("_dispatch_main_q") - .allowlist_function("dispatch_async_f") .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index afdef6d1dafb6b7572a18774961429fe21b1600a..a111bede4de059f0c0157775778bcc56ebdd4079 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -5,7 +5,3 @@ use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); - -pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { std::mem::transmute(&_dispatch_main_q) } -} diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m index 9e3a343ce5490d1facb7a987566c9fc6a9eb1456..0ae2bda6a7646d48c3485a1e811ffc2c5ecb8888 100644 --- a/crates/capture/src/dummy.m +++ b/crates/capture/src/dummy.m @@ -4,19 +4,4 @@ @end @implementation MyClass - -- (void)stream:(SCStream *)stream - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - ofType:(SCStreamOutputType)type { - printf("dummy capture handler called"); -} - -- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { - printf("dummy did stop with error called"); -} - -int main() { - [[MyClass alloc] init]; -} - @end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 70a8ca62b24dce463dc56b0598e04273cf4fd29d..c1c017c3599927f9eb6b823865e373b02491b44a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,18 +1,22 @@ mod bindings; -use std::{slice, str, ptr::{self}}; - +use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ - base::{id, nil}, - foundation::{NSArray, NSString, NSUInteger, NSInteger}, + base::{id, nil, YES}, + foundation::{NSArray, NSBundle, NSString, NSUInteger}, }; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; -use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; +use objc::{ + class, + declare::ClassDecl, + msg_send, + runtime::{Object, Protocol, Sel}, + sel, sel_impl, +}; use simplelog::SimpleLogger; - -use crate::bindings::{dispatch_get_main_queue, dispatch_queue_create, NSObject, CMSampleBufferRef}; +use std::{ptr, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -41,72 +45,51 @@ fn main() { println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); return; } - + + let applications: id = msg_send![content, applications]; let displays: id = msg_send![content, displays]; - - if let Some(display) = (0..displays.count()) - .map(|ix| displays.objectAtIndex(ix)) - .next() - { - - let display_id: u32 = msg_send![display, displayID]; - println!("display id {:?}", display_id); - - // let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - // decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); - // decl.add_protocol(Protocol::get("SCStreamDelegate").unwrap()); - // decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); - // decl.add_method(sel!(stream:didStopWithError:), did_stop_with_error as extern "C" fn(&Object, Sel, id, id)); - // let capture_output_class = decl.register(); - - // let output: id = msg_send![capture_output_class, alloc]; - // let output: id = msg_send![output, init]; - - let output: id = msg_send![class!(MyClass), alloc]; - let output: id = msg_send![output, init]; - - // Do we conform to the protocol? - let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; - dbg!(conforms); - assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); - - // Confirm we can send the protocol message to the object - let _: () = msg_send![output, stream:NSObject(ptr::null_mut()) didOutputSampleBuffer:NSObject(ptr::null_mut()) ofType:0]; - - let excluded_windows: id = msg_send![class!(NSArray), array]; - let filter: id = msg_send![class!(SCContentFilter), alloc]; - let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; - let config: id = msg_send![class!(SCStreamConfiguration), alloc]; - let config: id = msg_send![config, init]; - // Configure the display content width and height. - let _: () = msg_send![config, setWidth: 800]; - let _: () = msg_send![config, setHeight: 600]; - let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; - let _: () = msg_send![config, setQueueDepth: 5]; - - let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; - let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); - - let _: () = msg_send![stream, - addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen - sampleHandlerQueue: queue - error: &error - ]; - - let start_capture_completion = ConcreteBlock::new(move |error: id| { - if !error.is_null() { - println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); - return; - } - - println!("starting capture"); - }); - - assert!(!stream.is_null()); - let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; - } + let display: id = displays.objectAtIndex(0); + + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); + let capture_output_class = decl.register(); + + let output: id = msg_send![capture_output_class, alloc]; + let output: id = msg_send![output, init]; + + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; + // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; + let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + let config: id = msg_send![config, init]; + let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; + let _: () = msg_send![config, setQueueDepth: 6]; + let _: () = msg_send![config, setShowsCursor: YES]; + + let stream: id = msg_send![class!(SCStream), alloc]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; + let error: id = nil; + let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); + + let _: () = msg_send![stream, + addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen + sampleHandlerQueue: queue + error: &error + ]; + + let start_capture_completion = ConcreteBlock::new(move |error: id| { + if !error.is_null() { + println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } + + println!("starting capture"); + }); + + assert!(!stream.is_null()); + let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; + }); let _: id = msg_send![ @@ -136,18 +119,24 @@ impl gpui::View for ScreenCaptureView { } pub unsafe fn string_from_objc(string: id) -> String { - let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; - let bytes = string.UTF8String() as *const u8; - str::from_utf8(slice::from_raw_parts(bytes, len)) - .unwrap() - .to_string() -} - -extern "C" fn did_stop_with_error(this: &Object, _: Sel, stream: id, error: id) { - println!("did_stop_with_error"); + if string.is_null() { + Default::default() + } else { + let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; + let bytes = string.UTF8String() as *const u8; + str::from_utf8(slice::from_raw_parts(bytes, len)) + .unwrap() + .to_string() + } } -extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { +extern "C" fn sample_output( + _: &Object, + _: Sel, + _stream: id, + _buffer: id, + _kind: SCStreamOutputType, +) { println!("sample output"); } From 1611635e5fd957809790f25dfd5a0426490d0aaf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 15:49:07 +0200 Subject: [PATCH 13/89] Capture display frames and access underlying IOSurface --- crates/capture/build.rs | 2 + crates/capture/src/main.rs | 156 ++++++++++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 51ff36f705a1f969005c1bafc86fd560f446017f..550d1064ae7bcab0a8219ff1201e0071ccd847c1 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -24,6 +24,8 @@ fn main() { .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") .allowlist_type("SCStreamOutputType") + .allowlist_type("SCFrameStatus") + .allowlist_var("SCStreamFrameInfo.*") .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index c1c017c3599927f9eb6b823865e373b02491b44a..08a6cf03b2aaa9483b700aa96fde6e79a1ca413c 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -4,8 +4,10 @@ use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, - foundation::{NSArray, NSBundle, NSString, NSUInteger}, + foundation::{NSArray, NSString, NSUInteger}, }; +use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; +use core_media::{CMSampleBuffer, CMSampleBufferRef}; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{ @@ -49,6 +51,8 @@ fn main() { let applications: id = msg_send![content, applications]; let displays: id = msg_send![content, displays]; let display: id = displays.objectAtIndex(0); + let display_width: usize = msg_send![display, width]; + let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); @@ -63,6 +67,8 @@ fn main() { // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; let config: id = msg_send![config, init]; + let _: () = msg_send![config, setWidth: display_width]; + let _: () = msg_send![config, setHeight: display_height]; let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; @@ -134,12 +140,156 @@ extern "C" fn sample_output( _: &Object, _: Sel, _stream: id, - _buffer: id, + buffer: id, _kind: SCStreamOutputType, ) { - println!("sample output"); + let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; + let attachments = buffer.attachments(); + let attachments = attachments.first().expect("no attachments for sample"); + + unsafe { + let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; + let status = core_foundation::number::CFNumber::wrap_under_get_rule( + *attachments.get(string) as CFNumberRef, + ) + .to_i64() + .expect("invalid frame info status"); + + if status != bindings::SCFrameStatus_SCFrameStatusComplete { + println!("received incomplete frame"); + return; + } + } + + let image_buffer = buffer.image_buffer(); + dbg!(image_buffer.width(), image_buffer.height()); + let io_surface = image_buffer.io_surface(); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } + +mod core_media { + #![allow(non_snake_case)] + + use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use core_foundation::{ + array::{CFArray, CFArrayRef}, + base::{CFTypeID, TCFType}, + declare_TCFType, + dictionary::CFDictionary, + impl_CFTypeDescription, impl_TCFType, + string::CFString, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CMSampleBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMSampleBufferRef = *const __CMSampleBuffer; + + declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); + impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); + impl_CFTypeDescription!(CMSampleBuffer); + + impl CMSampleBuffer { + pub fn attachments(&self) -> Vec> { + unsafe { + let attachments = + CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); + CFArray::::wrap_under_get_rule(attachments) + .into_iter() + .map(|attachments| { + CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) + }) + .collect() + } + } + + pub fn image_buffer(&self) -> CVImageBuffer { + unsafe { + CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( + self.as_concrete_TypeRef(), + )) + } + } + } + + extern "C" { + fn CMSampleBufferGetTypeID() -> CFTypeID; + fn CMSampleBufferGetSampleAttachmentsArray( + buffer: CMSampleBufferRef, + create_if_necessary: bool, + ) -> CFArrayRef; + fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + } +} + +mod core_video { + #![allow(non_snake_case)] + + use crate::io_surface::{IOSurface, IOSurfaceRef}; + use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CVImageBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CVImageBufferRef = *const __CVImageBuffer; + + declare_TCFType!(CVImageBuffer, CVImageBufferRef); + impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); + impl_CFTypeDescription!(CVImageBuffer); + + impl CVImageBuffer { + pub fn io_surface(&self) -> IOSurface { + unsafe { + IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( + self.as_concrete_TypeRef(), + )) + } + } + + pub fn width(&self) -> usize { + unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } + } + + pub fn height(&self) -> usize { + unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } + } + } + + extern "C" { + fn CVImageBufferGetTypeID() -> CFTypeID; + fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; + fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + } +} + +mod io_surface { + #![allow(non_snake_case)] + + use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __IOSurface(c_void); + // The ref type must be a pointer to the underlying struct. + pub type IOSurfaceRef = *const __IOSurface; + + declare_TCFType!(IOSurface, IOSurfaceRef); + impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); + impl_CFTypeDescription!(IOSurface); + + extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; + } +} From d473b52f5abeea170523763027524f1789c66afc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 16:27:33 +0200 Subject: [PATCH 14/89] Remove `src/dummy.m` Co-Authored-By: Nathan Sobo --- crates/capture/build.rs | 7 ------- crates/capture/src/dummy.m | 7 ------- crates/capture/src/main.rs | 3 +-- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 crates/capture/src/dummy.m diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 550d1064ae7bcab0a8219ff1201e0071ccd847c1..a375d48f48670e912697c17181300643ebdf02c6 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -5,7 +5,6 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); - println!("cargo:rustc-link-arg=-ObjC"); let sdk_path = String::from_utf8( Command::new("xcrun") @@ -36,10 +35,4 @@ fn main() { bindings .write_to_file(out_path.join("bindings.rs")) .expect("couldn't write dispatch bindings"); - - println!("cargo:rerun-if-changed=src/dummy.m"); - cc::Build::new() - .file("src/dummy.m") - .flag("-mmacosx-version-min=12.3") - .compile("dummy"); } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m deleted file mode 100644 index 0ae2bda6a7646d48c3485a1e811ffc2c5ecb8888..0000000000000000000000000000000000000000 --- a/crates/capture/src/dummy.m +++ /dev/null @@ -1,7 +0,0 @@ -#import - -@interface MyClass : NSObject -@end - -@implementation MyClass -@end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 08a6cf03b2aaa9483b700aa96fde6e79a1ca413c..ac785a8d85141d965cdbbe65c386165523e76eac 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -14,7 +14,7 @@ use objc::{ class, declare::ClassDecl, msg_send, - runtime::{Object, Protocol, Sel}, + runtime::{Object, Sel}, sel, sel_impl, }; use simplelog::SimpleLogger; @@ -55,7 +55,6 @@ fn main() { let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); let capture_output_class = decl.register(); From e12eaf8c58a0e0a9be8e6dfc1dc1b8a25a5eb573 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 18:17:54 +0200 Subject: [PATCH 15/89] Start on `Scene::push_surface` Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/renderer.rs | 4 ++++ crates/gpui/src/scene.rs | 14 +++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 1d2f5fc8aeab255ff9bfb8880c5381ea2b707cc2..4ff73fcff28b36c1ff6509ae1e79eb5027aa6f45 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -13,6 +13,7 @@ use cocoa::base::{BOOL, NO, YES}; pub use dispatcher::Dispatcher; pub use fonts::FontSystem; use platform::{MacForegroundPlatform, MacPlatform}; +pub use renderer::Surface; use std::{rc::Rc, sync::Arc}; use window::Window; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 8fdbda15e84a8a2a212f80e424d6c722545e4322..711d9e48b54e1026e81fe4d41391015f25c8aaaa 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -37,6 +37,10 @@ struct PathSprite { shader_data: shaders::GPUISprite, } +pub struct Surface { + pub bounds: RectF, +} + impl Renderer { pub fn new( device: metal::Device, diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 086af5f64d1e93170a55ec8f0733f860cc098c1c..00b8d3c88b520a4c906b2abbdf43b112fbee5193 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -10,7 +10,7 @@ use crate::{ fonts::{FontId, GlyphId}, geometry::{rect::RectF, vector::Vector2F}, json::ToJson, - platform::CursorStyle, + platform::{current::Surface, CursorStyle}, ImageData, }; pub use mouse_region::*; @@ -34,6 +34,7 @@ pub struct Layer { quads: Vec, underlines: Vec, images: Vec, + surfaces: Vec, shadows: Vec, glyphs: Vec, image_glyphs: Vec, @@ -249,6 +250,10 @@ impl Scene { self.active_layer().push_image(image) } + pub fn push_surface(&mut self, surface: Surface) { + self.active_layer().push_surface(surface) + } + pub fn push_underline(&mut self, underline: Underline) { self.active_layer().push_underline(underline) } @@ -329,6 +334,7 @@ impl Layer { quads: Default::default(), underlines: Default::default(), images: Default::default(), + surfaces: Default::default(), shadows: Default::default(), image_glyphs: Default::default(), glyphs: Default::default(), @@ -391,6 +397,12 @@ impl Layer { } } + fn push_surface(&mut self, surface: Surface) { + if can_draw(surface.bounds) { + self.surfaces.push(surface); + } + } + pub fn images(&self) -> &[Image] { self.images.as_slice() } From c4110edb7893601abdaa8f5f849df40c8064827a Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:05:33 +0200 Subject: [PATCH 16/89] Extract `io_surface` crate and invoke custom callback on frame sample Co-Authored-By: Nathan Sobo --- Cargo.lock | 11 +++++ crates/capture/Cargo.toml | 1 + crates/capture/build.rs | 3 +- crates/capture/src/bindings.rs | 5 ++ crates/capture/src/main.rs | 59 ++++++++++++------------ crates/gpui/Cargo.toml | 1 + crates/gpui/build.rs | 1 - crates/gpui/src/platform/mac.rs | 2 +- crates/gpui/src/platform/mac/renderer.rs | 1 + crates/io_surface/Cargo.toml | 13 ++++++ crates/io_surface/src/io_surface.rs | 21 +++++++++ 11 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 crates/io_surface/Cargo.toml create mode 100644 crates/io_surface/src/io_surface.rs diff --git a/Cargo.lock b/Cargo.lock index 7dff1a8bb834c5c5cb30facd056e84bc7b3d8d82..4f6f6fe8392c68e5b41b7c84f19e9207e0994306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -762,6 +762,7 @@ dependencies = [ "core-graphics", "foreign-types", "gpui", + "io_surface", "log", "objc", "simplelog", @@ -2228,6 +2229,7 @@ dependencies = [ "futures", "gpui_macros", "image", + "io_surface", "lazy_static", "log", "metal", @@ -2596,6 +2598,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "io_surface" +version = "0.1.0" +dependencies = [ + "block", + "core-foundation", + "objc", +] + [[package]] name = "iovec" version = "0.1.4" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index aade52f2fbc9a561bd9856e538e9edb5d80a21f9..ebb35d8ef2203d6feb7a55dc7c0b6b286d56b421 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -10,6 +10,7 @@ identifier = "dev.zed.Capture" [dependencies] gpui = { path = "../gpui" } +io_surface = { path = "../io_surface" } block = "0.1" cocoa = "0.24" diff --git a/crates/capture/build.rs b/crates/capture/build.rs index a375d48f48670e912697c17181300643ebdf02c6..8e95dca2ff3d47021a6b7cd3ceb9d1945151bc08 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -24,8 +24,9 @@ fn main() { .allowlist_function("CMTimeMake") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") + .allowlist_type("dispatch_queue_t") .allowlist_var("SCStreamFrameInfo.*") - .allowlist_function("dispatch_queue_create") + .allowlist_var("_dispatch_main_q") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index a111bede4de059f0c0157775778bcc56ebdd4079..68487fce3250d90578264e681be0679c275ad4c9 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,7 +1,12 @@ #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] #![allow(non_snake_case)] +#![allow(unused)] use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +pub fn dispatch_get_main_queue() -> dispatch_queue_t { + unsafe { std::mem::transmute(&_dispatch_main_q) } +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ac785a8d85141d965cdbbe65c386165523e76eac..94561bde7e78aa517c34e2d0920d41c43977a311 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; +use crate::bindings::{dispatch_get_main_queue, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, @@ -8,6 +8,7 @@ use cocoa::{ }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; use core_media::{CMSampleBuffer, CMSampleBufferRef}; +use gpui::elements::Canvas; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{ @@ -18,7 +19,7 @@ use objc::{ sel, sel_impl, }; use simplelog::SimpleLogger; -use std::{ptr, slice, str}; +use std::{ffi::c_void, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -55,11 +56,17 @@ fn main() { let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_ivar::<*mut c_void>("callback"); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); let capture_output_class = decl.register(); let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let callback = Box::new(|buffer: CMSampleBufferRef| { + dbg!("HEY!"); + }) as Box; + let callback = Box::into_raw(Box::new(callback)); + (*output).set_ivar("callback", callback as *mut c_void); let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; @@ -75,7 +82,7 @@ fn main() { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); + let queue = dispatch_get_main_queue(); let _: () = msg_send![stream, addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen @@ -107,19 +114,29 @@ fn main() { }); } -struct ScreenCaptureView; +struct ScreenCaptureView { + surface: io_surface::IOSurface, +} impl gpui::Entity for ScreenCaptureView { type Event = (); } +// impl ScreenCaptureView { +// pub fn new() -> Self {} +// } + impl gpui::View for ScreenCaptureView { fn ui_name() -> &'static str { "View" } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - Empty::new().boxed() + Canvas::new(|bounds, _, cx| { + + // cx.scene.push_surface(surface) + }) + .boxed() } } @@ -136,13 +153,18 @@ pub unsafe fn string_from_objc(string: id) -> String { } extern "C" fn sample_output( - _: &Object, + this: &Object, _: Sel, _stream: id, buffer: id, _kind: SCStreamOutputType, ) { let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; + let callback = unsafe { *this.get_ivar::<*mut c_void>("callback") }; + let callback = callback as *mut Box; + let x = unsafe { callback.as_mut().unwrap() }; + (*x)(buffer.as_concrete_TypeRef()); + let attachments = buffer.attachments(); let attachments = attachments.first().expect("no attachments for sample"); @@ -228,11 +250,11 @@ mod core_media { mod core_video { #![allow(non_snake_case)] - use crate::io_surface::{IOSurface, IOSurfaceRef}; use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; + use io_surface::{IOSurface, IOSurfaceRef}; use std::ffi::c_void; #[repr(C)] @@ -269,26 +291,3 @@ mod core_video { fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; } } - -mod io_surface { - #![allow(non_snake_case)] - - use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, - }; - use std::ffi::c_void; - - #[repr(C)] - pub struct __IOSurface(c_void); - // The ref type must be a pointer to the underlying struct. - pub type IOSurfaceRef = *const __IOSurface; - - declare_TCFType!(IOSurface, IOSurfaceRef); - impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); - impl_CFTypeDescription!(IOSurface); - - extern "C" { - fn IOSurfaceGetTypeID() -> CFTypeID; - } -} diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 08c32135fb5170644033232e20ae9859128e81b2..6539c292fca18b6e25d0cc93d86242e1eba01bda 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -60,6 +60,7 @@ png = "0.16" simplelog = "0.9" [target.'cfg(target_os = "macos")'.dependencies] +io_surface = { path = "../io_surface" } anyhow = "1" block = "0.1" cocoa = "0.24" diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 967423d3f73297ecb9a48b11558c3e364a3173f8..836d586c26bea742d7b31832d0b17b64d0295a05 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -19,7 +19,6 @@ fn generate_dispatch_bindings() { .header("src/platform/mac/dispatch.h") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") - .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 4ff73fcff28b36c1ff6509ae1e79eb5027aa6f45..c1a4d45561dae5307a0f628588c37b8977809a61 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,5 +1,5 @@ mod atlas; -pub mod dispatcher; +mod dispatcher; mod event; mod fonts; mod geometry; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 711d9e48b54e1026e81fe4d41391015f25c8aaaa..25870c81fbf47243f1cbaaa6b1ae3c93ebbeb477 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -39,6 +39,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, + pub native_surface: io_surface::IOSurface, } impl Renderer { diff --git a/crates/io_surface/Cargo.toml b/crates/io_surface/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..2d9324b6d8951def6dd62d82d728479b2f6226e0 --- /dev/null +++ b/crates/io_surface/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "io_surface" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/io_surface.rs" +doctest = false + +[dependencies] +block = "0.1" +core-foundation = "0.9.3" +objc = "0.2" \ No newline at end of file diff --git a/crates/io_surface/src/io_surface.rs b/crates/io_surface/src/io_surface.rs new file mode 100644 index 0000000000000000000000000000000000000000..13b1fd5ce1f7f8e64102d60210ae24ffd6914460 --- /dev/null +++ b/crates/io_surface/src/io_surface.rs @@ -0,0 +1,21 @@ +#![allow(non_snake_case)] + +use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, +}; +use std::ffi::c_void; + +#[repr(C)] +pub struct __IOSurface(c_void); +// The ref type must be a pointer to the underlying struct. +pub type IOSurfaceRef = *const __IOSurface; + +declare_TCFType!(IOSurface, IOSurfaceRef); +impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); +impl_CFTypeDescription!(IOSurface); + +#[link(name = "IOSurface", kind = "framework")] +extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; +} From d30e360664cc9b318aa045eaa16837754bc9b8bd Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:08:54 +0200 Subject: [PATCH 17/89] Retrieve IOSurface in Rust callback as opposed to doing so in delegate --- crates/capture/src/main.rs | 46 +++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 94561bde7e78aa517c34e2d0920d41c43977a311..ae02002e9eb247c658aa07f87fc410934de9a03b 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -63,7 +63,24 @@ fn main() { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; let callback = Box::new(|buffer: CMSampleBufferRef| { - dbg!("HEY!"); + let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); + let attachments = buffer.attachments(); + let attachments = attachments.first().expect("no attachments for sample"); + let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; + let status = core_foundation::number::CFNumber::wrap_under_get_rule( + *attachments.get(string) as CFNumberRef, + ) + .to_i64() + .expect("invalid frame info status"); + + if status != bindings::SCFrameStatus_SCFrameStatusComplete { + println!("received incomplete frame"); + return; + } + + let image_buffer = buffer.image_buffer(); + dbg!(image_buffer.width(), image_buffer.height()); + let io_surface = image_buffer.io_surface(); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -159,32 +176,11 @@ extern "C" fn sample_output( buffer: id, _kind: SCStreamOutputType, ) { - let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; - let callback = unsafe { *this.get_ivar::<*mut c_void>("callback") }; - let callback = callback as *mut Box; - let x = unsafe { callback.as_mut().unwrap() }; - (*x)(buffer.as_concrete_TypeRef()); - - let attachments = buffer.attachments(); - let attachments = attachments.first().expect("no attachments for sample"); - unsafe { - let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; - let status = core_foundation::number::CFNumber::wrap_under_get_rule( - *attachments.get(string) as CFNumberRef, - ) - .to_i64() - .expect("invalid frame info status"); - - if status != bindings::SCFrameStatus_SCFrameStatusComplete { - println!("received incomplete frame"); - return; - } + let callback = *this.get_ivar::<*mut c_void>("callback"); + let callback = &mut *(callback as *mut Box); + (*callback)(buffer as CMSampleBufferRef); } - - let image_buffer = buffer.image_buffer(); - dbg!(image_buffer.width(), image_buffer.height()); - let io_surface = image_buffer.io_surface(); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { From 71d9a880d6ef80b5128dc0d594b592e7c76c3839 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:28:17 +0200 Subject: [PATCH 18/89] WIP: Start pushing native surface to Scene This is segfaulting for some reason, so that's the next step to figure out. --- Cargo.lock | 3 ++ crates/capture/Cargo.toml | 3 ++ crates/capture/src/main.rs | 98 ++++++++++++++++++++++++++++---------- 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f6f6fe8392c68e5b41b7c84f19e9207e0994306..aa030b9575372fa5b3fc291b1f9c0bc743808a1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,10 +761,13 @@ dependencies = [ "core-foundation", "core-graphics", "foreign-types", + "futures", "gpui", "io_surface", "log", "objc", + "parking_lot 0.11.2", + "postage", "simplelog", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index ebb35d8ef2203d6feb7a55dc7c0b6b286d56b421..8c6e901bbd58c76bac4737f9495cb7a8343762ff 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -17,8 +17,11 @@ cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" foreign-types = "0.3" +futures = "0.3" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" +parking_lot = "0.11.1" +postage = { version = "0.4.1", features = ["futures-traits"] } simplelog = "0.9" [build-dependencies] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ae02002e9eb247c658aa07f87fc410934de9a03b..1b2ce6f8e8805d4894727a6c8f84a9deab2d19a8 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -8,8 +8,15 @@ use cocoa::{ }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; use core_media::{CMSampleBuffer, CMSampleBufferRef}; -use gpui::elements::Canvas; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use futures::StreamExt; +use gpui::{ + actions, + elements::{Canvas, *}, + keymap::Binding, + platform::current::Surface, + Menu, MenuItem, ViewContext, +}; +use io_surface::IOSurface; use log::LevelFilter; use objc::{ class, @@ -18,8 +25,9 @@ use objc::{ runtime::{Object, Sel}, sel, sel_impl, }; +use parking_lot::Mutex; use simplelog::SimpleLogger; -use std::{ffi::c_void, slice, str}; +use std::{cell::RefCell, ffi::c_void, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -42,10 +50,29 @@ fn main() { }], }]); + cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + }); +} + +struct ScreenCaptureView { + surface: Option, +} + +impl gpui::Entity for ScreenCaptureView { + type Event = (); +} + +impl ScreenCaptureView { + pub fn new(cx: &mut ViewContext) -> Self { + let (surface_tx, mut surface_rx) = postage::watch::channel::>(); + let surface_tx = RefCell::new(surface_tx); unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { if !error.is_null() { - println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); + println!( + "ERROR {}", + string_from_objc(msg_send![error, localizedDescription]) + ); return; } @@ -57,11 +84,16 @@ fn main() { let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); - decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); + decl.add_method( + sel!(stream:didOutputSampleBuffer:ofType:), + sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType), + ); let capture_output_class = decl.register(); let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + + // TODO: we could probably move this into a background queue. let callback = Box::new(|buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); let attachments = buffer.attachments(); @@ -79,8 +111,11 @@ fn main() { } let image_buffer = buffer.image_buffer(); - dbg!(image_buffer.width(), image_buffer.height()); let io_surface = image_buffer.io_surface(); + println!("before segfault"); + *surface_tx.borrow_mut().borrow_mut() = Some(io_surface); + + println!("!!!!!!!!!!!!!!!!!"); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -109,7 +144,10 @@ fn main() { let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { - println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + println!( + "error starting capture... error? {}", + string_from_objc(msg_send![error, localizedDescription]) + ); return; } @@ -117,8 +155,10 @@ fn main() { }); assert!(!stream.is_null()); - let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; - + let _: () = msg_send![ + stream, + startCaptureWithCompletionHandler: start_capture_completion + ]; }); let _: id = msg_send![ @@ -127,31 +167,39 @@ fn main() { ]; } - // cx.add_window(Default::default(), |_| ScreenCaptureView); - }); -} - -struct ScreenCaptureView { - surface: io_surface::IOSurface, -} + cx.spawn_weak(|this, mut cx| async move { + while let Some(surface) = surface_rx.next().await { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.surface = surface; + cx.notify(); + }) + } else { + break; + } + } + }) + .detach(); -impl gpui::Entity for ScreenCaptureView { - type Event = (); + Self { surface: None } + } } -// impl ScreenCaptureView { -// pub fn new() -> Self {} -// } - impl gpui::View for ScreenCaptureView { fn ui_name() -> &'static str { "View" } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - Canvas::new(|bounds, _, cx| { - - // cx.scene.push_surface(surface) + let surface = self.surface.clone(); + Canvas::new(move |bounds, _, cx| { + if let Some(native_surface) = surface.clone() { + dbg!("!!!!"); + cx.scene.push_surface(Surface { + bounds, + native_surface, + }); + } }) .boxed() } From db88ee2f4ce8fd30fe955141ea3a29620064582d Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 13:15:38 -0600 Subject: [PATCH 19/89] Handle captured frames in the background --- crates/capture/build.rs | 3 +-- crates/capture/src/bindings.rs | 4 ---- crates/capture/src/main.rs | 23 ++++++++++++----------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 8e95dca2ff3d47021a6b7cd3ceb9d1945151bc08..482e6487ceba34adf30434c8b47537c3f82c75e3 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -22,11 +22,10 @@ fn main() { .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") + .allowlist_function("dispatch_queue_create") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") - .allowlist_type("dispatch_queue_t") .allowlist_var("SCStreamFrameInfo.*") - .allowlist_var("_dispatch_main_q") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index 68487fce3250d90578264e681be0679c275ad4c9..a1c0b0da3ef5175063e48cd13d9b37d33c6369b3 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -6,7 +6,3 @@ use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); - -pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { std::mem::transmute(&_dispatch_main_q) } -} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 1b2ce6f8e8805d4894727a6c8f84a9deab2d19a8..e83135fc6427ad84d2ad2cc5881e9812c0ab91bd 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use crate::bindings::{dispatch_get_main_queue, SCStreamOutputType}; +use crate::bindings::SCStreamOutputType; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, @@ -27,7 +27,7 @@ use objc::{ }; use parking_lot::Mutex; use simplelog::SimpleLogger; -use std::{cell::RefCell, ffi::c_void, slice, str}; +use std::{ffi::c_void, ptr, slice, str, sync::Arc}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -65,7 +65,8 @@ impl gpui::Entity for ScreenCaptureView { impl ScreenCaptureView { pub fn new(cx: &mut ViewContext) -> Self { let (surface_tx, mut surface_rx) = postage::watch::channel::>(); - let surface_tx = RefCell::new(surface_tx); + let surface_tx = Arc::new(Mutex::new(surface_tx)); + unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { if !error.is_null() { @@ -92,9 +93,9 @@ impl ScreenCaptureView { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let surface_tx = surface_tx.clone(); - // TODO: we could probably move this into a background queue. - let callback = Box::new(|buffer: CMSampleBufferRef| { + let callback = Box::new(move |buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); let attachments = buffer.attachments(); let attachments = attachments.first().expect("no attachments for sample"); @@ -112,10 +113,7 @@ impl ScreenCaptureView { let image_buffer = buffer.image_buffer(); let io_surface = image_buffer.io_surface(); - println!("before segfault"); - *surface_tx.borrow_mut().borrow_mut() = Some(io_surface); - - println!("!!!!!!!!!!!!!!!!!"); + *surface_tx.lock().borrow_mut() = Some(io_surface); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -134,7 +132,10 @@ impl ScreenCaptureView { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - let queue = dispatch_get_main_queue(); + let queue = bindings::dispatch_queue_create( + ptr::null(), + bindings::NSObject(ptr::null_mut()), + ); let _: () = msg_send![stream, addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen @@ -172,6 +173,7 @@ impl ScreenCaptureView { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { this.surface = surface; + println!("NEW SURFACE!"); cx.notify(); }) } else { @@ -194,7 +196,6 @@ impl gpui::View for ScreenCaptureView { let surface = self.surface.clone(); Canvas::new(move |bounds, _, cx| { if let Some(native_surface) = surface.clone() { - dbg!("!!!!"); cx.scene.push_surface(Surface { bounds, native_surface, From 0430bbf7d95dee8605eac1ac31995f4724ca8c41 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 13:28:40 -0600 Subject: [PATCH 20/89] WIP: Start on Renderer::render_surfaces but really it's nothing --- crates/gpui/src/platform/mac/renderer.rs | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 25870c81fbf47243f1cbaaa6b1ae3c93ebbeb477..41883a2f73c6e8afd7cd7ba6772747139dd3291f 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -773,6 +773,79 @@ impl Renderer { } } + fn render_surfaces( + &mut self, + surfaces: &[Surface], + scale_factor: f32, + offset: &mut usize, + drawable_size: Vector2F, + command_encoder: &metal::RenderCommandEncoderRef, + ) { + if surfaces.is_empty() { + return; + } + + for surface in surfaces { + let origin = surface.bounds.origin() * scale_factor; + let target_size = surface.bounds.size() * scale_factor; + // let corner_radius = surface.corner_radius * scale_factor; + // let border_width = surface.border.width * scale_factor; + // let (alloc_id, atlas_bounds) = self.image_cache.render(&surface.native_surface); + } + + // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + // command_encoder.set_vertex_buffer( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + // Some(&self.unit_vertices), + // 0, + // ); + // command_encoder.set_vertex_bytes( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + // mem::size_of::() as u64, + // [drawable_size.to_float2()].as_ptr() as *const c_void, + // ); + + // for (atlas_id, images) in images_by_atlas { + // align_offset(offset); + // let next_offset = *offset + images.len() * mem::size_of::(); + // assert!( + // next_offset <= INSTANCE_BUFFER_SIZE, + // "instance buffer exhausted" + // ); + + // let texture = self.image_cache.atlas_texture(atlas_id).unwrap(); + // command_encoder.set_vertex_buffer( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + // Some(&self.instances), + // *offset as u64, + // ); + // command_encoder.set_vertex_bytes( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + // mem::size_of::() as u64, + // [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr() + // as *const c_void, + // ); + // command_encoder.set_fragment_texture( + // shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, + // Some(texture), + // ); + + // unsafe { + // let buffer_contents = + // (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + // std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len()); + // } + + // command_encoder.draw_primitives_instanced( + // metal::MTLPrimitiveType::Triangle, + // 0, + // 6, + // images.len() as u64, + // ); + // *offset = next_offset; + // } + } + fn render_path_sprites( &mut self, layer_id: usize, From 531ffc01c91bb0bccc82be6f90e3ff0870d7ea4b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 20:34:59 -0600 Subject: [PATCH 21/89] Pass CVImageBuffers into GPUI instead of IOSurfaces --- Cargo.lock | 22 ++++---- crates/capture/Cargo.toml | 2 +- crates/capture/src/main.rs | 71 +++++------------------- crates/gpui/Cargo.toml | 2 +- crates/gpui/src/platform/mac/renderer.rs | 3 +- crates/io_surface/src/io_surface.rs | 21 ------- crates/{io_surface => media}/Cargo.toml | 4 +- crates/media/src/media.rs | 66 ++++++++++++++++++++++ 8 files changed, 95 insertions(+), 96 deletions(-) delete mode 100644 crates/io_surface/src/io_surface.rs rename crates/{io_surface => media}/Cargo.toml (67%) create mode 100644 crates/media/src/media.rs diff --git a/Cargo.lock b/Cargo.lock index aa030b9575372fa5b3fc291b1f9c0bc743808a1e..7b4fbdaaa3b6d072349094bc856718e5b346bcc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,8 +763,8 @@ dependencies = [ "foreign-types", "futures", "gpui", - "io_surface", "log", + "media", "objc", "parking_lot 0.11.2", "postage", @@ -2232,9 +2232,9 @@ dependencies = [ "futures", "gpui_macros", "image", - "io_surface", "lazy_static", "log", + "media", "metal", "num_cpus", "objc", @@ -2601,15 +2601,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "io_surface" -version = "0.1.0" -dependencies = [ - "block", - "core-foundation", - "objc", -] - [[package]] name = "iovec" version = "0.1.4" @@ -3039,6 +3030,15 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "media" +version = "0.1.0" +dependencies = [ + "block", + "core-foundation", + "objc", +] + [[package]] name = "memchr" version = "2.5.0" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 8c6e901bbd58c76bac4737f9495cb7a8343762ff..f52fde195edf4301ddf3242add8c1398f4972b1e 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -10,7 +10,7 @@ identifier = "dev.zed.Capture" [dependencies] gpui = { path = "../gpui" } -io_surface = { path = "../io_surface" } +media = { path = "../media" } block = "0.1" cocoa = "0.24" diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e83135fc6427ad84d2ad2cc5881e9812c0ab91bd..ed9f56cd18ee3618a6936e4c30b46ab4a04d3f49 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -16,8 +16,8 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; -use io_surface::IOSurface; use log::LevelFilter; +use media::core_video::{self, CVImageBuffer}; use objc::{ class, declare::ClassDecl, @@ -55,7 +55,7 @@ fn main() { } struct ScreenCaptureView { - surface: Option, + image_buffer: Option, } impl gpui::Entity for ScreenCaptureView { @@ -64,8 +64,9 @@ impl gpui::Entity for ScreenCaptureView { impl ScreenCaptureView { pub fn new(cx: &mut ViewContext) -> Self { - let (surface_tx, mut surface_rx) = postage::watch::channel::>(); - let surface_tx = Arc::new(Mutex::new(surface_tx)); + let (image_buffer_tx, mut image_buffer_rx) = + postage::watch::channel::>(); + let image_buffer_tx = Arc::new(Mutex::new(image_buffer_tx)); unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { @@ -93,7 +94,7 @@ impl ScreenCaptureView { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; - let surface_tx = surface_tx.clone(); + let surface_tx = image_buffer_tx.clone(); let callback = Box::new(move |buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); @@ -112,8 +113,7 @@ impl ScreenCaptureView { } let image_buffer = buffer.image_buffer(); - let io_surface = image_buffer.io_surface(); - *surface_tx.lock().borrow_mut() = Some(io_surface); + *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -169,10 +169,10 @@ impl ScreenCaptureView { } cx.spawn_weak(|this, mut cx| async move { - while let Some(surface) = surface_rx.next().await { + while let Some(image_buffer) = image_buffer_rx.next().await { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { - this.surface = surface; + this.image_buffer = image_buffer; println!("NEW SURFACE!"); cx.notify(); }) @@ -183,7 +183,7 @@ impl ScreenCaptureView { }) .detach(); - Self { surface: None } + Self { image_buffer: None } } } @@ -193,12 +193,12 @@ impl gpui::View for ScreenCaptureView { } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - let surface = self.surface.clone(); + let image_buffer = self.image_buffer.clone(); Canvas::new(move |bounds, _, cx| { - if let Some(native_surface) = surface.clone() { + if let Some(image_buffer) = image_buffer.clone() { cx.scene.push_surface(Surface { bounds, - native_surface, + image_buffer, }); } }) @@ -291,48 +291,3 @@ mod core_media { fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; } } - -mod core_video { - #![allow(non_snake_case)] - - use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, - }; - use io_surface::{IOSurface, IOSurfaceRef}; - use std::ffi::c_void; - - #[repr(C)] - pub struct __CVImageBuffer(c_void); - // The ref type must be a pointer to the underlying struct. - pub type CVImageBufferRef = *const __CVImageBuffer; - - declare_TCFType!(CVImageBuffer, CVImageBufferRef); - impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); - impl_CFTypeDescription!(CVImageBuffer); - - impl CVImageBuffer { - pub fn io_surface(&self) -> IOSurface { - unsafe { - IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( - self.as_concrete_TypeRef(), - )) - } - } - - pub fn width(&self) -> usize { - unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } - } - - pub fn height(&self) -> usize { - unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } - } - } - - extern "C" { - fn CVImageBufferGetTypeID() -> CFTypeID; - fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; - fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; - fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; - } -} diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 6539c292fca18b6e25d0cc93d86242e1eba01bda..51bc416e191b726b85121e28f681dd6550e81215 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -60,7 +60,7 @@ png = "0.16" simplelog = "0.9" [target.'cfg(target_os = "macos")'.dependencies] -io_surface = { path = "../io_surface" } +media = { path = "../media" } anyhow = "1" block = "0.1" cocoa = "0.24" diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 41883a2f73c6e8afd7cd7ba6772747139dd3291f..6c1d3cef219535044888d3a17527d8531acbc913 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -39,7 +39,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, - pub native_surface: io_surface::IOSurface, + pub image_buffer: media::core_video::CVImageBuffer, } impl Renderer { @@ -790,7 +790,6 @@ impl Renderer { let target_size = surface.bounds.size() * scale_factor; // let corner_radius = surface.corner_radius * scale_factor; // let border_width = surface.border.width * scale_factor; - // let (alloc_id, atlas_bounds) = self.image_cache.render(&surface.native_surface); } // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); diff --git a/crates/io_surface/src/io_surface.rs b/crates/io_surface/src/io_surface.rs deleted file mode 100644 index 13b1fd5ce1f7f8e64102d60210ae24ffd6914460..0000000000000000000000000000000000000000 --- a/crates/io_surface/src/io_surface.rs +++ /dev/null @@ -1,21 +0,0 @@ -#![allow(non_snake_case)] - -use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, -}; -use std::ffi::c_void; - -#[repr(C)] -pub struct __IOSurface(c_void); -// The ref type must be a pointer to the underlying struct. -pub type IOSurfaceRef = *const __IOSurface; - -declare_TCFType!(IOSurface, IOSurfaceRef); -impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); -impl_CFTypeDescription!(IOSurface); - -#[link(name = "IOSurface", kind = "framework")] -extern "C" { - fn IOSurfaceGetTypeID() -> CFTypeID; -} diff --git a/crates/io_surface/Cargo.toml b/crates/media/Cargo.toml similarity index 67% rename from crates/io_surface/Cargo.toml rename to crates/media/Cargo.toml index 2d9324b6d8951def6dd62d82d728479b2f6226e0..ff472fe35252e7556323851751fb27d4abab9d9b 100644 --- a/crates/io_surface/Cargo.toml +++ b/crates/media/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "io_surface" +name = "media" version = "0.1.0" edition = "2021" [lib] -path = "src/io_surface.rs" +path = "src/media.rs" doctest = false [dependencies] diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs new file mode 100644 index 0000000000000000000000000000000000000000..435c03e93618fbd1ad71b93b02a464e4475a8b28 --- /dev/null +++ b/crates/media/src/media.rs @@ -0,0 +1,66 @@ +#![allow(non_snake_case)] + +use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, +}; +use std::ffi::c_void; + +pub mod io_surface { + use super::*; + + #[repr(C)] + pub struct __IOSurface(c_void); + // The ref type must be a pointer to the underlying struct. + pub type IOSurfaceRef = *const __IOSurface; + + declare_TCFType!(IOSurface, IOSurfaceRef); + impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); + impl_CFTypeDescription!(IOSurface); + + #[link(name = "IOSurface", kind = "framework")] + extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; + } +} + +pub mod core_video { + #![allow(non_snake_case)] + + use super::*; + use io_surface::{IOSurface, IOSurfaceRef}; + + #[repr(C)] + pub struct __CVImageBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CVImageBufferRef = *const __CVImageBuffer; + + declare_TCFType!(CVImageBuffer, CVImageBufferRef); + impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); + impl_CFTypeDescription!(CVImageBuffer); + + impl CVImageBuffer { + pub fn io_surface(&self) -> IOSurface { + unsafe { + IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( + self.as_concrete_TypeRef(), + )) + } + } + + pub fn width(&self) -> usize { + unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } + } + + pub fn height(&self) -> usize { + unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } + } + } + + extern "C" { + fn CVImageBufferGetTypeID() -> CFTypeID; + fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; + fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + } +} From 37da841716cbe14d70cb05200ea3c4b0187eec84 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 21:50:41 -0600 Subject: [PATCH 22/89] Start on using CVMetalTextureCache --- Cargo.lock | 1 + crates/gpui/src/platform/mac/renderer.rs | 30 ++++++- crates/media/Cargo.toml | 3 +- crates/media/src/media.rs | 100 +++++++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b4fbdaaa3b6d072349094bc856718e5b346bcc1..46abc687655697b6ed618d4193f8f8bca388588d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,6 +3036,7 @@ version = "0.1.0" dependencies = [ "block", "core-foundation", + "metal", "objc", ] diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 6c1d3cef219535044888d3a17527d8531acbc913..1023d5a5ef749558ffcbd8fe6a444c83de965410 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -9,10 +9,13 @@ use crate::{ scene::{Glyph, Icon, Image, ImageGlyph, Layer, Quad, Scene, Shadow, Underline}, }; use cocoa::foundation::NSUInteger; +use core_foundation::base::TCFType; +use foreign_types::ForeignTypeRef; use log::warn; +use media::core_video::{self, CVMetalTextureCache}; use metal::{MTLPixelFormat, MTLResourceOptions, NSRange}; use shaders::ToFloat2 as _; -use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, sync::Arc, vec}; +use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; const SHADERS_METALLIB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/shaders.metallib")); const INSTANCE_BUFFER_SIZE: usize = 8192 * 1024; // This is an arbitrary decision. There's probably a more optimal value. @@ -29,6 +32,7 @@ pub struct Renderer { underline_pipeline_state: metal::RenderPipelineState, unit_vertices: metal::Buffer, instances: metal::Buffer, + cv_texture_cache: core_video::CVMetalTextureCache, } struct PathSprite { @@ -39,7 +43,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, - pub image_buffer: media::core_video::CVImageBuffer, + pub image_buffer: core_video::CVImageBuffer, } impl Renderer { @@ -128,6 +132,7 @@ impl Renderer { "underline_fragment", pixel_format, ); + let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()); Self { sprite_cache, image_cache, @@ -140,6 +145,7 @@ impl Renderer { underline_pipeline_state, unit_vertices, instances, + cv_texture_cache, } } @@ -786,10 +792,26 @@ impl Renderer { } for surface in surfaces { - let origin = surface.bounds.origin() * scale_factor; - let target_size = surface.bounds.size() * scale_factor; + // let origin = surface.bounds.origin() * scale_factor; + // let target_size = surface.bounds.size() * scale_factor; // let corner_radius = surface.corner_radius * scale_factor; // let border_width = surface.border.width * scale_factor; + + let width = surface.image_buffer.width(); + let height = surface.image_buffer.height(); + + // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat + // I found at least one code example that manually maps them. Not sure what other options we have. + let pixel_format = surface.image_buffer.pixel_format_type(); + + let texture = self.cv_texture_cache.create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + pixel_format, + width, + height, + 0, + ); } // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index ff472fe35252e7556323851751fb27d4abab9d9b..06f73caaf58635dff3ba53fe48f7845cb30122dc 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,4 +10,5 @@ doctest = false [dependencies] block = "0.1" core-foundation = "0.9.3" -objc = "0.2" \ No newline at end of file +metal = "0.21.0" +objc = "0.2" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 435c03e93618fbd1ad71b93b02a464e4475a8b28..855435359cb4dd18408883b86fb7581021dfe343 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -1,11 +1,15 @@ #![allow(non_snake_case)] +#![allow(non_camel_case_types)] use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; +use objc::runtime; use std::ffi::c_void; +pub type id = *mut runtime::Object; + pub mod io_surface { use super::*; @@ -27,8 +31,14 @@ pub mod io_surface { pub mod core_video { #![allow(non_snake_case)] + use std::ptr; + use super::*; + use core_foundation::{ + base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, + }; use io_surface::{IOSurface, IOSurfaceRef}; + use metal::{MTLDevice, MTLPixelFormat}; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -63,4 +73,94 @@ pub mod core_video { fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; } + + #[repr(C)] + pub struct __CVMetalTextureCache(c_void); + pub type CVMetalTextureCacheRef = *const __CVMetalTextureCache; + + declare_TCFType!(CVMetalTextureCache, CVMetalTextureCacheRef); + impl_TCFType!( + CVMetalTextureCache, + CVMetalTextureCacheRef, + CVMetalTextureCacheGetTypeID + ); + impl_CFTypeDescription!(CVMetalTextureCache); + + impl CVMetalTextureCache { + pub fn new(metal_device: *mut MTLDevice) -> Self { + unsafe { + let mut this = ptr::null(); + let result = CVMetalTextureCacheCreate( + kCFAllocatorDefault, + ptr::null_mut(), + metal_device, + ptr::null_mut(), + &mut this, + ); + // TODO: Check result + CVMetalTextureCache::wrap_under_create_rule(this) + } + } + + pub fn create_texture_from_image( + &self, + source: CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + ) -> CVMetalTexture { + unsafe { + let mut this = ptr::null(); + let result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + self.as_concrete_TypeRef(), + source, + texture_attributes, + pixel_format, + width, + height, + plane_index, + &mut this, + ); + // TODO: Check result + CVMetalTexture::wrap_under_create_rule(this) + } + } + } + + extern "C" { + fn CVMetalTextureCacheGetTypeID() -> CFTypeID; + fn CVMetalTextureCacheCreate( + allocator: CFAllocatorRef, + cache_attributes: CFDictionaryRef, + metal_device: *const MTLDevice, + texture_attributes: CFDictionaryRef, + cache_out: *mut CVMetalTextureCacheRef, + ) -> i32; // TODO: This should be a CVReturn enum + fn CVMetalTextureCacheCreateTextureFromImage( + allocator: CFAllocatorRef, + texture_cache: CVMetalTextureCacheRef, + source_image: CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + texture_out: *mut CVMetalTextureRef, + ) -> i32; + } + + #[repr(C)] + pub struct __CVMetalTexture(c_void); + pub type CVMetalTextureRef = *const __CVMetalTexture; + + declare_TCFType!(CVMetalTexture, CVMetalTextureRef); + impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); + impl_CFTypeDescription!(CVMetalTexture); + + extern "C" { + fn CVMetalTextureGetTypeID() -> CFTypeID; + } } From 79a7a0e0e709a61965c6c15070b14f94f3fe8b60 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:02:48 +0200 Subject: [PATCH 23/89] Capture screen in BGRA8 and render it in `capture` example app --- Cargo.lock | 3 +- crates/capture/Cargo.toml | 1 - crates/capture/src/main.rs | 5 +- crates/gpui/src/platform/mac/renderer.rs | 144 +++++++++++++---------- crates/gpui/src/scene.rs | 8 +- crates/media/Cargo.toml | 4 + crates/media/build.rs | 29 +++++ crates/media/src/bindings.h | 1 + crates/media/src/bindings.rs | 8 ++ crates/media/src/media.rs | 24 +++- 10 files changed, 156 insertions(+), 71 deletions(-) create mode 100644 crates/media/build.rs create mode 100644 crates/media/src/bindings.h create mode 100644 crates/media/src/bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 46abc687655697b6ed618d4193f8f8bca388588d..b2d886db0adbef9051cdf2d52d2d47cc36be470e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -756,7 +756,6 @@ version = "0.1.0" dependencies = [ "bindgen", "block", - "cc", "cocoa", "core-foundation", "core-graphics", @@ -3034,8 +3033,10 @@ dependencies = [ name = "media" version = "0.1.0" dependencies = [ + "bindgen", "block", "core-foundation", + "foreign-types", "metal", "objc", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index f52fde195edf4301ddf3242add8c1398f4972b1e..54b1d0990e697ab7dfccb7f266abd3f3ad515bc2 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -26,4 +26,3 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" -cc = "1.0" \ No newline at end of file diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ed9f56cd18ee3618a6936e4c30b46ab4a04d3f49..af67ed9da45a0b10ec1af0d2c1eb974e9fd66d0b 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -128,6 +128,10 @@ impl ScreenCaptureView { let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; + let _: () = msg_send![ + config, + setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA + ]; let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; @@ -173,7 +177,6 @@ impl ScreenCaptureView { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { this.image_buffer = image_buffer; - println!("NEW SURFACE!"); cx.notify(); }) } else { diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 1023d5a5ef749558ffcbd8fe6a444c83de965410..b0301e9cb6ddd02facc3436f014e3a883e65e69a 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -383,6 +383,13 @@ impl Renderer { drawable_size, command_encoder, ); + self.render_surfaces( + layer.surfaces(), + scale_factor, + offset, + drawable_size, + command_encoder, + ); } command_encoder.end_encoding(); @@ -791,80 +798,87 @@ impl Renderer { return; } - for surface in surfaces { - // let origin = surface.bounds.origin() * scale_factor; - // let target_size = surface.bounds.size() * scale_factor; - // let corner_radius = surface.corner_radius * scale_factor; - // let border_width = surface.border.width * scale_factor; - - let width = surface.image_buffer.width(); - let height = surface.image_buffer.height(); + command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + Some(&self.unit_vertices), + 0, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + mem::size_of::() as u64, + [drawable_size.to_float2()].as_ptr() as *const c_void, + ); - // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat - // I found at least one code example that manually maps them. Not sure what other options we have. - let pixel_format = surface.image_buffer.pixel_format_type(); + for surface in surfaces { + let origin = surface.bounds.origin() * scale_factor; + let source_size = vec2i( + surface.image_buffer.width() as i32, + surface.image_buffer.height() as i32, + ); + let target_size = surface.bounds.size(); + let pixel_format = if surface.image_buffer.pixel_format_type() + == core_video::kCVPixelFormatType_32BGRA + { + MTLPixelFormat::BGRA8Unorm + } else { + panic!("unsupported pixel format") + }; let texture = self.cv_texture_cache.create_texture_from_image( surface.image_buffer.as_concrete_TypeRef(), ptr::null(), pixel_format, - width, - height, + source_size.x() as usize, + source_size.y() as usize, 0, ); - } - // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, - // Some(&self.unit_vertices), - // 0, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, - // mem::size_of::() as u64, - // [drawable_size.to_float2()].as_ptr() as *const c_void, - // ); - - // for (atlas_id, images) in images_by_atlas { - // align_offset(offset); - // let next_offset = *offset + images.len() * mem::size_of::(); - // assert!( - // next_offset <= INSTANCE_BUFFER_SIZE, - // "instance buffer exhausted" - // ); - - // let texture = self.image_cache.atlas_texture(atlas_id).unwrap(); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, - // Some(&self.instances), - // *offset as u64, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, - // mem::size_of::() as u64, - // [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr() - // as *const c_void, - // ); - // command_encoder.set_fragment_texture( - // shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, - // Some(texture), - // ); - - // unsafe { - // let buffer_contents = - // (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; - // std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len()); - // } - - // command_encoder.draw_primitives_instanced( - // metal::MTLPrimitiveType::Triangle, - // 0, - // 6, - // images.len() as u64, - // ); - // *offset = next_offset; - // } + align_offset(offset); + let next_offset = *offset + mem::size_of::(); + assert!( + next_offset <= INSTANCE_BUFFER_SIZE, + "instance buffer exhausted" + ); + + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + Some(&self.instances), + *offset as u64, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + mem::size_of::() as u64, + [source_size.to_float2()].as_ptr() as *const c_void, + ); + command_encoder.set_fragment_texture( + shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, + Some(texture.as_texture_ref()), + ); + + unsafe { + let buffer_contents = + (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + std::ptr::write( + buffer_contents, + shaders::GPUIImage { + origin: origin.to_float2(), + target_size: target_size.to_float2(), + source_size: source_size.to_float2(), + atlas_origin: Default::default(), + border_top: Default::default(), + border_right: Default::default(), + border_bottom: Default::default(), + border_left: Default::default(), + border_color: Default::default(), + corner_radius: Default::default(), + }, + ); + } + + command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6); + *offset = next_offset; + } } fn render_path_sprites( diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 00b8d3c88b520a4c906b2abbdf43b112fbee5193..dbd46da1a07f6899621071288b826990a5465d22 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -397,14 +397,18 @@ impl Layer { } } + pub fn images(&self) -> &[Image] { + self.images.as_slice() + } + fn push_surface(&mut self, surface: Surface) { if can_draw(surface.bounds) { self.surfaces.push(surface); } } - pub fn images(&self) -> &[Image] { - self.images.as_slice() + pub fn surfaces(&self) -> &[Surface] { + self.surfaces.as_slice() } fn push_shadow(&mut self, shadow: Shadow) { diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 06f73caaf58635dff3ba53fe48f7845cb30122dc..c2ca04abfa4f18a176df7aa7cb6e5409e19ce0b7 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,5 +10,9 @@ doctest = false [dependencies] block = "0.1" core-foundation = "0.9.3" +foreign-types = "0.3" metal = "0.21.0" objc = "0.2" + +[build-dependencies] +bindgen = "0.59.2" diff --git a/crates/media/build.rs b/crates/media/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..3446311d8939c463c1e42576c1e1bf41a317ee08 --- /dev/null +++ b/crates/media/build.rs @@ -0,0 +1,29 @@ +use std::{env, path::PathBuf, process::Command}; + +fn main() { + let sdk_path = String::from_utf8( + Command::new("xcrun") + .args(&["--sdk", "macosx", "--show-sdk-path"]) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + let sdk_path = sdk_path.trim_end(); + + println!("cargo:rerun-if-changed=src/bindings.h"); + let bindings = bindgen::Builder::default() + .header("src/bindings.h") + .clang_arg(format!("-isysroot{}", sdk_path)) + .clang_arg("-xobjective-c") + .allowlist_var("kCVPixelFormatType_.*") + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .layout_tests(false) + .generate() + .expect("unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write dispatch bindings"); +} diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h new file mode 100644 index 0000000000000000000000000000000000000000..d5ba4eccd695c9305ed206963af5493d7480d080 --- /dev/null +++ b/crates/media/src/bindings.h @@ -0,0 +1 @@ +#import \ No newline at end of file diff --git a/crates/media/src/bindings.rs b/crates/media/src/bindings.rs new file mode 100644 index 0000000000000000000000000000000000000000..a1c0b0da3ef5175063e48cd13d9b37d33c6369b3 --- /dev/null +++ b/crates/media/src/bindings.rs @@ -0,0 +1,8 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(unused)] + +use objc::*; + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 855435359cb4dd18408883b86fb7581021dfe343..5db184f3b289b09f107e9a7a5a1f5da336ad1bc0 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -1,6 +1,8 @@ #![allow(non_snake_case)] #![allow(non_camel_case_types)] +mod bindings; + use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, @@ -34,11 +36,13 @@ pub mod core_video { use std::ptr; use super::*; + pub use crate::bindings::*; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, }; + use foreign_types::ForeignTypeRef; use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat}; + use metal::{MTLDevice, MTLPixelFormat, MTLTexture}; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -65,13 +69,19 @@ pub mod core_video { pub fn height(&self) -> usize { unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } } + + pub fn pixel_format_type(&self) -> OSType { + unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } + } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVImageBufferGetTypeID() -> CFTypeID; fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; } #[repr(C)] @@ -130,6 +140,7 @@ pub mod core_video { } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureCacheGetTypeID() -> CFTypeID; fn CVMetalTextureCacheCreate( @@ -160,7 +171,18 @@ pub mod core_video { impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); impl_CFTypeDescription!(CVMetalTexture); + impl CVMetalTexture { + pub fn as_texture_ref(&self) -> &metal::TextureRef { + unsafe { + let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef()); + &metal::TextureRef::from_ptr(texture as *mut _) + } + } + } + + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureGetTypeID() -> CFTypeID; + fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } } From fcf6aa15ebca1ccb09e14a89c936411046f8e253 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:09:14 +0200 Subject: [PATCH 24/89] Return results for fallible media APIs --- Cargo.lock | 1 + crates/gpui/src/platform/mac/renderer.rs | 21 ++++++++++-------- crates/media/Cargo.toml | 1 + crates/media/build.rs | 1 + crates/media/src/bindings.h | 3 ++- crates/media/src/media.rs | 28 ++++++++++++++---------- 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2d886db0adbef9051cdf2d52d2d47cc36be470e..669fedf6592c4914762aabce8e2146d4acc5bb8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3033,6 +3033,7 @@ dependencies = [ name = "media" version = "0.1.0" dependencies = [ + "anyhow", "bindgen", "block", "core-foundation", diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index b0301e9cb6ddd02facc3436f014e3a883e65e69a..8bf512a1a7c2345a207b935a12631cac23483643 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -132,7 +132,7 @@ impl Renderer { "underline_fragment", pixel_format, ); - let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()); + let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()).unwrap(); Self { sprite_cache, image_cache, @@ -825,14 +825,17 @@ impl Renderer { panic!("unsupported pixel format") }; - let texture = self.cv_texture_cache.create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), - ptr::null(), - pixel_format, - source_size.x() as usize, - source_size.y() as usize, - 0, - ); + let texture = self + .cv_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + pixel_format, + source_size.x() as usize, + source_size.y() as usize, + 0, + ) + .unwrap(); align_offset(offset); let next_offset = *offset + mem::size_of::(); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index c2ca04abfa4f18a176df7aa7cb6e5409e19ce0b7..8773fbbe63eabdfaa83972573287525cae769aa7 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -8,6 +8,7 @@ path = "src/media.rs" doctest = false [dependencies] +anyhow = "1.0" block = "0.1" core-foundation = "0.9.3" foreign-types = "0.3" diff --git a/crates/media/build.rs b/crates/media/build.rs index 3446311d8939c463c1e42576c1e1bf41a317ee08..50834ebe7f9e1e389a656c8de77af6ee06c3d456 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -17,6 +17,7 @@ fn main() { .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_var("kCVPixelFormatType_.*") + .allowlist_var("kCVReturn.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index d5ba4eccd695c9305ed206963af5493d7480d080..4e19d2701d7b0c74214784b9487af515eef062a9 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1 +1,2 @@ -#import \ No newline at end of file +#import +#import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 5db184f3b289b09f107e9a7a5a1f5da336ad1bc0..b313f5f8fd00f1de2964a8b46b936652e1f158b1 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -33,16 +33,16 @@ pub mod io_surface { pub mod core_video { #![allow(non_snake_case)] - use std::ptr; - use super::*; pub use crate::bindings::*; + use anyhow::{anyhow, Result}; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, }; use foreign_types::ForeignTypeRef; use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat, MTLTexture}; + use metal::{MTLDevice, MTLPixelFormat}; + use std::ptr; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -97,7 +97,7 @@ pub mod core_video { impl_CFTypeDescription!(CVMetalTextureCache); impl CVMetalTextureCache { - pub fn new(metal_device: *mut MTLDevice) -> Self { + pub fn new(metal_device: *mut MTLDevice) -> Result { unsafe { let mut this = ptr::null(); let result = CVMetalTextureCacheCreate( @@ -107,8 +107,11 @@ pub mod core_video { ptr::null_mut(), &mut this, ); - // TODO: Check result - CVMetalTextureCache::wrap_under_create_rule(this) + if result == kCVReturnSuccess { + Ok(CVMetalTextureCache::wrap_under_create_rule(this)) + } else { + Err(anyhow!("could not create texture cache, code: {}", result)) + } } } @@ -120,7 +123,7 @@ pub mod core_video { width: usize, height: usize, plane_index: usize, - ) -> CVMetalTexture { + ) -> Result { unsafe { let mut this = ptr::null(); let result = CVMetalTextureCacheCreateTextureFromImage( @@ -134,8 +137,11 @@ pub mod core_video { plane_index, &mut this, ); - // TODO: Check result - CVMetalTexture::wrap_under_create_rule(this) + if result == kCVReturnSuccess { + Ok(CVMetalTexture::wrap_under_create_rule(this)) + } else { + Err(anyhow!("could not create texture, code: {}", result)) + } } } } @@ -149,7 +155,7 @@ pub mod core_video { metal_device: *const MTLDevice, texture_attributes: CFDictionaryRef, cache_out: *mut CVMetalTextureCacheRef, - ) -> i32; // TODO: This should be a CVReturn enum + ) -> CVReturn; fn CVMetalTextureCacheCreateTextureFromImage( allocator: CFAllocatorRef, texture_cache: CVMetalTextureCacheRef, @@ -160,7 +166,7 @@ pub mod core_video { height: usize, plane_index: usize, texture_out: *mut CVMetalTextureRef, - ) -> i32; + ) -> CVReturn; } #[repr(C)] From 3f66dd678a134bcf85e5481acf0d5edc8d2692a6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:10:27 +0200 Subject: [PATCH 25/89] :fire: --- crates/media/src/media.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index b313f5f8fd00f1de2964a8b46b936652e1f158b1..42a6eabac735f00f45783200ab4628b502b4b9e0 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -7,11 +7,8 @@ use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; -use objc::runtime; use std::ffi::c_void; -pub type id = *mut runtime::Object; - pub mod io_surface { use super::*; From a02e388ea2adc12579b334b32e3d47dc1c4ac658 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 12:02:57 +0200 Subject: [PATCH 26/89] Honor scale factor when rendering --- crates/capture/src/main.rs | 4 ++-- crates/gpui/src/platform/mac/renderer.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index af67ed9da45a0b10ec1af0d2c1eb974e9fd66d0b..36efe369f920317c2b6684ddcb334faae08e4e15 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -123,8 +123,8 @@ impl ScreenCaptureView { // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; let config: id = msg_send![config, init]; - let _: () = msg_send![config, setWidth: display_width]; - let _: () = msg_send![config, setHeight: display_height]; + let _: () = msg_send![config, setWidth: display_width * 2]; + let _: () = msg_send![config, setHeight: display_height * 2]; let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 8bf512a1a7c2345a207b935a12631cac23483643..ad0915ed5bdb6a546ce7618c153581f5cb772440 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -816,7 +816,7 @@ impl Renderer { surface.image_buffer.width() as i32, surface.image_buffer.height() as i32, ); - let target_size = surface.bounds.size(); + let target_size = surface.bounds.size() * scale_factor; let pixel_format = if surface.image_buffer.pixel_format_type() == core_video::kCVPixelFormatType_32BGRA { From f621d290fe5891aa9f3d4021e1059ccac7038fe6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 14:33:58 +0200 Subject: [PATCH 27/89] Move `core_media` into `media` crate --- crates/capture/build.rs | 3 -- crates/capture/src/bindings.h | 1 - crates/capture/src/main.rs | 64 +++-------------------------------- crates/media/build.rs | 1 + crates/media/src/bindings.h | 1 + crates/media/src/media.rs | 61 ++++++++++++++++++++++++++++++++- 6 files changed, 67 insertions(+), 64 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 482e6487ceba34adf30434c8b47537c3f82c75e3..7adb758ddecda4f9a29f71732d881c88cc0ce8c1 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,9 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { - println!("cargo:rustc-link-lib=framework=CoreMedia"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( @@ -21,7 +19,6 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") - .allowlist_function("CMTimeMake") .allowlist_function("dispatch_queue_create") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h index 45ac30446bf515d084c95a9f327ed6b332952a39..5e3c5258919a5d2109486af9229bcad83ac2ade6 100644 --- a/crates/capture/src/bindings.h +++ b/crates/capture/src/bindings.h @@ -1,3 +1,2 @@ -#import #import #import diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 36efe369f920317c2b6684ddcb334faae08e4e15..0a05471f4f3322abfb4519032efdf8af892d2fd7 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -7,7 +7,6 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger}, }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; -use core_media::{CMSampleBuffer, CMSampleBufferRef}; use futures::StreamExt; use gpui::{ actions, @@ -17,7 +16,10 @@ use gpui::{ Menu, MenuItem, ViewContext, }; use log::LevelFilter; -use media::core_video::{self, CVImageBuffer}; +use media::{ + core_media::{CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_video::{self, CVImageBuffer}, +}; use objc::{ class, declare::ClassDecl, @@ -125,7 +127,7 @@ impl ScreenCaptureView { let config: id = msg_send![config, init]; let _: () = msg_send![config, setWidth: display_width * 2]; let _: () = msg_send![config, setHeight: display_height * 2]; - let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; + let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; let _: () = msg_send![ @@ -238,59 +240,3 @@ extern "C" fn sample_output( fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } - -mod core_media { - #![allow(non_snake_case)] - - use crate::core_video::{CVImageBuffer, CVImageBufferRef}; - use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFTypeID, TCFType}, - declare_TCFType, - dictionary::CFDictionary, - impl_CFTypeDescription, impl_TCFType, - string::CFString, - }; - use std::ffi::c_void; - - #[repr(C)] - pub struct __CMSampleBuffer(c_void); - // The ref type must be a pointer to the underlying struct. - pub type CMSampleBufferRef = *const __CMSampleBuffer; - - declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); - impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); - impl_CFTypeDescription!(CMSampleBuffer); - - impl CMSampleBuffer { - pub fn attachments(&self) -> Vec> { - unsafe { - let attachments = - CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); - CFArray::::wrap_under_get_rule(attachments) - .into_iter() - .map(|attachments| { - CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) - }) - .collect() - } - } - - pub fn image_buffer(&self) -> CVImageBuffer { - unsafe { - CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( - self.as_concrete_TypeRef(), - )) - } - } - } - - extern "C" { - fn CMSampleBufferGetTypeID() -> CFTypeID; - fn CMSampleBufferGetSampleAttachmentsArray( - buffer: CMSampleBufferRef, - create_if_necessary: bool, - ) -> CFArrayRef; - fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; - } -} diff --git a/crates/media/build.rs b/crates/media/build.rs index 50834ebe7f9e1e389a656c8de77af6ee06c3d456..0e610113d2234f2e6763fbe9bba4630e3448bcaf 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -16,6 +16,7 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") + .allowlist_function("CMTimeMake") .allowlist_var("kCVPixelFormatType_.*") .allowlist_var("kCVReturn.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index 4e19d2701d7b0c74214784b9487af515eef062a9..3a4689f708bf5c6dca344ac5090dfb95c7a8bf56 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1,2 +1,3 @@ +#import #import #import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 42a6eabac735f00f45783200ab4628b502b4b9e0..86ba8d393e91c0f4d0c174862579cdb980f8b4fb 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -31,7 +31,8 @@ pub mod core_video { #![allow(non_snake_case)] use super::*; - pub use crate::bindings::*; + pub use crate::bindings::kCVPixelFormatType_32BGRA; + use crate::bindings::{kCVReturnSuccess, CVReturn, OSType}; use anyhow::{anyhow, Result}; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, @@ -189,3 +190,61 @@ pub mod core_video { fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } } + +pub mod core_media { + #![allow(non_snake_case)] + + pub use crate::bindings::CMTimeMake; + use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use core_foundation::{ + array::{CFArray, CFArrayRef}, + base::{CFTypeID, TCFType}, + declare_TCFType, + dictionary::CFDictionary, + impl_CFTypeDescription, impl_TCFType, + string::CFString, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CMSampleBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMSampleBufferRef = *const __CMSampleBuffer; + + declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); + impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); + impl_CFTypeDescription!(CMSampleBuffer); + + impl CMSampleBuffer { + pub fn attachments(&self) -> Vec> { + unsafe { + let attachments = + CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); + CFArray::::wrap_under_get_rule(attachments) + .into_iter() + .map(|attachments| { + CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) + }) + .collect() + } + } + + pub fn image_buffer(&self) -> CVImageBuffer { + unsafe { + CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( + self.as_concrete_TypeRef(), + )) + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMSampleBufferGetTypeID() -> CFTypeID; + fn CMSampleBufferGetSampleAttachmentsArray( + buffer: CMSampleBufferRef, + create_if_necessary: bool, + ) -> CFArrayRef; + fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + } +} From 7054fa61f2678e6a3797bd96566835637d6a12ba Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 15:49:55 +0200 Subject: [PATCH 28/89] Start compressing captured frames as H264 --- crates/capture/src/main.rs | 19 +++- crates/media/build.rs | 7 ++ crates/media/src/bindings.h | 2 + crates/media/src/media.rs | 170 +++++++++++++++++++++++++++++++++++- 4 files changed, 195 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 0a05471f4f3322abfb4519032efdf8af892d2fd7..a727dff37cf0a25c9f91c7a95a299fd7e57322b1 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -17,8 +17,9 @@ use gpui::{ }; use log::LevelFilter; use media::{ - core_media::{CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_media::{kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, core_video::{self, CVImageBuffer}, + video_toolbox::VTCompressionSession, }; use objc::{ class, @@ -85,6 +86,14 @@ impl ScreenCaptureView { let display: id = displays.objectAtIndex(0); let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; + let compression_session = VTCompressionSession::new( + display_width, + display_height, + kCMVideoCodecType_H264, + None, + ptr::null(), + ) + .unwrap(); let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); @@ -114,7 +123,15 @@ impl ScreenCaptureView { return; } + let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); + compression_session + .encode_frame( + image_buffer.as_concrete_TypeRef(), + timing_info.presentationTimeStamp, + timing_info.duration, + ) + .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); diff --git a/crates/media/build.rs b/crates/media/build.rs index 0e610113d2234f2e6763fbe9bba4630e3448bcaf..22233be8f8da05fdc244eaf69817427b51a8a418 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -16,9 +16,16 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") + .allowlist_type("CMItemIndex") + .allowlist_type("CMSampleTimingInfo") + .allowlist_type("CMVideoCodecType") + .allowlist_type("VTEncodeInfoFlags") .allowlist_function("CMTimeMake") .allowlist_var("kCVPixelFormatType_.*") .allowlist_var("kCVReturn.*") + .allowlist_var("VTEncodeInfoFlags_.*") + .allowlist_var("kCMVideoCodecType_.*") + .allowlist_var("kCMTime.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index 3a4689f708bf5c6dca344ac5090dfb95c7a8bf56..4df283d0c199669b0ccecf9bd59254bdcd3c2ec3 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1,3 +1,5 @@ +#import #import #import #import +#import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 86ba8d393e91c0f4d0c174862579cdb980f8b4fb..6ff7799b49623fcff95add026ae63229cb1c7031 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -194,11 +194,15 @@ pub mod core_video { pub mod core_media { #![allow(non_snake_case)] - pub use crate::bindings::CMTimeMake; + pub use crate::bindings::{ + kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, CMSampleTimingInfo, CMTime, + CMTimeMake, CMVideoCodecType, + }; use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use anyhow::{anyhow, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, - base::{CFTypeID, TCFType}, + base::{CFTypeID, OSStatus, TCFType}, declare_TCFType, dictionary::CFDictionary, impl_CFTypeDescription, impl_TCFType, @@ -236,6 +240,27 @@ pub mod core_media { )) } } + + pub fn sample_timing_info(&self, index: usize) -> Result { + unsafe { + let mut timing_info = CMSampleTimingInfo { + duration: kCMTimeInvalid, + presentationTimeStamp: kCMTimeInvalid, + decodeTimeStamp: kCMTimeInvalid, + }; + let result = CMSampleBufferGetSampleTimingInfo( + self.as_concrete_TypeRef(), + index as CMItemIndex, + &mut timing_info, + ); + + if result == 0 { + Ok(timing_info) + } else { + Err(anyhow!("error getting sample timing info, code {}", result)) + } + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -246,5 +271,146 @@ pub mod core_media { create_if_necessary: bool, ) -> CFArrayRef; fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + fn CMSampleBufferGetSampleTimingInfo( + buffer: CMSampleBufferRef, + index: CMItemIndex, + timing_info_out: *mut CMSampleTimingInfo, + ) -> OSStatus; + } +} + +pub mod video_toolbox { + #![allow(non_snake_case)] + + use super::*; + use crate::{ + core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_video::CVImageBufferRef, + }; + use anyhow::{anyhow, Result}; + use bindings::VTEncodeInfoFlags; + use core_foundation::{ + base::OSStatus, + dictionary::{CFDictionary, CFDictionaryRef, CFMutableDictionary}, + mach_port::CFAllocatorRef, + }; + use std::ptr; + + #[repr(C)] + pub struct __VTCompressionSession(c_void); + // The ref type must be a pointer to the underlying struct. + pub type VTCompressionSessionRef = *const __VTCompressionSession; + + declare_TCFType!(VTCompressionSession, VTCompressionSessionRef); + impl_TCFType!( + VTCompressionSession, + VTCompressionSessionRef, + VTCompressionSessionGetTypeID + ); + impl_CFTypeDescription!(VTCompressionSession); + + impl VTCompressionSession { + pub fn new( + width: usize, + height: usize, + codec: CMVideoCodecType, + callback: VTCompressionOutputCallback, + callback_data: *const c_void, + ) -> Result { + unsafe { + let mut this = ptr::null(); + let result = VTCompressionSessionCreate( + ptr::null(), + width as i32, + height as i32, + codec, + ptr::null(), + ptr::null(), + ptr::null(), + Some(Self::output), + callback_data, + &mut this, + ); + + if result == 0 { + Ok(Self::wrap_under_create_rule(this)) + } else { + Err(anyhow!( + "error creating compression session, code {}", + result + )) + } + } + } + + extern "C" fn output( + outputCallbackRefCon: *mut c_void, + sourceFrameRefCon: *mut c_void, + status: OSStatus, + infoFlags: VTEncodeInfoFlags, + sampleBuffer: CMSampleBufferRef, + ) { + println!("YO!"); + } + + pub fn encode_frame( + &self, + buffer: CVImageBufferRef, + presentation_timestamp: CMTime, + duration: CMTime, + ) -> Result<()> { + unsafe { + let result = VTCompressionSessionEncodeFrame( + self.as_concrete_TypeRef(), + buffer, + presentation_timestamp, + duration, + ptr::null(), + ptr::null(), + ptr::null_mut(), + ); + if result == 0 { + Ok(()) + } else { + Err(anyhow!("error encoding frame, code {}", result)) + } + } + } + } + + type VTCompressionOutputCallback = Option< + unsafe extern "C" fn( + outputCallbackRefCon: *mut c_void, + sourceFrameRefCon: *mut c_void, + status: OSStatus, + infoFlags: VTEncodeInfoFlags, + sampleBuffer: CMSampleBufferRef, + ), + >; + + #[link(name = "VideoToolbox", kind = "framework")] + extern "C" { + fn VTCompressionSessionGetTypeID() -> CFTypeID; + fn VTCompressionSessionCreate( + allocator: CFAllocatorRef, + width: i32, + height: i32, + codec_type: CMVideoCodecType, + encoder_specification: CFDictionaryRef, + source_image_buffer_attributes: CFDictionaryRef, + compressed_data_allocator: CFAllocatorRef, + output_callback: VTCompressionOutputCallback, + output_callback_ref_con: *const c_void, + compression_session_out: *mut VTCompressionSessionRef, + ) -> OSStatus; + fn VTCompressionSessionEncodeFrame( + session: VTCompressionSessionRef, + image_buffer: CVImageBufferRef, + presentation_timestamp: CMTime, + duration: CMTime, + frame_properties: CFDictionaryRef, + source_frame_ref_con: *const c_void, + output_flags: *mut VTEncodeInfoFlags, + ) -> OSStatus; } } From 600029a9189dccd2f817c5934ecab061214eebf7 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 18:02:05 +0200 Subject: [PATCH 29/89] WIP: Start converting H264 samples to Annex-B NALs Co-Authored-By: Nathan Sobo --- crates/media/build.rs | 1 + crates/media/src/media.rs | 117 +++++++++++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/crates/media/build.rs b/crates/media/build.rs index 22233be8f8da05fdc244eaf69817427b51a8a418..1ee8f23e41e97c9e9f24fd1eb163242256c1407c 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -26,6 +26,7 @@ fn main() { .allowlist_var("VTEncodeInfoFlags_.*") .allowlist_var("kCMVideoCodecType_.*") .allowlist_var("kCMTime.*") + .allowlist_var("kCMSampleAttachmentKey_.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 6ff7799b49623fcff95add026ae63229cb1c7031..c6aa1e3d0a46674c0b59d8b7640989372a8de404 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -208,7 +208,7 @@ pub mod core_media { impl_CFTypeDescription, impl_TCFType, string::CFString, }; - use std::ffi::c_void; + use std::{ffi::c_void, ptr}; #[repr(C)] pub struct __CMSampleBuffer(c_void); @@ -261,6 +261,14 @@ pub mod core_media { } } } + + pub fn format_description(&self) -> CMFormatDescription { + unsafe { + CMFormatDescription::wrap_under_get_rule(CMSampleBufferGetFormatDescription( + self.as_concrete_TypeRef(), + )) + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -276,6 +284,71 @@ pub mod core_media { index: CMItemIndex, timing_info_out: *mut CMSampleTimingInfo, ) -> OSStatus; + fn CMSampleBufferGetFormatDescription(buffer: CMSampleBufferRef) -> CMFormatDescriptionRef; + } + + #[repr(C)] + pub struct __CMFormatDescription(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMFormatDescriptionRef = *const __CMFormatDescription; + + declare_TCFType!(CMFormatDescription, CMFormatDescriptionRef); + impl_TCFType!( + CMFormatDescription, + CMFormatDescriptionRef, + CMFormatDescriptionGetTypeID + ); + impl_CFTypeDescription!(CMFormatDescription); + + impl CMFormatDescription { + pub fn h264_parameter_set_count(&self) -> usize { + unsafe { + let mut count = 0; + let result = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + self.as_concrete_TypeRef(), + 0, + ptr::null_mut(), + ptr::null_mut(), + &mut count, + ptr::null_mut(), + ); + assert_eq!(result, 0); + count + } + } + + pub fn h264_parameter_set_at_index(&self, index: usize) -> Result<&[u8]> { + unsafe { + let mut bytes = ptr::null(); + let mut len = 0; + let result = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + self.as_concrete_TypeRef(), + index, + &mut bytes, + &mut len, + ptr::null_mut(), + ptr::null_mut(), + ); + if result == 0 { + Ok(std::slice::from_raw_parts(bytes, len)) + } else { + Err(anyhow!("error getting parameter set, code: {}", result)) + } + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMFormatDescriptionGetTypeID() -> CFTypeID; + fn CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + video_desc: CMFormatDescriptionRef, + parameter_set_index: usize, + parameter_set_pointer_out: *mut *const u8, + parameter_set_size_out: *mut usize, + parameter_set_count_out: *mut usize, + NALUnitHeaderLengthOut: *mut isize, + ) -> OSStatus; } } @@ -284,15 +357,17 @@ pub mod video_toolbox { use super::*; use crate::{ - core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_media::{CMSampleBuffer, CMSampleBufferRef, CMTime, CMVideoCodecType}, core_video::CVImageBufferRef, }; use anyhow::{anyhow, Result}; use bindings::VTEncodeInfoFlags; use core_foundation::{ base::OSStatus, - dictionary::{CFDictionary, CFDictionaryRef, CFMutableDictionary}, + dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, + number::{CFBooleanGetValue, CFBooleanRef}, + string::CFStringRef, }; use std::ptr; @@ -343,13 +418,39 @@ pub mod video_toolbox { } } - extern "C" fn output( - outputCallbackRefCon: *mut c_void, - sourceFrameRefCon: *mut c_void, + unsafe extern "C" fn output( + output_callback_ref_con: *mut c_void, + source_frame_ref_con: *mut c_void, status: OSStatus, - infoFlags: VTEncodeInfoFlags, - sampleBuffer: CMSampleBufferRef, + info_flags: VTEncodeInfoFlags, + sample_buffer: CMSampleBufferRef, ) { + if status != 0 { + println!("error encoding frame, code: {}", status); + return; + } + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + + let mut is_iframe = false; + let attachments = sample_buffer.attachments(); + if let Some(attachments) = attachments.first() { + is_iframe = attachments + .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) + .map_or(true, |not_sync| { + CFBooleanGetValue(*not_sync as CFBooleanRef) + }); + } + + const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + if is_iframe { + let format_description = sample_buffer.format_description(); + for ix in 0..format_description.h264_parameter_set_count() { + let parameter_set = format_description.h264_parameter_set_at_index(ix); + stream.extend(START_CODE); + stream.extend(parameter_set); + } + } + println!("YO!"); } From 047b5114f1158a35201558b2c3c2b12c5aa92b53 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 20:03:47 +0200 Subject: [PATCH 30/89] Start on a new, more abstract `CompressionSession` primitive Co-Authored-By: Nathan Sobo --- Cargo.lock | 7 +- crates/capture/Cargo.toml | 2 + crates/capture/src/compression_session.rs | 178 ++++++++++++++++++++++ crates/capture/src/main.rs | 55 +++++-- crates/media/Cargo.toml | 1 + crates/media/src/media.rs | 54 +------ 6 files changed, 236 insertions(+), 61 deletions(-) create mode 100644 crates/capture/src/compression_session.rs diff --git a/Cargo.lock b/Cargo.lock index 669fedf6592c4914762aabce8e2146d4acc5bb8b..6b34c02f0d191cbb5262755fe7dff2e7ad928f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,9 +663,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "bzip2-sys" @@ -754,8 +754,10 @@ dependencies = [ name = "capture" version = "0.1.0" dependencies = [ + "anyhow", "bindgen", "block", + "bytes", "cocoa", "core-foundation", "core-graphics", @@ -3036,6 +3038,7 @@ dependencies = [ "anyhow", "bindgen", "block", + "bytes", "core-foundation", "foreign-types", "metal", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 54b1d0990e697ab7dfccb7f266abd3f3ad515bc2..c2bee8312a3e7caf25a98b1d5a783c8b4874a545 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -12,7 +12,9 @@ identifier = "dev.zed.Capture" gpui = { path = "../gpui" } media = { path = "../media" } +anyhow = "1.0.38" block = "0.1" +bytes = "1.2" cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" diff --git a/crates/capture/src/compression_session.rs b/crates/capture/src/compression_session.rs new file mode 100644 index 0000000000000000000000000000000000000000..1a771d057b1afee4d45113a8ce49dd2908648d20 --- /dev/null +++ b/crates/capture/src/compression_session.rs @@ -0,0 +1,178 @@ +use anyhow::Result; +use core_foundation::base::{OSStatus, TCFType}; +use media::{ + core_media::{CMSampleBufferRef, CMSampleTimingInfo, CMVideoCodecType}, + core_video::CVImageBuffer, + video_toolbox::{VTCompressionSession, VTEncodeInfoFlags}, +}; +use std::ffi::c_void; + +pub struct CompressionSession { + session: VTCompressionSession, + output_callback: Box, +} + +impl + CompressionSession +{ + pub fn new(width: usize, height: usize, codec: CMVideoCodecType, callback: F) -> Result { + let callback = Box::new(callback); + let session = VTCompressionSession::new( + width, + height, + codec, + Some(Self::output_callback), + callback.as_ref() as *const _ as *const c_void, + )?; + Ok(Self { + session, + output_callback: callback, + }) + } + + pub fn encode_frame(&self, buffer: &CVImageBuffer, timing: CMSampleTimingInfo) -> Result<()> { + self.session.encode_frame( + buffer.as_concrete_TypeRef(), + timing.presentationTimeStamp, + timing.duration, + ) + } + + extern "C" fn output_callback( + output_callback_ref_con: *mut c_void, + _: *mut c_void, + status: OSStatus, + flags: VTEncodeInfoFlags, + sample_buffer: CMSampleBufferRef, + ) { + let callback = unsafe { &mut *(output_callback_ref_con as *mut F) }; + callback(status, flags, sample_buffer); + } +} + +// unsafe extern "C" fn output( +// output_callback_ref_con: *mut c_void, +// source_frame_ref_con: *mut c_void, +// status: OSStatus, +// info_flags: VTEncodeInfoFlags, +// sample_buffer: CMSampleBufferRef, +// ) { +// if status != 0 { +// println!("error encoding frame, code: {}", status); +// return; +// } +// let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + +// let mut is_iframe = false; +// let attachments = sample_buffer.attachments(); +// if let Some(attachments) = attachments.first() { +// is_iframe = attachments +// .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) +// .map_or(true, |not_sync| { +// CFBooleanGetValue(*not_sync as CFBooleanRef) +// }); +// } + +// const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; +// if is_iframe { +// let format_description = sample_buffer.format_description(); +// for ix in 0..format_description.h264_parameter_set_count() { +// let parameter_set = format_description.h264_parameter_set_at_index(ix); +// stream.extend(START_CODE); +// stream.extend(parameter_set); +// } +// } + +// println!("YO!"); +// } + +// static void videoFrameFinishedEncoding(void *outputCallbackRefCon, +// void *sourceFrameRefCon, +// OSStatus status, +// VTEncodeInfoFlags infoFlags, +// CMSampleBufferRef sampleBuffer) { +// // Check if there were any errors encoding +// if (status != noErr) { +// NSLog(@"Error encoding video, err=%lld", (int64_t)status); +// return; +// } + +// // In this example we will use a NSMutableData object to store the +// // elementary stream. +// NSMutableData *elementaryStream = [NSMutableData data]; + +// // Find out if the sample buffer contains an I-Frame. +// // If so we will write the SPS and PPS NAL units to the elementary stream. +// BOOL isIFrame = NO; +// CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); +// if (CFArrayGetCount(attachmentsArray)) { +// CFBooleanRef notSync; +// CFDictionaryRef dict = CFArrayGetValueAtIndex(attachmentsArray, 0); +// BOOL keyExists = CFDictionaryGetValueIfPresent(dict, +// kCMSampleAttachmentKey_NotSync, +// (const void **)¬Sync); +// // An I-Frame is a sync frame +// isIFrame = !keyExists || !CFBooleanGetValue(notSync); +// } + +// // This is the start code that we will write to +// // the elementary stream before every NAL unit +// static const size_t startCodeLength = 4; +// static const uint8_t startCode[] = {0x00, 0x00, 0x00, 0x01}; + +// // Write the SPS and PPS NAL units to the elementary stream before every I-Frame +// if (isIFrame) { +// CMFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); + +// // Find out how many parameter sets there are +// size_t numberOfParameterSets; +// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, +// 0, NULL, NULL, +// &numberOfParameterSets, +// NULL); + +// // Write each parameter set to the elementary stream +// for (int i = 0; i < numberOfParameterSets; i++) { +// const uint8_t *parameterSetPointer; +// size_t parameterSetLength; +// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, +// i, +// ¶meterSetPointer, +// ¶meterSetLength, +// NULL, NULL); + +// // Write the parameter set to the elementary stream +// [elementaryStream appendBytes:startCode length:startCodeLength]; +// [elementaryStream appendBytes:parameterSetPointer length:parameterSetLength]; +// } +// } + +// // Get a pointer to the raw AVCC NAL unit data in the sample buffer +// size_t blockBufferLength; +// uint8_t *bufferDataPointer = NULL; +// CMBlockBufferGetDataPointer(CMSampleBufferGetDataBuffer(sampleBuffer), +// 0, +// NULL, +// &blockBufferLength, +// (char **)&bufferDataPointer); + +// // Loop through all the NAL units in the block buffer +// // and write them to the elementary stream with +// // start codes instead of AVCC length headers +// size_t bufferOffset = 0; +// static const int AVCCHeaderLength = 4; +// while (bufferOffset < blockBufferLength - AVCCHeaderLength) { +// // Read the NAL unit length +// uint32_t NALUnitLength = 0; +// memcpy(&NALUnitLength, bufferDataPointer + bufferOffset, AVCCHeaderLength); +// // Convert the length value from Big-endian to Little-endian +// NALUnitLength = CFSwapInt32BigToHost(NALUnitLength); +// // Write start code to the elementary stream +// [elementaryStream appendBytes:startCode length:startCodeLength]; +// // Write the NAL unit without the AVCC length header to the elementary stream +// [elementaryStream appendBytes:bufferDataPointer + bufferOffset + AVCCHeaderLength +// length:NALUnitLength]; +// // Move to the next NAL unit in the block buffer +// bufferOffset += AVCCHeaderLength + NALUnitLength; +// } +// } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index a727dff37cf0a25c9f91c7a95a299fd7e57322b1..b1d8f11795f393f701de4b406a7dc9bf084ee9b3 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,12 +1,18 @@ mod bindings; +mod compression_session; -use crate::bindings::SCStreamOutputType; +use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; use block::ConcreteBlock; +use bytes::BytesMut; use cocoa::{ base::{id, nil, YES}, foundation::{NSArray, NSString, NSUInteger}, }; -use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; +use core_foundation::{ + base::TCFType, + number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, + string::CFStringRef, +}; use futures::StreamExt; use gpui::{ actions, @@ -17,7 +23,10 @@ use gpui::{ }; use log::LevelFilter; use media::{ - core_media::{kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_media::{ + kCMSampleAttachmentKey_NotSync, kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, + CMTimeMake, + }, core_video::{self, CVImageBuffer}, video_toolbox::VTCompressionSession, }; @@ -86,12 +95,40 @@ impl ScreenCaptureView { let display: id = displays.objectAtIndex(0); let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; - let compression_session = VTCompressionSession::new( + let mut compression_buffer = BytesMut::new(); + let compression_session = CompressionSession::new( display_width, display_height, kCMVideoCodecType_H264, - None, - ptr::null(), + move |status, flags, sample_buffer| { + if status != 0 { + println!("error encoding frame, code: {}", status); + return; + } + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + + let mut is_iframe = false; + let attachments = sample_buffer.attachments(); + if let Some(attachments) = attachments.first() { + is_iframe = attachments + .find(kCMSampleAttachmentKey_NotSync as CFStringRef) + .map_or(true, |not_sync| { + CFBooleanGetValue(*not_sync as CFBooleanRef) + }); + } + + const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + if is_iframe { + let format_description = sample_buffer.format_description(); + for ix in 0..format_description.h264_parameter_set_count() { + let parameter_set = + format_description.h264_parameter_set_at_index(ix).unwrap(); + compression_buffer.extend_from_slice(&START_CODE); + compression_buffer.extend_from_slice(parameter_set); + let nal_unit = compression_buffer.split(); + } + } + }, ) .unwrap(); @@ -126,11 +163,7 @@ impl ScreenCaptureView { let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); compression_session - .encode_frame( - image_buffer.as_concrete_TypeRef(), - timing_info.presentationTimeStamp, - timing_info.duration, - ) + .encode_frame(&image_buffer, timing_info) .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 8773fbbe63eabdfaa83972573287525cae769aa7..aad2b74c021273afa9abef6479ecb1fc9ed3829a 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] anyhow = "1.0" block = "0.1" +bytes = "1.2" core-foundation = "0.9.3" foreign-types = "0.3" metal = "0.21.0" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index c6aa1e3d0a46674c0b59d8b7640989372a8de404..5c5f43af9d539801e4e0baf0572259716d0be6c5 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -195,8 +195,8 @@ pub mod core_media { #![allow(non_snake_case)] pub use crate::bindings::{ - kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, CMSampleTimingInfo, CMTime, - CMTimeMake, CMVideoCodecType, + kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, + CMSampleTimingInfo, CMTime, CMTimeMake, CMVideoCodecType, }; use crate::core_video::{CVImageBuffer, CVImageBufferRef}; use anyhow::{anyhow, Result}; @@ -357,18 +357,12 @@ pub mod video_toolbox { use super::*; use crate::{ - core_media::{CMSampleBuffer, CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, core_video::CVImageBufferRef, }; use anyhow::{anyhow, Result}; - use bindings::VTEncodeInfoFlags; - use core_foundation::{ - base::OSStatus, - dictionary::CFDictionaryRef, - mach_port::CFAllocatorRef, - number::{CFBooleanGetValue, CFBooleanRef}, - string::CFStringRef, - }; + pub use bindings::VTEncodeInfoFlags; + use core_foundation::{base::OSStatus, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef}; use std::ptr; #[repr(C)] @@ -402,7 +396,7 @@ pub mod video_toolbox { ptr::null(), ptr::null(), ptr::null(), - Some(Self::output), + callback, callback_data, &mut this, ); @@ -418,42 +412,6 @@ pub mod video_toolbox { } } - unsafe extern "C" fn output( - output_callback_ref_con: *mut c_void, - source_frame_ref_con: *mut c_void, - status: OSStatus, - info_flags: VTEncodeInfoFlags, - sample_buffer: CMSampleBufferRef, - ) { - if status != 0 { - println!("error encoding frame, code: {}", status); - return; - } - let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - - let mut is_iframe = false; - let attachments = sample_buffer.attachments(); - if let Some(attachments) = attachments.first() { - is_iframe = attachments - .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) - .map_or(true, |not_sync| { - CFBooleanGetValue(*not_sync as CFBooleanRef) - }); - } - - const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - if is_iframe { - let format_description = sample_buffer.format_description(); - for ix in 0..format_description.h264_parameter_set_count() { - let parameter_set = format_description.h264_parameter_set_at_index(ix); - stream.extend(START_CODE); - stream.extend(parameter_set); - } - } - - println!("YO!"); - } - pub fn encode_frame( &self, buffer: CVImageBufferRef, From b51abc5a639883de5416f260cc146012068791d2 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 31 Aug 2022 13:36:29 -0600 Subject: [PATCH 31/89] Read the frame data out of the CMSampleBuffer Still not sending it anywhere, but think I'm reading it correctly. --- Cargo.lock | 1 + crates/capture/Cargo.toml | 5 +--- crates/capture/script/capture | 5 ---- crates/capture/src/main.rs | 20 +++++++++++++++ crates/media/src/media.rs | 48 ++++++++++++++++++++++++++++++++++- 5 files changed, 69 insertions(+), 10 deletions(-) delete mode 100755 crates/capture/script/capture diff --git a/Cargo.lock b/Cargo.lock index 6b34c02f0d191cbb5262755fe7dff2e7ad928f4c..60e54aeb9f3173f2de0ac898c99a9cf3f6909299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,6 +757,7 @@ dependencies = [ "anyhow", "bindgen", "block", + "byteorder", "bytes", "cocoa", "core-foundation", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index c2bee8312a3e7caf25a98b1d5a783c8b4874a545..cdae75fc2b3d3d648c6f72bc4bad1f5dac442b44 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -4,10 +4,6 @@ version = "0.1.0" edition = "2021" description = "An example of screen capture" -[package.metadata.bundle] -name = "Capture" -identifier = "dev.zed.Capture" - [dependencies] gpui = { path = "../gpui" } media = { path = "../media" } @@ -15,6 +11,7 @@ media = { path = "../media" } anyhow = "1.0.38" block = "0.1" bytes = "1.2" +byteorder = "1.4" cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" diff --git a/crates/capture/script/capture b/crates/capture/script/capture deleted file mode 100755 index 0e6cf1f409ad0f2aa0eae0a6aefc86c2fe203416..0000000000000000000000000000000000000000 --- a/crates/capture/script/capture +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cargo bundle -TTY=`tty` -open ../../target/debug/bundle/osx/Capture.app --stdout $TTY --stderr $TTY diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index b1d8f11795f393f701de4b406a7dc9bf084ee9b3..97a5557c66dafa2b79fd1f20d6761e78539bc74a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -3,6 +3,7 @@ mod compression_session; use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; use block::ConcreteBlock; +use byteorder::{BigEndian, ReadBytesExt}; use bytes::BytesMut; use cocoa::{ base::{id, nil, YES}, @@ -128,6 +129,25 @@ impl ScreenCaptureView { let nal_unit = compression_buffer.split(); } } + + let data = sample_buffer.data(); + let mut data = data.bytes(); + + const AVCC_HEADER_LENGTH: usize = 4; + while data.len() - AVCC_HEADER_LENGTH > 0 { + let nal_unit_len = match data.read_u32::() { + Ok(len) => len as usize, + Err(error) => { + log::error!("error decoding nal unit length: {}", error); + return; + } + }; + compression_buffer.extend_from_slice(&START_CODE); + compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); + data = &data[nal_unit_len..]; + + let nal_unit = compression_buffer.split(); + } }, ) .unwrap(); diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 5c5f43af9d539801e4e0baf0572259716d0be6c5..ebe3ef7f4d2daa2d6e59be5f9fa3af5428652f4f 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -269,6 +269,14 @@ pub mod core_media { )) } } + + pub fn data(&self) -> CMBlockBuffer { + unsafe { + CMBlockBuffer::wrap_under_get_rule(CMSampleBufferGetDataBuffer( + self.as_concrete_TypeRef(), + )) + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -285,11 +293,11 @@ pub mod core_media { timing_info_out: *mut CMSampleTimingInfo, ) -> OSStatus; fn CMSampleBufferGetFormatDescription(buffer: CMSampleBufferRef) -> CMFormatDescriptionRef; + fn CMSampleBufferGetDataBuffer(sample_buffer: CMSampleBufferRef) -> CMBlockBufferRef; } #[repr(C)] pub struct __CMFormatDescription(c_void); - // The ref type must be a pointer to the underlying struct. pub type CMFormatDescriptionRef = *const __CMFormatDescription; declare_TCFType!(CMFormatDescription, CMFormatDescriptionRef); @@ -350,6 +358,44 @@ pub mod core_media { NALUnitHeaderLengthOut: *mut isize, ) -> OSStatus; } + + #[repr(C)] + pub struct __CMBlockBuffer(c_void); + pub type CMBlockBufferRef = *const __CMBlockBuffer; + + declare_TCFType!(CMBlockBuffer, CMBlockBufferRef); + impl_TCFType!(CMBlockBuffer, CMBlockBufferRef, CMBlockBufferGetTypeID); + impl_CFTypeDescription!(CMBlockBuffer); + + impl CMBlockBuffer { + pub fn bytes(&self) -> &[u8] { + unsafe { + let mut bytes = ptr::null(); + let mut len = 0; + let result = CMBlockBufferGetDataPointer( + self.as_concrete_TypeRef(), + 0, + &mut 0, + &mut len, + &mut bytes, + ); + assert!(result == 0, "could not get block buffer data"); + std::slice::from_raw_parts(bytes, len) + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMBlockBufferGetTypeID() -> CFTypeID; + fn CMBlockBufferGetDataPointer( + buffer: CMBlockBufferRef, + offset: usize, + length_at_offset_out: *mut usize, + total_length_out: *mut usize, + data_pointer_out: *mut *const u8, + ) -> OSStatus; + } } pub mod video_toolbox { From 6c28b2172847fdb849b565655c0151fefe2c0eec Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 1 Sep 2022 17:52:12 +0200 Subject: [PATCH 32/89] WIP: Start binding LiveKit --- LiveKitObjC/LKRoom.m | 28 ++ LiveKitObjC/LiveKitObjC-Bridging-Header.h | 3 + .../LiveKitObjC.xcodeproj/project.pbxproj | 342 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + .../xcshareddata/swiftpm/Package.resolved | 50 +++ LiveKitObjC/Room.swift | 23 ++ crates/capture/build.rs | 2 + crates/capture/src/main.rs | 116 +++--- 9 files changed, 522 insertions(+), 57 deletions(-) create mode 100644 LiveKitObjC/LKRoom.m create mode 100644 LiveKitObjC/LiveKitObjC-Bridging-Header.h create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved create mode 100644 LiveKitObjC/Room.swift diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m new file mode 100644 index 0000000000000000000000000000000000000000..ea4d7d44029ed12086e10931233f318c77d79fbf --- /dev/null +++ b/LiveKitObjC/LKRoom.m @@ -0,0 +1,28 @@ +// +// LKRoom.m +// LiveKitObjC +// +// Created by Antonio Scandurra on 01/09/22. +// + +#import +#import + +@interface LKRoom: NSObject { +} + +@property (nonatomic, retain) SLKRoom* room; +@end + +@implementation LKRoom +-(id)init { + if (self = [super init]) { + self.room = [[SLKRoom alloc] init]; + } + return self; +} + +-(void)connectWithURL:(NSString *)url token:(NSString *)token callback:(void(^)(void))callback { + [self.room connectWithUrl:url token:token callback:callback]; +} +@end diff --git a/LiveKitObjC/LiveKitObjC-Bridging-Header.h b/LiveKitObjC/LiveKitObjC-Bridging-Header.h new file mode 100644 index 0000000000000000000000000000000000000000..e11d920b1208f882eefc99eab4a412153f464220 --- /dev/null +++ b/LiveKitObjC/LiveKitObjC-Bridging-Header.h @@ -0,0 +1,3 @@ +// +// Use this file to import your target's public headers that you would like to expose to Swift. +// diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj new file mode 100644 index 0000000000000000000000000000000000000000..93053cb68d41dd6ebf0c41aef98b1310988c01b3 --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -0,0 +1,342 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 55; + objects = { + +/* Begin PBXBuildFile section */ + AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; + AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; + AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; + AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBDD28C121E6001AD7BE /* LiveKit */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; + AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; + AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; + AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LKRoom.m; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + AFA4DBCB28C0F7F5001AD7BE /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, + AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + AFA4DBC428C0F7F5001AD7BE = { + isa = PBXGroup; + children = ( + AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */, + AFA4DBD828C0F87F001AD7BE /* Room.swift */, + AFA4DBCE28C0F7F5001AD7BE /* Products */, + AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, + ); + sourceTree = ""; + }; + AFA4DBCE28C0F7F5001AD7BE /* Products */ = { + isa = PBXGroup; + children = ( + AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + AFA4DBC928C0F7F5001AD7BE /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */ = { + isa = PBXNativeTarget; + buildConfigurationList = AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */; + buildPhases = ( + AFA4DBC928C0F7F5001AD7BE /* Headers */, + AFA4DBCA28C0F7F5001AD7BE /* Sources */, + AFA4DBCB28C0F7F5001AD7BE /* Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = LiveKitObjC; + packageProductDependencies = ( + AFA4DBD528C0F839001AD7BE /* LiveKit */, + AFA4DBDD28C121E6001AD7BE /* LiveKit */, + ); + productName = LiveKitObjC; + productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; + productType = "com.apple.product-type.library.static"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + AFA4DBC528C0F7F5001AD7BE /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1340; + TargetAttributes = { + AFA4DBCC28C0F7F5001AD7BE = { + CreatedOnToolsVersion = 13.4.1; + LastSwiftMigration = 1340; + }; + }; + }; + buildConfigurationList = AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */; + compatibilityVersion = "Xcode 13.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = AFA4DBC428C0F7F5001AD7BE; + packageReferences = ( + AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, + AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, + ); + productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + AFA4DBCA28C0F7F5001AD7BE /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, + AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + AFA4DBCF28C0F7F5001AD7BE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 12.3; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + AFA4DBD028C0F7F5001AD7BE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 12.3; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = macosx; + }; + name = Release; + }; + AFA4DBD228C0F7F5001AD7BE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = NO; + EXECUTABLE_PREFIX = lib; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/../Frameworks", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + AFA4DBD328C0F7F5001AD7BE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = NO; + EXECUTABLE_PREFIX = lib; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/../Frameworks", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AFA4DBCF28C0F7F5001AD7BE /* Debug */, + AFA4DBD028C0F7F5001AD7BE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AFA4DBD228C0F7F5001AD7BE /* Debug */, + AFA4DBD328C0F7F5001AD7BE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/livekit/client-sdk-swift"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 1.0.0; + }; + }; + AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/livekit/client-sdk-swift"; + requirement = { + branch = main; + kind = branch; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + AFA4DBD528C0F839001AD7BE /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; + AFA4DBDD28C121E6001AD7BE /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; +} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000000000000000000000000000000000..919434a6254f0e9651f402737811be6634a03e9c --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000000000000000000000000000000000000..18d981003d68d0546c4804ac2ff47dd97c6e7921 --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000000000000000000000000000000000000..904852142c50c1a09b50d68bb3a0f0d7fe8214fb --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,50 @@ +{ + "pins" : [ + { + "identity" : "client-sdk-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/client-sdk-swift", + "state" : { + "branch" : "main", + "revision" : "e3de0d06e14825a7c9f1204f18f66898afb56cd6" + } + }, + { + "identity" : "promises", + "kind" : "remoteSourceControl", + "location" : "https://github.com/google/promises.git", + "state" : { + "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version" : "2.1.1" + } + }, + { + "identity" : "specs", + "kind" : "remoteSourceControl", + "location" : "https://github.com/webrtc-sdk/Specs.git", + "state" : { + "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version" : "104.5112.2" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version" : "1.4.4" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version" : "1.20.1" + } + } + ], + "version" : 2 +} diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift new file mode 100644 index 0000000000000000000000000000000000000000..1019dc78941d68245f8ab10e7c1533ee8c90c098 --- /dev/null +++ b/LiveKitObjC/Room.swift @@ -0,0 +1,23 @@ +// +// LKRoom.swift +// LiveKitObjC +// +// Created by Antonio Scandurra on 01/09/22. +// + +import Foundation +import LiveKit + +@objc public class SLKRoom: NSObject, RoomDelegate { + lazy var room = Room(delegate: self) + + @objc public func connect( + url: String, + token: String, + callback: @convention(block) @escaping () -> Void + ) { + self.room.connect(url, token).then { room in + callback() + } + } +} diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 7adb758ddecda4f9a29f71732d881c88cc0ce8c1..e88423bed1ebf34319e02436037b9b2ad9e035f9 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,8 @@ use std::{env, path::PathBuf, process::Command}; fn main() { + println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); + println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 97a5557c66dafa2b79fd1f20d6761e78539bc74a..e456b032eacae3adb1b09a035aafe8c4c4c477a2 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -48,6 +48,8 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); fn main() { + class!(LKRoom); + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::App::new(()).unwrap().run(|cx| { @@ -97,60 +99,60 @@ impl ScreenCaptureView { let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; let mut compression_buffer = BytesMut::new(); - let compression_session = CompressionSession::new( - display_width, - display_height, - kCMVideoCodecType_H264, - move |status, flags, sample_buffer| { - if status != 0 { - println!("error encoding frame, code: {}", status); - return; - } - let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - - let mut is_iframe = false; - let attachments = sample_buffer.attachments(); - if let Some(attachments) = attachments.first() { - is_iframe = attachments - .find(kCMSampleAttachmentKey_NotSync as CFStringRef) - .map_or(true, |not_sync| { - CFBooleanGetValue(*not_sync as CFBooleanRef) - }); - } - - const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - if is_iframe { - let format_description = sample_buffer.format_description(); - for ix in 0..format_description.h264_parameter_set_count() { - let parameter_set = - format_description.h264_parameter_set_at_index(ix).unwrap(); - compression_buffer.extend_from_slice(&START_CODE); - compression_buffer.extend_from_slice(parameter_set); - let nal_unit = compression_buffer.split(); - } - } - - let data = sample_buffer.data(); - let mut data = data.bytes(); - - const AVCC_HEADER_LENGTH: usize = 4; - while data.len() - AVCC_HEADER_LENGTH > 0 { - let nal_unit_len = match data.read_u32::() { - Ok(len) => len as usize, - Err(error) => { - log::error!("error decoding nal unit length: {}", error); - return; - } - }; - compression_buffer.extend_from_slice(&START_CODE); - compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); - data = &data[nal_unit_len..]; - - let nal_unit = compression_buffer.split(); - } - }, - ) - .unwrap(); + // let compression_session = CompressionSession::new( + // display_width, + // display_height, + // kCMVideoCodecType_H264, + // move |status, flags, sample_buffer| { + // if status != 0 { + // println!("error encoding frame, code: {}", status); + // return; + // } + // let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + + // let mut is_iframe = false; + // let attachments = sample_buffer.attachments(); + // if let Some(attachments) = attachments.first() { + // is_iframe = attachments + // .find(kCMSampleAttachmentKey_NotSync as CFStringRef) + // .map_or(true, |not_sync| { + // CFBooleanGetValue(*not_sync as CFBooleanRef) + // }); + // } + + // const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + // if is_iframe { + // let format_description = sample_buffer.format_description(); + // for ix in 0..format_description.h264_parameter_set_count() { + // let parameter_set = + // format_description.h264_parameter_set_at_index(ix).unwrap(); + // compression_buffer.extend_from_slice(&START_CODE); + // compression_buffer.extend_from_slice(parameter_set); + // let nal_unit = compression_buffer.split(); + // } + // } + + // let data = sample_buffer.data(); + // let mut data = data.bytes(); + + // const AVCC_HEADER_LENGTH: usize = 4; + // while data.len() - AVCC_HEADER_LENGTH > 0 { + // let nal_unit_len = match data.read_u32::() { + // Ok(len) => len as usize, + // Err(error) => { + // log::error!("error decoding nal unit length: {}", error); + // return; + // } + // }; + // compression_buffer.extend_from_slice(&START_CODE); + // compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); + // data = &data[nal_unit_len..]; + + // let nal_unit = compression_buffer.split(); + // } + // }, + // ) + // .unwrap(); let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); @@ -182,9 +184,9 @@ impl ScreenCaptureView { let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); - compression_session - .encode_frame(&image_buffer, timing_info) - .unwrap(); + // compression_session + // .encode_frame(&image_buffer, timing_info) + // .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); From 46abb1cbf601ab70ccd972c3f8e9f20a06aa1c22 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 1 Sep 2022 19:55:02 +0200 Subject: [PATCH 33/89] WIP --- LiveKitObjC/LKRoom.m | 4 ++++ .../LiveKitObjC.xcodeproj/project.pbxproj | 23 +++++-------------- .../xcshareddata/swiftpm/Package.resolved | 4 ++-- crates/capture/build.rs | 1 + crates/capture/src/main.rs | 8 ++++++- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m index ea4d7d44029ed12086e10931233f318c77d79fbf..02fc01d2f4ba1642f9fc0891ea13f03da30f8c54 100644 --- a/LiveKitObjC/LKRoom.m +++ b/LiveKitObjC/LKRoom.m @@ -26,3 +26,7 @@ [self.room connectWithUrl:url token:token callback:callback]; } @end + +LKRoom* BuildLKRoom() { + return [[LKRoom alloc] init]; +} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index 93053cb68d41dd6ebf0c41aef98b1310988c01b3..9c7a6136b35fa0db571578ff90b0a7fce9ded7a2 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -10,7 +10,6 @@ AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; - AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBDD28C121E6001AD7BE /* LiveKit */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -26,7 +25,6 @@ buildActionMask = 2147483647; files = ( AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, - AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -79,7 +77,6 @@ name = LiveKitObjC; packageProductDependencies = ( AFA4DBD528C0F839001AD7BE /* LiveKit */, - AFA4DBDD28C121E6001AD7BE /* LiveKit */, ); productName = LiveKitObjC; productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; @@ -111,7 +108,6 @@ mainGroup = AFA4DBC428C0F7F5001AD7BE; packageReferences = ( AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, - AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, ); productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; projectDirPath = ""; @@ -247,10 +243,13 @@ AFA4DBD228C0F7F5001AD7BE /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; + KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", @@ -267,10 +266,13 @@ AFA4DBD328C0F7F5001AD7BE /* Release */ = { isa = XCBuildConfiguration; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; + KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", @@ -315,14 +317,6 @@ minimumVersion = 1.0.0; }; }; - AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/livekit/client-sdk-swift"; - requirement = { - branch = main; - kind = branch; - }; - }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -331,11 +325,6 @@ package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; productName = LiveKit; }; - AFA4DBDD28C121E6001AD7BE /* LiveKit */ = { - isa = XCSwiftPackageProductDependency; - package = AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; - productName = LiveKit; - }; /* End XCSwiftPackageProductDependency section */ }; rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 904852142c50c1a09b50d68bb3a0f0d7fe8214fb..21468123fbe00e45f20a852757d01279e5120d13 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "branch" : "main", - "revision" : "e3de0d06e14825a7c9f1204f18f66898afb56cd6" + "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", + "version" : "1.0.4" } }, { diff --git a/crates/capture/build.rs b/crates/capture/build.rs index e88423bed1ebf34319e02436037b9b2ad9e035f9..630cf1ed6b3bcbcc0de248bc3f0a646e57d81917 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { + println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e456b032eacae3adb1b09a035aafe8c4c4c477a2..6a4c53aede6d8bfb02f4784f5aaca3fa8468511a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -47,8 +47,14 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); +extern "C" { + fn BuildLKRoom() -> *const c_void; +} + fn main() { - class!(LKRoom); + unsafe { + BuildLKRoom(); + } SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); From 5fec784580e6d43689cbfc7046f96149c71e499b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 2 Sep 2022 11:42:39 +0200 Subject: [PATCH 34/89] Link Swift stdlib --- Cargo.lock | 2 + .../LiveKitObjC.xcodeproj/project.pbxproj | 8 --- crates/capture/Cargo.toml | 2 + crates/capture/build.rs | 56 +++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60e54aeb9f3173f2de0ac898c99a9cf3f6909299..bf60a083674c709d0cf927b3b91b39a120893aed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,8 @@ dependencies = [ "objc", "parking_lot 0.11.2", "postage", + "serde", + "serde_json", "simplelog", ] diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index 9c7a6136b35fa0db571578ff90b0a7fce9ded7a2..db7445a28f200140914bc4d07bbac189b8b07e5d 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -243,11 +243,8 @@ AFA4DBD228C0F7F5001AD7BE /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; - BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( @@ -256,7 +253,6 @@ "@loader_path/../Frameworks", ); PRODUCT_NAME = "$(TARGET_NAME)"; - SKIP_INSTALL = YES; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; @@ -266,11 +262,8 @@ AFA4DBD328C0F7F5001AD7BE /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; - BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( @@ -279,7 +272,6 @@ "@loader_path/../Frameworks", ); PRODUCT_NAME = "$(TARGET_NAME)"; - SKIP_INSTALL = YES; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_VERSION = 5.0; }; diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index cdae75fc2b3d3d648c6f72bc4bad1f5dac442b44..339198f7642cd77192c8758dd84b71e063d2f23f 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -25,3 +25,5 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = { version = "1.0", features = ["preserve_order"] } \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 630cf1ed6b3bcbcc0de248bc3f0a646e57d81917..b16ed189607cf04513f3b51526e7fb68dfcc3257 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,62 @@ +use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftTargetInfo { + pub triple: String, + pub unversioned_triple: String, + pub module_triple: String, + pub swift_runtime_compatibility_version: String, + #[serde(rename = "librariesRequireRPath")] + pub libraries_require_rpath: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftPaths { + pub runtime_library_paths: Vec, + pub runtime_library_import_paths: Vec, + pub runtime_resource_path: String, +} + +#[derive(Debug, Deserialize)] +pub struct SwiftTarget { + pub target: SwiftTargetInfo, + pub paths: SwiftPaths, +} + +const MACOS_TARGET_VERSION: &str = "12"; + +pub fn link_swift_libs() { + let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + if arch == "aarch64" { + arch = "arm64".into(); + } + let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); + + let swift_target_info_str = Command::new("swift") + .args(&["-target", &target, "-print-target-info"]) + .output() + .unwrap() + .stdout; + + let swift_target_info: SwiftTarget = serde_json::from_slice(&swift_target_info_str).unwrap(); + if swift_target_info.target.libraries_require_rpath { + panic!("Libraries require RPath! Change minimum MacOS value to fix.") + } + + swift_target_info + .paths + .runtime_library_paths + .iter() + .for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + fn main() { + link_swift_libs(); println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); From 52f32b50b296fdb693896057510b31268ed6985a Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 2 Sep 2022 18:35:16 +0200 Subject: [PATCH 35/89] WIP: Talk to Swift via C without involving Objective-C Co-Authored-By: Nathan Sobo --- LiveKitObjC/LKRoom.m | 32 ------------------ .../LiveKitObjC.xcodeproj/project.pbxproj | 12 +++---- LiveKitObjC/Room.swift | 26 +++++++++++++-- crates/capture/build.rs | 4 ++- crates/capture/src/main.rs | 33 +++++++++++++++---- 5 files changed, 60 insertions(+), 47 deletions(-) delete mode 100644 LiveKitObjC/LKRoom.m diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m deleted file mode 100644 index 02fc01d2f4ba1642f9fc0891ea13f03da30f8c54..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LKRoom.m +++ /dev/null @@ -1,32 +0,0 @@ -// -// LKRoom.m -// LiveKitObjC -// -// Created by Antonio Scandurra on 01/09/22. -// - -#import -#import - -@interface LKRoom: NSObject { -} - -@property (nonatomic, retain) SLKRoom* room; -@end - -@implementation LKRoom --(id)init { - if (self = [super init]) { - self.room = [[SLKRoom alloc] init]; - } - return self; -} - --(void)connectWithURL:(NSString *)url token:(NSString *)token callback:(void(^)(void))callback { - [self.room connectWithUrl:url token:token callback:callback]; -} -@end - -LKRoom* BuildLKRoom() { - return [[LKRoom alloc] init]; -} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index db7445a28f200140914bc4d07bbac189b8b07e5d..c1293713ecd86888de0d9cd8708652f662e307d7 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -9,14 +9,12 @@ /* Begin PBXBuildFile section */ AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; - AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; - AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LKRoom.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -34,7 +32,6 @@ AFA4DBC428C0F7F5001AD7BE = { isa = PBXGroup; children = ( - AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */, AFA4DBD828C0F87F001AD7BE /* Room.swift */, AFA4DBCE28C0F7F5001AD7BE /* Products */, AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, @@ -124,7 +121,6 @@ buildActionMask = 2147483647; files = ( AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, - AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -245,13 +241,15 @@ buildSettings = { CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = YES; EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = NO; + KEEP_PRIVATE_EXTERNS = YES; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", "@loader_path/../Frameworks", ); + PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; @@ -264,13 +262,15 @@ buildSettings = { CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = YES; EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = NO; + KEEP_PRIVATE_EXTERNS = YES; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", "@loader_path/../Frameworks", ); + PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_VERSION = 5.0; diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift index 1019dc78941d68245f8ab10e7c1533ee8c90c098..9397b82649863ae3d72f8f24fcb84336c99c0cb8 100644 --- a/LiveKitObjC/Room.swift +++ b/LiveKitObjC/Room.swift @@ -8,10 +8,18 @@ import Foundation import LiveKit -@objc public class SLKRoom: NSObject, RoomDelegate { +public class LKRoom: RoomDelegate { lazy var room = Room(delegate: self) - @objc public func connect( + init() { + print("INIT!\n"); + } + + deinit { + print("DEINIT!\n"); + } + + public func connect( url: String, token: String, callback: @convention(block) @escaping () -> Void @@ -20,4 +28,18 @@ import LiveKit callback() } } + +} + + +@_cdecl("LKRoomCreate") +public func LKRoomCreate() -> UnsafeMutableRawPointer { + Unmanaged.passRetained(LKRoom()).toOpaque() +} + +@_cdecl("LKRoomDestroy") +public func LKRoomDestroy(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); } + + diff --git a/crates/capture/build.rs b/crates/capture/build.rs index b16ed189607cf04513f3b51526e7fb68dfcc3257..58cf51203a9b3da4010f6eb35a7210c1bf493797 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -58,9 +58,11 @@ pub fn link_swift_libs() { fn main() { link_swift_libs(); println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); - println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); + println!("cargo:rustc-link-search=native=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/libs"); + println!("cargo:rustc-link-search=framework=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/frameworks"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=WebRTC"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 6a4c53aede6d8bfb02f4784f5aaca3fa8468511a..f952f3ba22b5e7246c277e76dcf763874bc9a771 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -10,7 +10,7 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger}, }; use core_foundation::{ - base::TCFType, + base::{CFRelease, TCFType}, number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, string::CFStringRef, }; @@ -35,7 +35,7 @@ use objc::{ class, declare::ClassDecl, msg_send, - runtime::{Object, Sel}, + runtime::{Class, Object, Sel}, sel, sel_impl, }; use parking_lot::Mutex; @@ -48,13 +48,34 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); extern "C" { - fn BuildLKRoom() -> *const c_void; + fn LKRoomCreate() -> *const c_void; + fn LKRoomDestroy(ptr: *const c_void); } -fn main() { - unsafe { - BuildLKRoom(); +struct Room { + native_room: *const c_void, +} + +impl Room { + pub fn new() -> Self { + Self { + native_room: unsafe { LKRoomCreate() }, + } + } +} + +impl Drop for Room { + fn drop(&mut self) { + unsafe { LKRoomDestroy(self.native_room) } } +} + +fn main() { + println!("Creating room..."); + let room = Room::new(); + + println!("Dropping room..."); + drop(room); SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); From 4bcc008cbf4a9e87ddd069867dc7b01636d313c3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 2 Sep 2022 12:56:38 -0600 Subject: [PATCH 36/89] WIP: Start on live_kit crate that uses a C-based bridge --- Cargo.lock | 11 ++- crates/capture/Cargo.toml | 3 +- crates/capture/build.rs | 67 ++----------- crates/capture/src/main.rs | 24 +---- crates/live_kit/Cargo.toml | 15 +++ crates/live_kit/LiveKitBridge/.gitignore | 9 ++ .../live_kit/LiveKitBridge/Package.resolved | 50 ++++++++++ crates/live_kit/LiveKitBridge/Package.swift | 27 ++++++ crates/live_kit/LiveKitBridge/README.md | 3 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 12 +++ crates/live_kit/build.rs | 95 +++++++++++++++++++ crates/live_kit/src/live_kit.rs | 24 +++++ 12 files changed, 253 insertions(+), 87 deletions(-) create mode 100644 crates/live_kit/Cargo.toml create mode 100644 crates/live_kit/LiveKitBridge/.gitignore create mode 100644 crates/live_kit/LiveKitBridge/Package.resolved create mode 100644 crates/live_kit/LiveKitBridge/Package.swift create mode 100644 crates/live_kit/LiveKitBridge/README.md create mode 100644 crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift create mode 100644 crates/live_kit/build.rs create mode 100644 crates/live_kit/src/live_kit.rs diff --git a/Cargo.lock b/Cargo.lock index bf60a083674c709d0cf927b3b91b39a120893aed..1e176cda4206c39775f73b74b2a03ec592ab7336 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,13 +765,12 @@ dependencies = [ "foreign-types", "futures", "gpui", + "live_kit", "log", "media", "objc", "parking_lot 0.11.2", "postage", - "serde", - "serde_json", "simplelog", ] @@ -2918,6 +2917,14 @@ dependencies = [ "rand_chacha 0.3.1", ] +[[package]] +name = "live_kit" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "lock_api" version = "0.4.7" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 339198f7642cd77192c8758dd84b71e063d2f23f..d572c6077ce1aa54dcb61c9521af88cafd7c538c 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -6,6 +6,7 @@ description = "An example of screen capture" [dependencies] gpui = { path = "../gpui" } +live_kit = { path = "../live_kit" } media = { path = "../media" } anyhow = "1.0.38" @@ -25,5 +26,3 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" -serde = { version = "1.0", features = ["derive", "rc"] } -serde_json = { version = "1.0", features = ["preserve_order"] } \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 58cf51203a9b3da4010f6eb35a7210c1bf493797..e2ffda4c53f5f1602def95fa03ea4fffcde79bae 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,68 +1,15 @@ -use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftTargetInfo { - pub triple: String, - pub unversioned_triple: String, - pub module_triple: String, - pub swift_runtime_compatibility_version: String, - #[serde(rename = "librariesRequireRPath")] - pub libraries_require_rpath: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftPaths { - pub runtime_library_paths: Vec, - pub runtime_library_import_paths: Vec, - pub runtime_resource_path: String, -} - -#[derive(Debug, Deserialize)] -pub struct SwiftTarget { - pub target: SwiftTargetInfo, - pub paths: SwiftPaths, -} - -const MACOS_TARGET_VERSION: &str = "12"; - -pub fn link_swift_libs() { - let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "aarch64" { - arch = "arm64".into(); - } - let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); - - let swift_target_info_str = Command::new("swift") - .args(&["-target", &target, "-print-target-info"]) - .output() - .unwrap() - .stdout; - - let swift_target_info: SwiftTarget = serde_json::from_slice(&swift_target_info_str).unwrap(); - if swift_target_info.target.libraries_require_rpath { - panic!("Libraries require RPath! Change minimum MacOS value to fix.") - } +fn main() { + println!( + "cargo:rustc-link-search=framework={}", + "crates/live_kit/LiveKitBridge/.build/arm64-apple-macosx/debug" + ); - swift_target_info - .paths - .runtime_library_paths - .iter() - .for_each(|path| { - println!("cargo:rustc-link-search=native={}", path); - }); -} + // Find frameworks as a sibling of the executable at runtime + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); -fn main() { - link_swift_libs(); - println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); - println!("cargo:rustc-link-search=native=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/libs"); - println!("cargo:rustc-link-search=framework=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/frameworks"); - println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-link-lib=framework=WebRTC"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index f952f3ba22b5e7246c277e76dcf763874bc9a771..75df9be3689a45a818714b0d2ba1d14ebe49157a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -22,6 +22,7 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; +use live_kit::Room; use log::LevelFilter; use media::{ core_media::{ @@ -47,29 +48,6 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); -extern "C" { - fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(ptr: *const c_void); -} - -struct Room { - native_room: *const c_void, -} - -impl Room { - pub fn new() -> Self { - Self { - native_room: unsafe { LKRoomCreate() }, - } - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { LKRoomDestroy(self.native_room) } - } -} - fn main() { println!("Creating room..."); let room = Room::new(); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..15f768345672074e78dffed37a8e0892cbc61972 --- /dev/null +++ b/crates/live_kit/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "live_kit" +version = "0.1.0" +edition = "2021" +description = "Bindings to LiveKit Swift client SDK" + +[lib] +path = "src/live_kit.rs" +doctest = false + +[dependencies] + +[build-dependencies] +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = { version = "1.0", features = ["preserve_order"] } diff --git a/crates/live_kit/LiveKitBridge/.gitignore b/crates/live_kit/LiveKitBridge/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..3b29812086f28a2b21884e57ead495ffd9434178 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +/*.xcodeproj +xcuserdata/ +DerivedData/ +.swiftpm/config/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved new file mode 100644 index 0000000000000000000000000000000000000000..df0b5d6243f553c6be159e388bf60dc4e46406b5 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -0,0 +1,50 @@ +{ + "pins" : [ + { + "identity" : "client-sdk-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/client-sdk-swift.git", + "state" : { + "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", + "version" : "1.0.4" + } + }, + { + "identity" : "promises", + "kind" : "remoteSourceControl", + "location" : "https://github.com/google/promises.git", + "state" : { + "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version" : "2.1.1" + } + }, + { + "identity" : "specs", + "kind" : "remoteSourceControl", + "location" : "https://github.com/webrtc-sdk/Specs.git", + "state" : { + "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version" : "104.5112.2" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version" : "1.4.4" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version" : "1.20.1" + } + } + ], + "version" : 2 +} diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift new file mode 100644 index 0000000000000000000000000000000000000000..9e7039998d495f73cd78325813ed05e6f41ba38b --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -0,0 +1,27 @@ +// swift-tools-version: 5.6 + +import PackageDescription + +let package = Package( + name: "LiveKitBridge", + platforms: [ + .macOS(.v10_15) + ], + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "LiveKitBridge", + type: .static, + targets: ["LiveKitBridge"]), + ], + dependencies: [ + .package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "LiveKitBridge", + dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]), + ] +) diff --git a/crates/live_kit/LiveKitBridge/README.md b/crates/live_kit/LiveKitBridge/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b982c672866a3748a40a8b2a40795b1d2b048196 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/README.md @@ -0,0 +1,3 @@ +# LiveKitBridge + +A description of this package. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift new file mode 100644 index 0000000000000000000000000000000000000000..11df49e6c6b39263e1d340200177d24789a36800 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -0,0 +1,12 @@ +import Foundation +import LiveKit + +@_cdecl("LKRoomCreate") +public func LKRoomCreate() -> UnsafeMutableRawPointer { + Unmanaged.passRetained(Room()).toOpaque() +} + +@_cdecl("LKRoomDestroy") +public func LKRoomDestroy(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); +} diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..dab126d12cb6c47d240361472156765a8cad3805 --- /dev/null +++ b/crates/live_kit/build.rs @@ -0,0 +1,95 @@ +use serde::Deserialize; +use std::{env, path::PathBuf, process::Command}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftTargetInfo { + pub triple: String, + pub unversioned_triple: String, + pub module_triple: String, + pub swift_runtime_compatibility_version: String, + #[serde(rename = "librariesRequireRPath")] + pub libraries_require_rpath: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftPaths { + pub runtime_library_paths: Vec, + pub runtime_library_import_paths: Vec, + pub runtime_resource_path: String, +} + +#[derive(Debug, Deserialize)] +pub struct SwiftTarget { + pub target: SwiftTargetInfo, + pub paths: SwiftPaths, +} + +const MACOS_TARGET_VERSION: &str = "12"; + +fn main() { + build_bridge(); + link_swift_stdlib(); +} + +fn build_bridge() { + let profile = env::var("PROFILE").unwrap(); + let package_name = "LiveKitBridge"; + let package_root = env::current_dir().unwrap().join(package_name); + if !Command::new("swift") + .args(&["build", "-c", &profile]) + .current_dir(&package_root) + .status() + .unwrap() + .success() + { + panic!( + "Failed to compile swift package in {}", + package_root.display() + ); + } + + let swift_target_info = get_swift_target(); + let swift_out_dir_path = format!( + "{}/.build/{}/{}", + package_root.display(), + swift_target_info.target.unversioned_triple, + profile + ); + + println!("cargo:rustc-link-search=native={}", swift_out_dir_path); + println!( + "cargo:rustc-link-search=framework={}", + "/Users/nathan/src/zed/crates/live_kit/frameworks" + ); + println!("cargo:rustc-link-lib=static={}", package_name); + println!("cargo:rustc-link-lib=framework=WebRTC"); +} + +fn link_swift_stdlib() { + let target = get_swift_target(); + if target.target.libraries_require_rpath { + panic!("Libraries require RPath! Change minimum MacOS value to fix.") + } + + target.paths.runtime_library_paths.iter().for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + +fn get_swift_target() -> SwiftTarget { + let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + if arch == "aarch64" { + arch = "arm64".into(); + } + let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); + + let swift_target_info_str = Command::new("swift") + .args(&["-target", &target, "-print-target-info"]) + .output() + .unwrap() + .stdout; + + serde_json::from_slice(&swift_target_info_str).unwrap() +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs new file mode 100644 index 0000000000000000000000000000000000000000..3dcaaa74ed73c30ae4e1b105c053ebe9f8ea8a3d --- /dev/null +++ b/crates/live_kit/src/live_kit.rs @@ -0,0 +1,24 @@ +use std::ffi::c_void; + +extern "C" { + fn LKRoomCreate() -> *const c_void; + fn LKRoomDestroy(ptr: *const c_void); +} + +pub struct Room { + native_room: *const c_void, +} + +impl Room { + pub fn new() -> Self { + Self { + native_room: unsafe { LKRoomCreate() }, + } + } +} + +impl Drop for Room { + fn drop(&mut self) { + unsafe { LKRoomDestroy(self.native_room) } + } +} From 3c2566fc11ed3c575e0379f42deb5935c1d8f779 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 08:56:49 -0600 Subject: [PATCH 37/89] Copy WebRTC.framework when building livekit crate We determine the location of the target executable directory in a somewhat hacky way, but it seems reasonably stable. Co-Authored-By: Antonio Scandurra --- crates/capture/build.rs | 7 +--- crates/live_kit/build.rs | 88 +++++++++++++++++++++++++++------------- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index e2ffda4c53f5f1602def95fa03ea4fffcde79bae..c637f2ce9b11b5209049160cf01755a7530cd7ad 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,12 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { - println!( - "cargo:rustc-link-search=framework={}", - "crates/live_kit/LiveKitBridge/.build/arm64-apple-macosx/debug" - ); - - // Find frameworks as a sibling of the executable at runtime + // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index dab126d12cb6c47d240361472156765a8cad3805..ca9c2d5cb9fa662826a9230e757e7b53cd85e22d 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -29,53 +29,70 @@ pub struct SwiftTarget { const MACOS_TARGET_VERSION: &str = "12"; fn main() { - build_bridge(); - link_swift_stdlib(); + let swift_target = get_swift_target(); + + build_bridge(&swift_target); + link_swift_stdlib(&swift_target); + link_webrtc_framework(&swift_target); } -fn build_bridge() { - let profile = env::var("PROFILE").unwrap(); - let package_name = "LiveKitBridge"; - let package_root = env::current_dir().unwrap().join(package_name); +fn build_bridge(swift_target: &SwiftTarget) { + let swift_package_root = swift_package_root(); if !Command::new("swift") - .args(&["build", "-c", &profile]) - .current_dir(&package_root) + .args(&["build", "-c", &env::var("PROFILE").unwrap()]) + .current_dir(&swift_package_root) .status() .unwrap() .success() { panic!( "Failed to compile swift package in {}", - package_root.display() + swift_package_root.display() ); } - let swift_target_info = get_swift_target(); - let swift_out_dir_path = format!( - "{}/.build/{}/{}", - package_root.display(), - swift_target_info.target.unversioned_triple, - profile - ); - - println!("cargo:rustc-link-search=native={}", swift_out_dir_path); println!( - "cargo:rustc-link-search=framework={}", - "/Users/nathan/src/zed/crates/live_kit/frameworks" + "cargo:rustc-link-search=native={}", + swift_target.out_dir_path().display() ); - println!("cargo:rustc-link-lib=static={}", package_name); - println!("cargo:rustc-link-lib=framework=WebRTC"); + println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME); } -fn link_swift_stdlib() { - let target = get_swift_target(); - if target.target.libraries_require_rpath { +fn link_swift_stdlib(swift_target: &SwiftTarget) { + if swift_target.target.libraries_require_rpath { panic!("Libraries require RPath! Change minimum MacOS value to fix.") } - target.paths.runtime_library_paths.iter().for_each(|path| { - println!("cargo:rustc-link-search=native={}", path); - }); + swift_target + .paths + .runtime_library_paths + .iter() + .for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + +fn link_webrtc_framework(swift_target: &SwiftTarget) { + let swift_out_dir_path = swift_target.out_dir_path(); + println!("cargo:rustc-link-lib=framework=WebRTC"); + println!( + "cargo:rustc-link-search=framework={}", + swift_out_dir_path.display() + ); + + let source_path = swift_out_dir_path.join("WebRTC.framework"); + let target_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); + assert!( + Command::new("cp") + .arg("-r") + .args(&[&source_path, &target_path]) + .status() + .unwrap() + .success(), + "could not copy WebRTC.framework from {:?} to {:?}", + source_path, + target_path + ); } fn get_swift_target() -> SwiftTarget { @@ -93,3 +110,18 @@ fn get_swift_target() -> SwiftTarget { serde_json::from_slice(&swift_target_info_str).unwrap() } + +const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; + +fn swift_package_root() -> PathBuf { + env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) +} + +impl SwiftTarget { + fn out_dir_path(&self) -> PathBuf { + swift_package_root() + .join(".build") + .join(&self.target.unversioned_triple) + .join(env::var("PROFILE").unwrap()) + } +} From 45d83b557b77ee7cc7cd255aafbc333277ad82b3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 09:05:53 -0600 Subject: [PATCH 38/89] Remove LiveKitObjC experiment Co-Authored-By: Antonio Scandurra --- LiveKitObjC/LiveKitObjC-Bridging-Header.h | 3 - .../LiveKitObjC.xcodeproj/project.pbxproj | 323 ------------------ .../contents.xcworkspacedata | 7 - .../xcshareddata/IDEWorkspaceChecks.plist | 8 - .../xcshareddata/swiftpm/Package.resolved | 50 --- LiveKitObjC/Room.swift | 45 --- 6 files changed, 436 deletions(-) delete mode 100644 LiveKitObjC/LiveKitObjC-Bridging-Header.h delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved delete mode 100644 LiveKitObjC/Room.swift diff --git a/LiveKitObjC/LiveKitObjC-Bridging-Header.h b/LiveKitObjC/LiveKitObjC-Bridging-Header.h deleted file mode 100644 index e11d920b1208f882eefc99eab4a412153f464220..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LiveKitObjC-Bridging-Header.h +++ /dev/null @@ -1,3 +0,0 @@ -// -// Use this file to import your target's public headers that you would like to expose to Swift. -// diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj deleted file mode 100644 index c1293713ecd86888de0d9cd8708652f662e307d7..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ /dev/null @@ -1,323 +0,0 @@ -// !$*UTF8*$! -{ - archiveVersion = 1; - classes = { - }; - objectVersion = 55; - objects = { - -/* Begin PBXBuildFile section */ - AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; - AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; -/* End PBXBuildFile section */ - -/* Begin PBXFileReference section */ - AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; - AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; - AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; -/* End PBXFileReference section */ - -/* Begin PBXFrameworksBuildPhase section */ - AFA4DBCB28C0F7F5001AD7BE /* Frameworks */ = { - isa = PBXFrameworksBuildPhase; - buildActionMask = 2147483647; - files = ( - AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXFrameworksBuildPhase section */ - -/* Begin PBXGroup section */ - AFA4DBC428C0F7F5001AD7BE = { - isa = PBXGroup; - children = ( - AFA4DBD828C0F87F001AD7BE /* Room.swift */, - AFA4DBCE28C0F7F5001AD7BE /* Products */, - AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, - ); - sourceTree = ""; - }; - AFA4DBCE28C0F7F5001AD7BE /* Products */ = { - isa = PBXGroup; - children = ( - AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */, - ); - name = Products; - sourceTree = ""; - }; -/* End PBXGroup section */ - -/* Begin PBXHeadersBuildPhase section */ - AFA4DBC928C0F7F5001AD7BE /* Headers */ = { - isa = PBXHeadersBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXHeadersBuildPhase section */ - -/* Begin PBXNativeTarget section */ - AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */ = { - isa = PBXNativeTarget; - buildConfigurationList = AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */; - buildPhases = ( - AFA4DBC928C0F7F5001AD7BE /* Headers */, - AFA4DBCA28C0F7F5001AD7BE /* Sources */, - AFA4DBCB28C0F7F5001AD7BE /* Frameworks */, - ); - buildRules = ( - ); - dependencies = ( - ); - name = LiveKitObjC; - packageProductDependencies = ( - AFA4DBD528C0F839001AD7BE /* LiveKit */, - ); - productName = LiveKitObjC; - productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; - productType = "com.apple.product-type.library.static"; - }; -/* End PBXNativeTarget section */ - -/* Begin PBXProject section */ - AFA4DBC528C0F7F5001AD7BE /* Project object */ = { - isa = PBXProject; - attributes = { - BuildIndependentTargetsInParallel = 1; - LastUpgradeCheck = 1340; - TargetAttributes = { - AFA4DBCC28C0F7F5001AD7BE = { - CreatedOnToolsVersion = 13.4.1; - LastSwiftMigration = 1340; - }; - }; - }; - buildConfigurationList = AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */; - compatibilityVersion = "Xcode 13.0"; - developmentRegion = en; - hasScannedForEncodings = 0; - knownRegions = ( - en, - Base, - ); - mainGroup = AFA4DBC428C0F7F5001AD7BE; - packageReferences = ( - AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, - ); - productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; - projectDirPath = ""; - projectRoot = ""; - targets = ( - AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */, - ); - }; -/* End PBXProject section */ - -/* Begin PBXSourcesBuildPhase section */ - AFA4DBCA28C0F7F5001AD7BE /* Sources */ = { - isa = PBXSourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXSourcesBuildPhase section */ - -/* Begin XCBuildConfiguration section */ - AFA4DBCF28C0F7F5001AD7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_ANALYZER_NONNULL = YES; - CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_ENABLE_OBJC_WEAK = YES; - CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_COMMA = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_DOCUMENTATION_COMMENTS = YES; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INFINITE_RECURSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; - CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; - CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; - CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; - CLANG_WARN_STRICT_PROTOTYPES = YES; - CLANG_WARN_SUSPICIOUS_MOVE = YES; - CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; - CLANG_WARN_UNREACHABLE_CODE = YES; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = dwarf; - ENABLE_STRICT_OBJC_MSGSEND = YES; - ENABLE_TESTABILITY = YES; - GCC_C_LANGUAGE_STANDARD = gnu11; - GCC_DYNAMIC_NO_PIC = NO; - GCC_NO_COMMON_BLOCKS = YES; - GCC_OPTIMIZATION_LEVEL = 0; - GCC_PREPROCESSOR_DEFINITIONS = ( - "DEBUG=1", - "$(inherited)", - ); - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 12.3; - MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; - MTL_FAST_MATH = YES; - ONLY_ACTIVE_ARCH = YES; - SDKROOT = macosx; - }; - name = Debug; - }; - AFA4DBD028C0F7F5001AD7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_ANALYZER_NONNULL = YES; - CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_ENABLE_OBJC_WEAK = YES; - CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_COMMA = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_DOCUMENTATION_COMMENTS = YES; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INFINITE_RECURSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; - CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; - CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; - CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; - CLANG_WARN_STRICT_PROTOTYPES = YES; - CLANG_WARN_SUSPICIOUS_MOVE = YES; - CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; - CLANG_WARN_UNREACHABLE_CODE = YES; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - ENABLE_NS_ASSERTIONS = NO; - ENABLE_STRICT_OBJC_MSGSEND = YES; - GCC_C_LANGUAGE_STANDARD = gnu11; - GCC_NO_COMMON_BLOCKS = YES; - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 12.3; - MTL_ENABLE_DEBUG_INFO = NO; - MTL_FAST_MATH = YES; - SDKROOT = macosx; - }; - name = Release; - }; - AFA4DBD228C0F7F5001AD7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - CLANG_ENABLE_MODULES = YES; - CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = YES; - EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = YES; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/../Frameworks", - "@loader_path/../Frameworks", - ); - PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; - PRODUCT_NAME = "$(TARGET_NAME)"; - SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; - SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 5.0; - }; - name = Debug; - }; - AFA4DBD328C0F7F5001AD7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - CLANG_ENABLE_MODULES = YES; - CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = YES; - EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = YES; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/../Frameworks", - "@loader_path/../Frameworks", - ); - PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; - PRODUCT_NAME = "$(TARGET_NAME)"; - SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; - SWIFT_VERSION = 5.0; - }; - name = Release; - }; -/* End XCBuildConfiguration section */ - -/* Begin XCConfigurationList section */ - AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - AFA4DBCF28C0F7F5001AD7BE /* Debug */, - AFA4DBD028C0F7F5001AD7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; - AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - AFA4DBD228C0F7F5001AD7BE /* Debug */, - AFA4DBD328C0F7F5001AD7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; -/* End XCConfigurationList section */ - -/* Begin XCRemoteSwiftPackageReference section */ - AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/livekit/client-sdk-swift"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 1.0.0; - }; - }; -/* End XCRemoteSwiftPackageReference section */ - -/* Begin XCSwiftPackageProductDependency section */ - AFA4DBD528C0F839001AD7BE /* LiveKit */ = { - isa = XCSwiftPackageProductDependency; - package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; - productName = LiveKit; - }; -/* End XCSwiftPackageProductDependency section */ - }; - rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; -} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata deleted file mode 100644 index 919434a6254f0e9651f402737811be6634a03e9c..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist deleted file mode 100644 index 18d981003d68d0546c4804ac2ff47dd97c6e7921..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +++ /dev/null @@ -1,8 +0,0 @@ - - - - - IDEDidComputeMac32BitWarning - - - diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved deleted file mode 100644 index 21468123fbe00e45f20a852757d01279e5120d13..0000000000000000000000000000000000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ /dev/null @@ -1,50 +0,0 @@ -{ - "pins" : [ - { - "identity" : "client-sdk-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/livekit/client-sdk-swift", - "state" : { - "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", - "version" : "1.0.4" - } - }, - { - "identity" : "promises", - "kind" : "remoteSourceControl", - "location" : "https://github.com/google/promises.git", - "state" : { - "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", - "version" : "2.1.1" - } - }, - { - "identity" : "specs", - "kind" : "remoteSourceControl", - "location" : "https://github.com/webrtc-sdk/Specs.git", - "state" : { - "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", - "version" : "104.5112.2" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", - "version" : "1.4.4" - } - }, - { - "identity" : "swift-protobuf", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-protobuf.git", - "state" : { - "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", - "version" : "1.20.1" - } - } - ], - "version" : 2 -} diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift deleted file mode 100644 index 9397b82649863ae3d72f8f24fcb84336c99c0cb8..0000000000000000000000000000000000000000 --- a/LiveKitObjC/Room.swift +++ /dev/null @@ -1,45 +0,0 @@ -// -// LKRoom.swift -// LiveKitObjC -// -// Created by Antonio Scandurra on 01/09/22. -// - -import Foundation -import LiveKit - -public class LKRoom: RoomDelegate { - lazy var room = Room(delegate: self) - - init() { - print("INIT!\n"); - } - - deinit { - print("DEINIT!\n"); - } - - public func connect( - url: String, - token: String, - callback: @convention(block) @escaping () -> Void - ) { - self.room.connect(url, token).then { room in - callback() - } - } - -} - - -@_cdecl("LKRoomCreate") -public func LKRoomCreate() -> UnsafeMutableRawPointer { - Unmanaged.passRetained(LKRoom()).toOpaque() -} - -@_cdecl("LKRoomDestroy") -public func LKRoomDestroy(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); -} - - From 5347c7d678d506cc0e0beb3c938b4d8eb18354cb Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 13:19:17 -0600 Subject: [PATCH 39/89] Connect to LiveKit room in capture example --- Cargo.lock | 21 ++ crates/capture/Cargo.toml | 4 + crates/capture/build.rs | 35 +-- crates/capture/src/bindings.h | 2 - crates/capture/src/bindings.rs | 8 - crates/capture/src/compression_session.rs | 178 ----------- crates/capture/src/live_kit_token.rs | 71 +++++ crates/capture/src/main.rs | 277 ++---------------- crates/live_kit/Cargo.toml | 2 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 11 + crates/live_kit/build.rs | 1 + crates/live_kit/src/live_kit.rs | 37 ++- 12 files changed, 173 insertions(+), 474 deletions(-) delete mode 100644 crates/capture/src/bindings.h delete mode 100644 crates/capture/src/bindings.rs delete mode 100644 crates/capture/src/compression_session.rs create mode 100644 crates/capture/src/live_kit_token.rs diff --git a/Cargo.lock b/Cargo.lock index 1e176cda4206c39775f73b74b2a03ec592ab7336..bafbaaf93c075064b7eaa34bb38e8b9a2be3a31d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,12 +765,16 @@ dependencies = [ "foreign-types", "futures", "gpui", + "hmac 0.12.1", + "jwt", "live_kit", "log", "media", "objc", "parking_lot 0.11.2", "postage", + "serde", + "sha2 0.10.2", "simplelog", ] @@ -2747,6 +2751,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5" +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64", + "crypto-common", + "digest 0.10.3", + "hmac 0.12.1", + "serde", + "serde_json", + "sha2 0.10.2", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -2921,6 +2940,8 @@ dependencies = [ name = "live_kit" version = "0.1.0" dependencies = [ + "core-foundation", + "futures", "serde", "serde_json", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index d572c6077ce1aa54dcb61c9521af88cafd7c538c..f8ed31097a81c8e96430c89df7b2a64b8e93a767 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -18,10 +18,14 @@ core-foundation = "0.9.3" core-graphics = "0.22.3" foreign-types = "0.3" futures = "0.3" +hmac = "0.12" +jwt = "0.16" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" parking_lot = "0.11.1" postage = { version = "0.4.1", features = ["futures-traits"] } +serde = { version = "1.0", features = ["derive", "rc"] } +sha2 = "0.10" simplelog = "0.9" [build-dependencies] diff --git a/crates/capture/build.rs b/crates/capture/build.rs index c637f2ce9b11b5209049160cf01755a7530cd7ad..41f60ff48611e0b2e174e46de5455ae5516ed15f 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,38 +1,7 @@ -use std::{env, path::PathBuf, process::Command}; - fn main() { // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); - - let sdk_path = String::from_utf8( - Command::new("xcrun") - .args(&["--sdk", "macosx", "--show-sdk-path"]) - .output() - .unwrap() - .stdout, - ) - .unwrap(); - let sdk_path = sdk_path.trim_end(); - - println!("cargo:rerun-if-changed=src/bindings.h"); - let bindings = bindgen::Builder::default() - .header("src/bindings.h") - .clang_arg(format!("-isysroot{}", sdk_path)) - .clang_arg("-xobjective-c") - .allowlist_function("dispatch_queue_create") - .allowlist_type("SCStreamOutputType") - .allowlist_type("SCFrameStatus") - .allowlist_var("SCStreamFrameInfo.*") - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) - .layout_tests(false) - .generate() - .expect("unable to generate bindings"); - - let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); - bindings - .write_to_file(out_path.join("bindings.rs")) - .expect("couldn't write dispatch bindings"); + // Register exported Objective-C selectors, protocols, etc + println!("cargo:rustc-link-arg=-Wl,-ObjC"); } diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h deleted file mode 100644 index 5e3c5258919a5d2109486af9229bcad83ac2ade6..0000000000000000000000000000000000000000 --- a/crates/capture/src/bindings.h +++ /dev/null @@ -1,2 +0,0 @@ -#import -#import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs deleted file mode 100644 index a1c0b0da3ef5175063e48cd13d9b37d33c6369b3..0000000000000000000000000000000000000000 --- a/crates/capture/src/bindings.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![allow(non_upper_case_globals)] -#![allow(non_camel_case_types)] -#![allow(non_snake_case)] -#![allow(unused)] - -use objc::*; - -include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/capture/src/compression_session.rs b/crates/capture/src/compression_session.rs deleted file mode 100644 index 1a771d057b1afee4d45113a8ce49dd2908648d20..0000000000000000000000000000000000000000 --- a/crates/capture/src/compression_session.rs +++ /dev/null @@ -1,178 +0,0 @@ -use anyhow::Result; -use core_foundation::base::{OSStatus, TCFType}; -use media::{ - core_media::{CMSampleBufferRef, CMSampleTimingInfo, CMVideoCodecType}, - core_video::CVImageBuffer, - video_toolbox::{VTCompressionSession, VTEncodeInfoFlags}, -}; -use std::ffi::c_void; - -pub struct CompressionSession { - session: VTCompressionSession, - output_callback: Box, -} - -impl - CompressionSession -{ - pub fn new(width: usize, height: usize, codec: CMVideoCodecType, callback: F) -> Result { - let callback = Box::new(callback); - let session = VTCompressionSession::new( - width, - height, - codec, - Some(Self::output_callback), - callback.as_ref() as *const _ as *const c_void, - )?; - Ok(Self { - session, - output_callback: callback, - }) - } - - pub fn encode_frame(&self, buffer: &CVImageBuffer, timing: CMSampleTimingInfo) -> Result<()> { - self.session.encode_frame( - buffer.as_concrete_TypeRef(), - timing.presentationTimeStamp, - timing.duration, - ) - } - - extern "C" fn output_callback( - output_callback_ref_con: *mut c_void, - _: *mut c_void, - status: OSStatus, - flags: VTEncodeInfoFlags, - sample_buffer: CMSampleBufferRef, - ) { - let callback = unsafe { &mut *(output_callback_ref_con as *mut F) }; - callback(status, flags, sample_buffer); - } -} - -// unsafe extern "C" fn output( -// output_callback_ref_con: *mut c_void, -// source_frame_ref_con: *mut c_void, -// status: OSStatus, -// info_flags: VTEncodeInfoFlags, -// sample_buffer: CMSampleBufferRef, -// ) { -// if status != 0 { -// println!("error encoding frame, code: {}", status); -// return; -// } -// let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - -// let mut is_iframe = false; -// let attachments = sample_buffer.attachments(); -// if let Some(attachments) = attachments.first() { -// is_iframe = attachments -// .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) -// .map_or(true, |not_sync| { -// CFBooleanGetValue(*not_sync as CFBooleanRef) -// }); -// } - -// const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; -// if is_iframe { -// let format_description = sample_buffer.format_description(); -// for ix in 0..format_description.h264_parameter_set_count() { -// let parameter_set = format_description.h264_parameter_set_at_index(ix); -// stream.extend(START_CODE); -// stream.extend(parameter_set); -// } -// } - -// println!("YO!"); -// } - -// static void videoFrameFinishedEncoding(void *outputCallbackRefCon, -// void *sourceFrameRefCon, -// OSStatus status, -// VTEncodeInfoFlags infoFlags, -// CMSampleBufferRef sampleBuffer) { -// // Check if there were any errors encoding -// if (status != noErr) { -// NSLog(@"Error encoding video, err=%lld", (int64_t)status); -// return; -// } - -// // In this example we will use a NSMutableData object to store the -// // elementary stream. -// NSMutableData *elementaryStream = [NSMutableData data]; - -// // Find out if the sample buffer contains an I-Frame. -// // If so we will write the SPS and PPS NAL units to the elementary stream. -// BOOL isIFrame = NO; -// CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); -// if (CFArrayGetCount(attachmentsArray)) { -// CFBooleanRef notSync; -// CFDictionaryRef dict = CFArrayGetValueAtIndex(attachmentsArray, 0); -// BOOL keyExists = CFDictionaryGetValueIfPresent(dict, -// kCMSampleAttachmentKey_NotSync, -// (const void **)¬Sync); -// // An I-Frame is a sync frame -// isIFrame = !keyExists || !CFBooleanGetValue(notSync); -// } - -// // This is the start code that we will write to -// // the elementary stream before every NAL unit -// static const size_t startCodeLength = 4; -// static const uint8_t startCode[] = {0x00, 0x00, 0x00, 0x01}; - -// // Write the SPS and PPS NAL units to the elementary stream before every I-Frame -// if (isIFrame) { -// CMFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); - -// // Find out how many parameter sets there are -// size_t numberOfParameterSets; -// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, -// 0, NULL, NULL, -// &numberOfParameterSets, -// NULL); - -// // Write each parameter set to the elementary stream -// for (int i = 0; i < numberOfParameterSets; i++) { -// const uint8_t *parameterSetPointer; -// size_t parameterSetLength; -// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, -// i, -// ¶meterSetPointer, -// ¶meterSetLength, -// NULL, NULL); - -// // Write the parameter set to the elementary stream -// [elementaryStream appendBytes:startCode length:startCodeLength]; -// [elementaryStream appendBytes:parameterSetPointer length:parameterSetLength]; -// } -// } - -// // Get a pointer to the raw AVCC NAL unit data in the sample buffer -// size_t blockBufferLength; -// uint8_t *bufferDataPointer = NULL; -// CMBlockBufferGetDataPointer(CMSampleBufferGetDataBuffer(sampleBuffer), -// 0, -// NULL, -// &blockBufferLength, -// (char **)&bufferDataPointer); - -// // Loop through all the NAL units in the block buffer -// // and write them to the elementary stream with -// // start codes instead of AVCC length headers -// size_t bufferOffset = 0; -// static const int AVCCHeaderLength = 4; -// while (bufferOffset < blockBufferLength - AVCCHeaderLength) { -// // Read the NAL unit length -// uint32_t NALUnitLength = 0; -// memcpy(&NALUnitLength, bufferDataPointer + bufferOffset, AVCCHeaderLength); -// // Convert the length value from Big-endian to Little-endian -// NALUnitLength = CFSwapInt32BigToHost(NALUnitLength); -// // Write start code to the elementary stream -// [elementaryStream appendBytes:startCode length:startCodeLength]; -// // Write the NAL unit without the AVCC length header to the elementary stream -// [elementaryStream appendBytes:bufferDataPointer + bufferOffset + AVCCHeaderLength -// length:NALUnitLength]; -// // Move to the next NAL unit in the block buffer -// bufferOffset += AVCCHeaderLength + NALUnitLength; -// } -// } diff --git a/crates/capture/src/live_kit_token.rs b/crates/capture/src/live_kit_token.rs new file mode 100644 index 0000000000000000000000000000000000000000..be4fc4f4a2ade98d8ed4d76b6553cba13d5ab18e --- /dev/null +++ b/crates/capture/src/live_kit_token.rs @@ -0,0 +1,71 @@ +use anyhow::Result; +use hmac::{Hmac, Mac}; +use jwt::SignWithKey; +use serde::Serialize; +use sha2::Sha256; +use std::{ + ops::Add, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ClaimGrants<'a> { + iss: &'a str, + sub: &'a str, + iat: u64, + exp: u64, + nbf: u64, + jwtid: &'a str, + video: VideoGrant<'a>, +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct VideoGrant<'a> { + room_create: Option, + room_join: Option, + room_list: Option, + room_record: Option, + room_admin: Option, + room: Option<&'a str>, + can_publish: Option, + can_subscribe: Option, + can_publish_data: Option, + hidden: Option, + recorder: Option, +} + +pub fn create_token( + api_key: &str, + secret_key: &str, + room_name: &str, + participant_name: &str, +) -> Result { + let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; + + let now = SystemTime::now(); + + let claims = ClaimGrants { + iss: api_key, + sub: participant_name, + iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), + exp: now + .add(DEFAULT_TTL) + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + nbf: 0, + jwtid: participant_name, + video: VideoGrant { + room: Some(room_name), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, + }; + Ok(claims.sign_with_key(&secret_key)?) +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 75df9be3689a45a818714b0d2ba1d14ebe49157a..83454ba77a20b43d28c2d17ab987cbbc3d620c8b 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,20 +1,5 @@ -mod bindings; -mod compression_session; +mod live_kit_token; -use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; -use block::ConcreteBlock; -use byteorder::{BigEndian, ReadBytesExt}; -use bytes::BytesMut; -use cocoa::{ - base::{id, nil, YES}, - foundation::{NSArray, NSString, NSUInteger}, -}; -use core_foundation::{ - base::{CFRelease, TCFType}, - number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, - string::CFStringRef, -}; -use futures::StreamExt; use gpui::{ actions, elements::{Canvas, *}, @@ -24,37 +9,12 @@ use gpui::{ }; use live_kit::Room; use log::LevelFilter; -use media::{ - core_media::{ - kCMSampleAttachmentKey_NotSync, kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, - CMTimeMake, - }, - core_video::{self, CVImageBuffer}, - video_toolbox::VTCompressionSession, -}; -use objc::{ - class, - declare::ClassDecl, - msg_send, - runtime::{Class, Object, Sel}, - sel, sel_impl, -}; -use parking_lot::Mutex; +use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; -use std::{ffi::c_void, ptr, slice, str, sync::Arc}; - -#[allow(non_upper_case_globals)] -const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); fn main() { - println!("Creating room..."); - let room = Room::new(); - - println!("Dropping room..."); - drop(room); - SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::App::new(()).unwrap().run(|cx| { @@ -70,12 +30,32 @@ fn main() { }], }]); - cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); + let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); + + let token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant", + ) + .unwrap(); + + let room = live_kit::Room::new(); + cx.spawn(|cx| async move { + println!("connecting..."); + room.connect("wss://zed.livekit.cloud", &token).await; + println!("connected!"); + drop(room); + }) + .detach(); + + // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); }); } struct ScreenCaptureView { - image_buffer: Option, + image_buffer: Option, } impl gpui::Entity for ScreenCaptureView { @@ -83,188 +63,7 @@ impl gpui::Entity for ScreenCaptureView { } impl ScreenCaptureView { - pub fn new(cx: &mut ViewContext) -> Self { - let (image_buffer_tx, mut image_buffer_rx) = - postage::watch::channel::>(); - let image_buffer_tx = Arc::new(Mutex::new(image_buffer_tx)); - - unsafe { - let block = ConcreteBlock::new(move |content: id, error: id| { - if !error.is_null() { - println!( - "ERROR {}", - string_from_objc(msg_send![error, localizedDescription]) - ); - return; - } - - let applications: id = msg_send![content, applications]; - let displays: id = msg_send![content, displays]; - let display: id = displays.objectAtIndex(0); - let display_width: usize = msg_send![display, width]; - let display_height: usize = msg_send![display, height]; - let mut compression_buffer = BytesMut::new(); - // let compression_session = CompressionSession::new( - // display_width, - // display_height, - // kCMVideoCodecType_H264, - // move |status, flags, sample_buffer| { - // if status != 0 { - // println!("error encoding frame, code: {}", status); - // return; - // } - // let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - - // let mut is_iframe = false; - // let attachments = sample_buffer.attachments(); - // if let Some(attachments) = attachments.first() { - // is_iframe = attachments - // .find(kCMSampleAttachmentKey_NotSync as CFStringRef) - // .map_or(true, |not_sync| { - // CFBooleanGetValue(*not_sync as CFBooleanRef) - // }); - // } - - // const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - // if is_iframe { - // let format_description = sample_buffer.format_description(); - // for ix in 0..format_description.h264_parameter_set_count() { - // let parameter_set = - // format_description.h264_parameter_set_at_index(ix).unwrap(); - // compression_buffer.extend_from_slice(&START_CODE); - // compression_buffer.extend_from_slice(parameter_set); - // let nal_unit = compression_buffer.split(); - // } - // } - - // let data = sample_buffer.data(); - // let mut data = data.bytes(); - - // const AVCC_HEADER_LENGTH: usize = 4; - // while data.len() - AVCC_HEADER_LENGTH > 0 { - // let nal_unit_len = match data.read_u32::() { - // Ok(len) => len as usize, - // Err(error) => { - // log::error!("error decoding nal unit length: {}", error); - // return; - // } - // }; - // compression_buffer.extend_from_slice(&START_CODE); - // compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); - // data = &data[nal_unit_len..]; - - // let nal_unit = compression_buffer.split(); - // } - // }, - // ) - // .unwrap(); - - let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_ivar::<*mut c_void>("callback"); - decl.add_method( - sel!(stream:didOutputSampleBuffer:ofType:), - sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType), - ); - let capture_output_class = decl.register(); - - let output: id = msg_send![capture_output_class, alloc]; - let output: id = msg_send![output, init]; - let surface_tx = image_buffer_tx.clone(); - - let callback = Box::new(move |buffer: CMSampleBufferRef| { - let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); - let attachments = buffer.attachments(); - let attachments = attachments.first().expect("no attachments for sample"); - let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; - let status = core_foundation::number::CFNumber::wrap_under_get_rule( - *attachments.get(string) as CFNumberRef, - ) - .to_i64() - .expect("invalid frame info status"); - - if status != bindings::SCFrameStatus_SCFrameStatusComplete { - println!("received incomplete frame"); - return; - } - - let timing_info = buffer.sample_timing_info(0).unwrap(); - let image_buffer = buffer.image_buffer(); - // compression_session - // .encode_frame(&image_buffer, timing_info) - // .unwrap(); - *surface_tx.lock().borrow_mut() = Some(image_buffer); - }) as Box; - let callback = Box::into_raw(Box::new(callback)); - (*output).set_ivar("callback", callback as *mut c_void); - - let filter: id = msg_send![class!(SCContentFilter), alloc]; - let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; - // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; - let config: id = msg_send![class!(SCStreamConfiguration), alloc]; - let config: id = msg_send![config, init]; - let _: () = msg_send![config, setWidth: display_width * 2]; - let _: () = msg_send![config, setHeight: display_height * 2]; - let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; - let _: () = msg_send![config, setQueueDepth: 6]; - let _: () = msg_send![config, setShowsCursor: YES]; - let _: () = msg_send![ - config, - setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA - ]; - - let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; - let error: id = nil; - let queue = bindings::dispatch_queue_create( - ptr::null(), - bindings::NSObject(ptr::null_mut()), - ); - - let _: () = msg_send![stream, - addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen - sampleHandlerQueue: queue - error: &error - ]; - - let start_capture_completion = ConcreteBlock::new(move |error: id| { - if !error.is_null() { - println!( - "error starting capture... error? {}", - string_from_objc(msg_send![error, localizedDescription]) - ); - return; - } - - println!("starting capture"); - }); - - assert!(!stream.is_null()); - let _: () = msg_send![ - stream, - startCaptureWithCompletionHandler: start_capture_completion - ]; - }); - - let _: id = msg_send![ - class!(SCShareableContent), - getShareableContentWithCompletionHandler: block - ]; - } - - cx.spawn_weak(|this, mut cx| async move { - while let Some(image_buffer) = image_buffer_rx.next().await { - if let Some(this) = this.upgrade(&cx) { - this.update(&mut cx, |this, cx| { - this.image_buffer = image_buffer; - cx.notify(); - }) - } else { - break; - } - } - }) - .detach(); - + pub fn new(_: &mut ViewContext) -> Self { Self { image_buffer: None } } } @@ -288,32 +87,6 @@ impl gpui::View for ScreenCaptureView { } } -pub unsafe fn string_from_objc(string: id) -> String { - if string.is_null() { - Default::default() - } else { - let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; - let bytes = string.UTF8String() as *const u8; - str::from_utf8(slice::from_raw_parts(bytes, len)) - .unwrap() - .to_string() - } -} - -extern "C" fn sample_output( - this: &Object, - _: Sel, - _stream: id, - buffer: id, - _kind: SCStreamOutputType, -) { - unsafe { - let callback = *this.get_ivar::<*mut c_void>("callback"); - let callback = &mut *(callback as *mut Box); - (*callback)(buffer as CMSampleBufferRef); - } -} - fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 15f768345672074e78dffed37a8e0892cbc61972..8cb4df32cc48eb2a6daee01d46cf518dd39610b0 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,6 +9,8 @@ path = "src/live_kit.rs" doctest = false [dependencies] +core-foundation = "0.9.3" +futures = "0.3" [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 11df49e6c6b39263e1d340200177d24789a36800..6a935d7ec4d3d8111082b77537a12d2ab0d269fd 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -10,3 +10,14 @@ public func LKRoomCreate() -> UnsafeMutableRawPointer { public func LKRoomDestroy(ptr: UnsafeRawPointer) { let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); } + +@_cdecl("LKRoomConnect") +public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + + room.connect(url as String, token as String).then { _ in + callback(callback_data); + }.catch { error in + print(error); + }; +} diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index ca9c2d5cb9fa662826a9230e757e7b53cd85e22d..06f49c66186523f7f84e5b10eff9b7a0787a6d82 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -37,6 +37,7 @@ fn main() { } fn build_bridge(swift_target: &SwiftTarget) { + println!("cargo:rerun-if-changed={}", SWIFT_PACKAGE_NAME); let swift_package_root = swift_package_root(); if !Command::new("swift") .args(&["build", "-c", &env::var("PROFILE").unwrap()]) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 3dcaaa74ed73c30ae4e1b105c053ebe9f8ea8a3d..346ba2173be81f82a22f27ad83e81439a0ae2240 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,8 +1,20 @@ +use core_foundation::{ + base::TCFType, + string::{CFString, CFStringRef}, +}; +use futures::{channel::oneshot, Future}; use std::ffi::c_void; extern "C" { fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(ptr: *const c_void); + fn LKRoomDestroy(room: *const c_void); + fn LKRoomConnect( + room: *const c_void, + url: CFStringRef, + token: CFStringRef, + callback: extern "C" fn(*mut c_void) -> (), + callback_data: *mut c_void, + ); } pub struct Room { @@ -15,6 +27,29 @@ impl Room { native_room: unsafe { LKRoomCreate() }, } } + + pub fn connect(&self, url: &str, token: &str) -> impl Future { + let url = CFString::new(url); + let token = CFString::new(token); + + let (tx, rx) = oneshot::channel(); + extern "C" fn did_connect(tx: *mut c_void) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; + let _ = tx.send(()); + } + + unsafe { + LKRoomConnect( + self.native_room, + url.as_concrete_TypeRef(), + token.as_concrete_TypeRef(), + did_connect, + Box::into_raw(Box::new(tx)) as *mut c_void, + ) + } + + async { rx.await.unwrap() } + } } impl Drop for Room { From 7bf64ec23e883cd87e43c0d112056ac8384295cb Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 21:19:00 -0600 Subject: [PATCH 40/89] Add ability to get a screen share track for a window And also list windows --- Cargo.lock | 1 + crates/capture/src/main.rs | 20 +++-- crates/live_kit/Cargo.toml | 1 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 16 ++-- crates/live_kit/src/live_kit.rs | 78 ++++++++++++++++--- 5 files changed, 93 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bafbaaf93c075064b7eaa34bb38e8b9a2be3a31d..a9448065c9d9c4ffd05404d821896c220e023eca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2941,6 +2941,7 @@ name = "live_kit" version = "0.1.0" dependencies = [ "core-foundation", + "core-graphics", "futures", "serde", "serde_json", diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 83454ba77a20b43d28c2d17ab987cbbc3d620c8b..07d284b7b77de580813761f787f21887d5f6ce33 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -7,7 +7,7 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; -use live_kit::Room; +use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; @@ -42,13 +42,17 @@ fn main() { .unwrap(); let room = live_kit::Room::new(); - cx.spawn(|cx| async move { - println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token).await; - println!("connected!"); - drop(room); - }) - .detach(); + cx.foreground() + .spawn(async move { + println!("connecting..."); + room.connect("wss://zed.livekit.cloud", &token).await; + let windows = live_kit::list_windows(); + println!("connected! {:?}", windows); + + let window_id = windows.iter().next().unwrap().id; + let track = LocalVideoTrack::screen_share_for_window(window_id); + }) + .detach(); // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); }); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 8cb4df32cc48eb2a6daee01d46cf518dd39610b0..325753b7f9f606cd2e4af25a57b0192a2b377693 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] core-foundation = "0.9.3" +core-graphics = "0.22.3" futures = "0.3" [build-dependencies] diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 6a935d7ec4d3d8111082b77537a12d2ab0d269fd..a5783d69b42bd9d438bfa602eb92d8223088006e 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,16 +1,16 @@ import Foundation import LiveKit +@_cdecl("LKRelease") +public func LKRelease(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); +} + @_cdecl("LKRoomCreate") public func LKRoomCreate() -> UnsafeMutableRawPointer { Unmanaged.passRetained(Room()).toOpaque() } -@_cdecl("LKRoomDestroy") -public func LKRoomDestroy(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); -} - @_cdecl("LKRoomConnect") public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); @@ -21,3 +21,9 @@ public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString print(error); }; } + +@_cdecl("LKCreateScreenShareTrackForWindow") +public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); + return Unmanaged.passRetained(track).toOpaque() +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 346ba2173be81f82a22f27ad83e81439a0ae2240..7ea94298fde85d293af02cc5d5f28a141541a190 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,13 +1,20 @@ use core_foundation::{ - base::TCFType, + array::CFArray, + base::{TCFType, TCFTypeRef}, + dictionary::CFDictionary, + number::CFNumber, string::{CFString, CFStringRef}, }; +use core_graphics::window::{ + kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, + kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, +}; use futures::{channel::oneshot, Future}; use std::ffi::c_void; extern "C" { + fn LKRelease(object: *const c_void); fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(room: *const c_void); fn LKRoomConnect( room: *const c_void, url: CFStringRef, @@ -15,17 +22,14 @@ extern "C" { callback: extern "C" fn(*mut c_void) -> (), callback_data: *mut c_void, ); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } -pub struct Room { - native_room: *const c_void, -} +pub struct Room(*const c_void); impl Room { pub fn new() -> Self { - Self { - native_room: unsafe { LKRoomCreate() }, - } + Self(unsafe { LKRoomCreate() }) } pub fn connect(&self, url: &str, token: &str) -> impl Future { @@ -40,7 +44,7 @@ impl Room { unsafe { LKRoomConnect( - self.native_room, + self.0, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, @@ -54,6 +58,60 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - unsafe { LKRoomDestroy(self.native_room) } + unsafe { LKRelease(self.0) } + } +} + +pub struct LocalVideoTrack(*const c_void); + +impl LocalVideoTrack { + pub fn screen_share_for_window(window_id: u32) -> Self { + Self(unsafe { LKCreateScreenShareTrackForWindow(window_id) }) + } +} + +impl Drop for LocalVideoTrack { + fn drop(&mut self) { + unsafe { LKRelease(self.0) } + } +} + +#[derive(Debug)] +pub struct WindowInfo { + pub id: u32, + pub owner_pid: i32, + pub owner_name: Option, +} + +pub fn list_windows() -> Vec { + unsafe { + let dicts = CFArray::::wrap_under_get_rule(CGWindowListCopyWindowInfo( + kCGWindowListOptionOnScreenOnly | kCGWindowListOptionExcludeDesktopElements, + kCGNullWindowID, + )); + + dicts + .iter() + .map(|dict| { + let id = + CFNumber::wrap_under_get_rule(*dict.get(kCGWindowNumber.as_void_ptr()) as _) + .to_i64() + .unwrap() as u32; + + let owner_pid = + CFNumber::wrap_under_get_rule(*dict.get(kCGWindowOwnerPID.as_void_ptr()) as _) + .to_i32() + .unwrap(); + + let owner_name = dict + .find(kCGWindowOwnerName.as_void_ptr()) + .map(|name| CFString::wrap_under_get_rule(*name as _).to_string()); + WindowInfo { + id, + owner_pid, + owner_name, + } + }) + .collect() } } From df3ab13441b6f668a86feb5953535c755641369c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 14:52:11 +0200 Subject: [PATCH 41/89] Publish screen-sharing video track on the newly-created Room --- crates/capture/src/main.rs | 3 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 11 +++++ crates/live_kit/build.rs | 6 ++- crates/live_kit/src/live_kit.rs | 48 +++++++++++++++---- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 07d284b7b77de580813761f787f21887d5f6ce33..66a01d976bcca6bfede890a3bb67c1c69ea7ef16 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -41,7 +41,7 @@ fn main() { ) .unwrap(); - let room = live_kit::Room::new(); + let room = Room::new(); cx.foreground() .spawn(async move { println!("connecting..."); @@ -51,6 +51,7 @@ fn main() { let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); + room.publish_video_track(&track).await; }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index a5783d69b42bd9d438bfa602eb92d8223088006e..5d30cc4d2866a96abd5cff71f1b084135eb8ab7e 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -22,6 +22,17 @@ public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString }; } +@_cdecl("LKRoomPublishVideoTrack") +public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); + room.localParticipant?.publishVideoTrack(track: track).then { _ in + callback(callback_data); + }.catch { error in + print(error); + }; +} + @_cdecl("LKCreateScreenShareTrackForWindow") public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index 06f49c66186523f7f84e5b10eff9b7a0787a6d82..4f23cc4e926ea12d8ae12c76a1e08ab6d344af75 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -37,7 +37,11 @@ fn main() { } fn build_bridge(swift_target: &SwiftTarget) { - println!("cargo:rerun-if-changed={}", SWIFT_PACKAGE_NAME); + println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); + println!( + "cargo:rerun-if-changed={}/Package.swift", + SWIFT_PACKAGE_NAME + ); let swift_package_root = swift_package_root(); if !Command::new("swift") .args(&["build", "-c", &env::var("PROFILE").unwrap()]) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 7ea94298fde85d293af02cc5d5f28a141541a190..a3b06cfcf63d181f8051e2553aa6fd6d0967fb1d 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -14,6 +14,7 @@ use std::ffi::c_void; extern "C" { fn LKRelease(object: *const c_void); + fn LKRoomCreate() -> *const c_void; fn LKRoomConnect( room: *const c_void, @@ -22,6 +23,13 @@ extern "C" { callback: extern "C" fn(*mut c_void) -> (), callback_data: *mut c_void, ); + fn LKRoomPublishVideoTrack( + room: *const c_void, + track: *const c_void, + callback: extern "C" fn(*mut c_void) -> (), + callback_data: *mut c_void, + ); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } @@ -35,25 +43,49 @@ impl Room { pub fn connect(&self, url: &str, token: &str) -> impl Future { let url = CFString::new(url); let token = CFString::new(token); - - let (tx, rx) = oneshot::channel(); - extern "C" fn did_connect(tx: *mut c_void) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; - let _ = tx.send(()); - } - + let (did_connect, tx, rx) = Self::build_done_callback(); unsafe { LKRoomConnect( self.0, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, - Box::into_raw(Box::new(tx)) as *mut c_void, + tx, ) } async { rx.await.unwrap() } } + + pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future { + let (did_publish, tx, rx) = Self::build_done_callback(); + unsafe { + LKRoomPublishVideoTrack( + self.0, + track.0, + did_publish, + Box::into_raw(Box::new(tx)) as *mut c_void, + ) + } + async { rx.await.unwrap() } + } + + fn build_done_callback() -> ( + extern "C" fn(*mut c_void), + *mut c_void, + oneshot::Receiver<()>, + ) { + let (tx, rx) = oneshot::channel(); + extern "C" fn done_callback(tx: *mut c_void) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; + let _ = tx.send(()); + } + ( + done_callback, + Box::into_raw(Box::new(tx)) as *mut c_void, + rx, + ) + } } impl Drop for Room { From b154c3c9ee9116652607342d14934cc4c6b82396 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 15:18:03 +0200 Subject: [PATCH 42/89] Expose Swift errors as `anyhow::Result` in live_kit --- Cargo.lock | 1 + crates/capture/src/main.rs | 6 ++-- crates/live_kit/Cargo.toml | 1 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 12 ++++---- crates/live_kit/src/live_kit.rs | 28 +++++++++++-------- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9448065c9d9c4ffd05404d821896c220e023eca..9dbdfe1fc69fc410774db911c3e6b62eec0115ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2940,6 +2940,7 @@ dependencies = [ name = "live_kit" version = "0.1.0" dependencies = [ + "anyhow", "core-foundation", "core-graphics", "futures", diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 66a01d976bcca6bfede890a3bb67c1c69ea7ef16..82a6934fa71a43e4e1447f0d2fd35868f928db8c 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -45,13 +45,15 @@ fn main() { cx.foreground() .spawn(async move { println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token).await; + room.connect("wss://zed.livekit.cloud", &token) + .await + .unwrap(); let windows = live_kit::list_windows(); println!("connected! {:?}", windows); let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); - room.publish_video_track(&track).await; + room.publish_video_track(&track).await.unwrap(); }) .detach(); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 325753b7f9f606cd2e4af25a57b0192a2b377693..d98b7eae88858d1b4dd1d7a71cc667ea86415eb9 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,6 +9,7 @@ path = "src/live_kit.rs" doctest = false [dependencies] +anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 5d30cc4d2866a96abd5cff71f1b084135eb8ab7e..a33aeeb6de6c8d33de3717a0a0c09b8583657620 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -12,24 +12,24 @@ public func LKRoomCreate() -> UnsafeMutableRawPointer { } @_cdecl("LKRoomConnect") -public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { +public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); room.connect(url as String, token as String).then { _ in - callback(callback_data); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?); }.catch { error in - print(error); + callback(callback_data, error.localizedDescription as CFString); }; } @_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { +public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); room.localParticipant?.publishVideoTrack(track: track).then { _ in - callback(callback_data); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?); }.catch { error in - print(error); + callback(callback_data, error.localizedDescription as CFString); }; } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index a3b06cfcf63d181f8051e2553aa6fd6d0967fb1d..0db2ac1e3411a04ee44e199384e6dbe04bd12e3a 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,3 +1,4 @@ +use anyhow::{anyhow, Context, Result}; use core_foundation::{ array::CFArray, base::{TCFType, TCFTypeRef}, @@ -20,13 +21,13 @@ extern "C" { room: *const c_void, url: CFStringRef, token: CFStringRef, - callback: extern "C" fn(*mut c_void) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef) -> (), callback_data: *mut c_void, ); fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, - callback: extern "C" fn(*mut c_void) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef) -> (), callback_data: *mut c_void, ); @@ -40,7 +41,7 @@ impl Room { Self(unsafe { LKRoomCreate() }) } - pub fn connect(&self, url: &str, token: &str) -> impl Future { + pub fn connect(&self, url: &str, token: &str) -> impl Future> { let url = CFString::new(url); let token = CFString::new(token); let (did_connect, tx, rx) = Self::build_done_callback(); @@ -54,10 +55,10 @@ impl Room { ) } - async { rx.await.unwrap() } + async { rx.await.unwrap().context("error connecting to room") } } - pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future { + pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { LKRoomPublishVideoTrack( @@ -67,18 +68,23 @@ impl Room { Box::into_raw(Box::new(tx)) as *mut c_void, ) } - async { rx.await.unwrap() } + async { rx.await.unwrap().context("error publishing video track") } } fn build_done_callback() -> ( - extern "C" fn(*mut c_void), + extern "C" fn(*mut c_void, CFStringRef), *mut c_void, - oneshot::Receiver<()>, + oneshot::Receiver>, ) { let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; - let _ = tx.send(()); + extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; + if error.is_null() { + let _ = tx.send(Ok(())); + } else { + let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; + let _ = tx.send(Err(anyhow!(error))); + } } ( done_callback, From 376e674748b16f0b08ee10ba0f99d166e91f8617 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 16:14:27 +0200 Subject: [PATCH 43/89] Avoid double boxing of oneshot when calling `LKRoomPublishVideoTrack` Co-Authored-By: Nathan Sobo --- crates/live_kit/src/live_kit.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 0db2ac1e3411a04ee44e199384e6dbe04bd12e3a..58a01d7f79db53aab91ad863603ee99e96ddb41e 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -61,12 +61,7 @@ impl Room { pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { - LKRoomPublishVideoTrack( - self.0, - track.0, - did_publish, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) + LKRoomPublishVideoTrack(self.0, track.0, did_publish, tx); } async { rx.await.unwrap().context("error publishing video track") } } From 37ca5651ee63665426d9fb31d9581524f48754a6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 16:20:37 +0200 Subject: [PATCH 44/89] Parameterize LiveKit URL Co-Authored-By: Nathan Sobo --- crates/capture/src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 82a6934fa71a43e4e1447f0d2fd35868f928db8c..967fd6689ca3a8461b7465f09c2a14a2c374a3bf 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -30,6 +30,7 @@ fn main() { }], }]); + let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap(); let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); @@ -45,9 +46,7 @@ fn main() { cx.foreground() .spawn(async move { println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token) - .await - .unwrap(); + room.connect(&live_kit_url, &token).await.unwrap(); let windows = live_kit::list_windows(); println!("connected! {:?}", windows); From 645338cff841d7c276d5a8c75ece2e3182a0fe88 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 18:32:39 +0200 Subject: [PATCH 45/89] Register to publishing of remote tracks --- crates/capture/src/main.rs | 37 +++++--- .../Sources/LiveKitBridge/LiveKitBridge.swift | 47 +++++++--- crates/live_kit/src/live_kit.rs | 93 +++++++++++++++++-- 3 files changed, 143 insertions(+), 34 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 967fd6689ca3a8461b7465f09c2a14a2c374a3bf..f89bc4bf9575d9e7dd2617cb242117dca9b46108 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,5 +1,7 @@ mod live_kit_token; +use std::time::Duration; + use gpui::{ actions, elements::{Canvas, *}, @@ -34,25 +36,38 @@ fn main() { let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - let token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant", - ) - .unwrap(); - - let room = Room::new(); + let background = cx.background().clone(); cx.foreground() .spawn(async move { println!("connecting..."); - room.connect(&live_kit_url, &token).await.unwrap(); + let user1_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-1", + ) + .unwrap(); + let room1 = Room::new("user-1 room"); + room1.connect(&live_kit_url, &user1_token).await.unwrap(); + + let user2_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-2", + ) + .unwrap(); + let room2 = Room::new("user-2 room"); + room2.connect(&live_kit_url, &user2_token).await.unwrap(); + let windows = live_kit::list_windows(); println!("connected! {:?}", windows); let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); - room.publish_video_track(&track).await.unwrap(); + room1.publish_video_track(&track).await.unwrap(); + + background.timer(Duration::from_secs(120)).await; }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index a33aeeb6de6c8d33de3717a0a0c09b8583657620..1802fd91fb2bfda66357c9340eff788e2e5b7523 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,40 +1,61 @@ import Foundation import LiveKit +class LKRoomDelegate: RoomDelegate { + var data: UnsafeRawPointer + var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + + init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + self.data = data + self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack + } + + func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { + self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + } +} + @_cdecl("LKRelease") public func LKRelease(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue() +} + +@_cdecl("LKRoomDelegateCreate") +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack) + return Unmanaged.passRetained(delegate).toOpaque() } @_cdecl("LKRoomCreate") -public func LKRoomCreate() -> UnsafeMutableRawPointer { - Unmanaged.passRetained(Room()).toOpaque() +public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer { + let delegate = Unmanaged.fromOpaque(delegate).takeUnretainedValue() + return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque() } @_cdecl("LKRoomConnect") public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() room.connect(url as String, token as String).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?) }.catch { error in - callback(callback_data, error.localizedDescription as CFString); - }; + callback(callback_data, error.localizedDescription as CFString) + } } @_cdecl("LKRoomPublishVideoTrack") public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); - let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() room.localParticipant?.publishVideoTrack(track: track).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?) }.catch { error in - callback(callback_data, error.localizedDescription as CFString); - }; + callback(callback_data, error.localizedDescription as CFString) + } } @_cdecl("LKCreateScreenShareTrackForWindow") public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) return Unmanaged.passRetained(track).toOpaque() } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 58a01d7f79db53aab91ad863603ee99e96ddb41e..0c254f99b32831c340fce6dc92b0760e724acabf 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -11,34 +11,56 @@ use core_graphics::window::{ kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, }; use futures::{channel::oneshot, Future}; -use std::ffi::c_void; +use std::{ + ffi::c_void, + sync::{Arc, Weak}, +}; extern "C" { fn LKRelease(object: *const c_void); - fn LKRoomCreate() -> *const c_void; + fn LKRoomDelegateCreate( + callback_data: *const c_void, + on_did_subscribe_to_remote_track: extern "C" fn( + callback_data: *mut c_void, + remote_track: *const c_void, + ), + ) -> *const c_void; + + fn LKRoomCreate(delegate: *const c_void) -> *const c_void; fn LKRoomConnect( room: *const c_void, url: CFStringRef, token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, - callback: extern "C" fn(*mut c_void, CFStringRef) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } -pub struct Room(*const c_void); +pub struct Room { + debug_name: &'static str, + native_room: *const c_void, + _delegate: RoomDelegate, +} impl Room { - pub fn new() -> Self { - Self(unsafe { LKRoomCreate() }) + pub fn new(debug_name: &'static str) -> Arc { + Arc::new_cyclic(|weak_room| { + let delegate = RoomDelegate::new(weak_room.clone()); + Self { + debug_name, + native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + _delegate: delegate, + } + }) } pub fn connect(&self, url: &str, token: &str) -> impl Future> { @@ -47,7 +69,7 @@ impl Room { let (did_connect, tx, rx) = Self::build_done_callback(); unsafe { LKRoomConnect( - self.0, + self.native_room, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, @@ -61,11 +83,15 @@ impl Room { pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { - LKRoomPublishVideoTrack(self.0, track.0, did_publish, tx); + LKRoomPublishVideoTrack(self.native_room, track.0, did_publish, tx); } async { rx.await.unwrap().context("error publishing video track") } } + fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) { + println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name); + } + fn build_done_callback() -> ( extern "C" fn(*mut c_void, CFStringRef), *mut c_void, @@ -91,7 +117,46 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - unsafe { LKRelease(self.0) } + unsafe { LKRelease(self.native_room) } + } +} + +struct RoomDelegate { + native_delegate: *const c_void, + weak_room: *const Room, +} + +impl RoomDelegate { + fn new(weak_room: Weak) -> Self { + let weak_room = Weak::into_raw(weak_room); + let native_delegate = unsafe { + LKRoomDelegateCreate( + weak_room as *const c_void, + Self::on_did_subscribe_to_remote_track, + ) + }; + Self { + native_delegate, + weak_room, + } + } + + extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let track = unsafe { RemoteVideoTrack(track) }; + if let Some(room) = room.upgrade() { + room.did_subscribe_to_remote_track(track); + } + let _ = Weak::into_raw(room); + } +} + +impl Drop for RoomDelegate { + fn drop(&mut self) { + unsafe { + LKRelease(self.native_delegate); + let _ = Weak::from_raw(self.weak_room); + } } } @@ -109,6 +174,14 @@ impl Drop for LocalVideoTrack { } } +pub struct RemoteVideoTrack(*const c_void); + +impl Drop for RemoteVideoTrack { + fn drop(&mut self) { + unsafe { LKRelease(self.0) } + } +} + #[derive(Debug)] pub struct WindowInfo { pub id: u32, From d407f521db12751bce5575b788958eecbef329b4 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 12:23:36 +0200 Subject: [PATCH 46/89] WIP: render screen-sharing frames --- Cargo.lock | 2 + crates/capture/src/main.rs | 92 +++++++++++-------- crates/gpui/src/platform/mac/renderer.rs | 2 +- crates/live_kit/Cargo.toml | 3 + .../live_kit/LiveKitBridge/Package.resolved | 3 +- crates/live_kit/LiveKitBridge/Package.swift | 2 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 56 +++++++++-- crates/live_kit/src/live_kit.rs | 79 +++++++++++++--- 8 files changed, 178 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9dbdfe1fc69fc410774db911c3e6b62eec0115ff..7486a62082b2a420b1de6575592202cc574ab6bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,6 +2944,8 @@ dependencies = [ "core-foundation", "core-graphics", "futures", + "media", + "parking_lot 0.11.2", "serde", "serde_json", ] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index f89bc4bf9575d9e7dd2617cb242117dca9b46108..ee58aadf1a716515c38a8a977ab5312dfee39a16 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,7 +1,6 @@ mod live_kit_token; -use std::time::Duration; - +use futures::StreamExt; use gpui::{ actions, elements::{Canvas, *}, @@ -13,6 +12,7 @@ use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; +use std::sync::Arc; actions!(capture, [Quit]); @@ -36,47 +36,46 @@ fn main() { let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - let background = cx.background().clone(); - cx.foreground() - .spawn(async move { - println!("connecting..."); - let user1_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-1", - ) - .unwrap(); - let room1 = Room::new("user-1 room"); - room1.connect(&live_kit_url, &user1_token).await.unwrap(); - - let user2_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-2", - ) + cx.spawn(|mut cx| async move { + let user1_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-1", + ) + .unwrap(); + let room1 = Room::new(); + room1.connect(&live_kit_url, &user1_token).await.unwrap(); + + let user2_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-2", + ) + .unwrap(); + let room2 = Room::new(); + room2.connect(&live_kit_url, &user2_token).await.unwrap(); + cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); + + let windows = live_kit::list_windows(); + let window = windows + .iter() + .find(|w| w.owner_name.as_deref() == Some("Safari")) .unwrap(); - let room2 = Room::new("user-2 room"); - room2.connect(&live_kit_url, &user2_token).await.unwrap(); - - let windows = live_kit::list_windows(); - println!("connected! {:?}", windows); - - let window_id = windows.iter().next().unwrap().id; - let track = LocalVideoTrack::screen_share_for_window(window_id); - room1.publish_video_track(&track).await.unwrap(); - - background.timer(Duration::from_secs(120)).await; - }) - .detach(); + let track = LocalVideoTrack::screen_share_for_window(window.id); + room1.publish_video_track(&track).await.unwrap(); - // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + std::mem::forget(track); + std::mem::forget(room1); + }) + .detach(); }); } struct ScreenCaptureView { image_buffer: Option, + _room: Arc, } impl gpui::Entity for ScreenCaptureView { @@ -84,8 +83,25 @@ impl gpui::Entity for ScreenCaptureView { } impl ScreenCaptureView { - pub fn new(_: &mut ViewContext) -> Self { - Self { image_buffer: None } + pub fn new(room: Arc, cx: &mut ViewContext) -> Self { + let mut remote_video_tracks = room.remote_video_tracks(); + cx.spawn_weak(|this, mut cx| async move { + if let Some(video_track) = remote_video_tracks.next().await { + video_track.add_renderer(move |frame| { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.image_buffer = Some(frame); + cx.notify(); + }); + } + }); + } + }) + .detach(); + Self { + image_buffer: None, + _room: room, + } } } diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index ad0915ed5bdb6a546ce7618c153581f5cb772440..21e67f3233050ba7ab50ea52382764528c91e4fc 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -822,7 +822,7 @@ impl Renderer { { MTLPixelFormat::BGRA8Unorm } else { - panic!("unsupported pixel format") + MTLPixelFormat::R8Unorm }; let texture = self diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index d98b7eae88858d1b4dd1d7a71cc667ea86415eb9..e88d4f7b24ccd21f9dd24480e031e01ca6078c31 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,10 +9,13 @@ path = "src/live_kit.rs" doctest = false [dependencies] +media = { path = "../media" } + anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" +parking_lot = "0.11.1" [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index df0b5d6243f553c6be159e388bf60dc4e46406b5..e95cffe979d210cf039f213f5689fe03c9cadd71 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -5,8 +5,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift.git", "state" : { - "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", - "version" : "1.0.4" + "revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4" } }, { diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index 9e7039998d495f73cd78325813ed05e6f41ba38b..fe715588045aa8c5f7ec23034128ec1177ae93a2 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "5cc3c001779ab147199ce3ea0dce465b846368b4"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 1802fd91fb2bfda66357c9340eff788e2e5b7523..fb1eb9a79e1e349a56a69d5f7b076a297cab6512 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,17 +1,49 @@ import Foundation import LiveKit +import WebRTC class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer - var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void - init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { self.data = data - self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack + self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack } func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + if track.kind == .video { + self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + } + } +} + +class LKVideoRenderer: NSObject, VideoRenderer { + var data: UnsafeRawPointer + var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void + var onDrop: @convention(c) (UnsafeRawPointer) -> Void + var adaptiveStreamIsEnabled: Bool = false + var adaptiveStreamSize: CGSize = .zero + + init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { + self.data = data + self.onFrame = onFrame + self.onDrop = onDrop + } + + deinit { + self.onDrop(self.data) + } + + func setSize(_ size: CGSize) { + print("Called setSize", size); + } + + func renderFrame(_ frame: RTCVideoFrame?) { + let buffer = frame?.buffer as? RTCCVPixelBuffer + if let pixelBuffer = buffer?.pixelBuffer { + self.onFrame(self.data, pixelBuffer) + } } } @@ -21,8 +53,8 @@ public func LKRelease(ptr: UnsafeRawPointer) { } @_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack) +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack) return Unmanaged.passRetained(delegate).toOpaque() } @@ -59,3 +91,15 @@ public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutable let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) return Unmanaged.passRetained(track).toOpaque() } + +@_cdecl("LKVideoRendererCreate") +public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() +} + +@_cdecl("LKVideoTrackAddRenderer") +public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack + let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() + track.add(videoRenderer: renderer) +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 0c254f99b32831c340fce6dc92b0760e724acabf..59ce860a7825108ac99aa8296e29fbd1a9876f14 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -10,7 +10,12 @@ use core_graphics::window::{ kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, }; -use futures::{channel::oneshot, Future}; +use futures::{ + channel::{mpsc, oneshot}, + Future, +}; +use media::core_video::{CVImageBuffer, CVImageBufferRef}; +use parking_lot::Mutex; use std::{ ffi::c_void, sync::{Arc, Weak}, @@ -20,8 +25,8 @@ extern "C" { fn LKRelease(object: *const c_void); fn LKRoomDelegateCreate( - callback_data: *const c_void, - on_did_subscribe_to_remote_track: extern "C" fn( + callback_data: *mut c_void, + on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, remote_track: *const c_void, ), @@ -42,22 +47,30 @@ extern "C" { callback_data: *mut c_void, ); + fn LKVideoRendererCreate( + callback_data: *mut c_void, + on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), + on_drop: extern "C" fn(callback_data: *mut c_void), + ) -> *const c_void; + + fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } pub struct Room { - debug_name: &'static str, native_room: *const c_void, + remote_video_track_subscribers: Mutex>>>, _delegate: RoomDelegate, } impl Room { - pub fn new(debug_name: &'static str) -> Arc { + pub fn new() -> Arc { Arc::new_cyclic(|weak_room| { let delegate = RoomDelegate::new(weak_room.clone()); Self { - debug_name, native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + remote_video_track_subscribers: Default::default(), _delegate: delegate, } }) @@ -88,8 +101,17 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) { - println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name); + pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver> { + let (tx, rx) = mpsc::unbounded(); + self.remote_video_track_subscribers.lock().push(tx); + rx + } + + fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { + let track = Arc::new(track); + self.remote_video_track_subscribers + .lock() + .retain(|tx| tx.unbounded_send(track.clone()).is_ok()); } fn build_done_callback() -> ( @@ -131,8 +153,8 @@ impl RoomDelegate { let weak_room = Weak::into_raw(weak_room); let native_delegate = unsafe { LKRoomDelegateCreate( - weak_room as *const c_void, - Self::on_did_subscribe_to_remote_track, + weak_room as *mut c_void, + Self::on_did_subscribe_to_remote_video_track, ) }; Self { @@ -141,11 +163,11 @@ impl RoomDelegate { } } - extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) { + extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) { let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track = unsafe { RemoteVideoTrack(track) }; + let track = RemoteVideoTrack(track); if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_track(track); + room.did_subscribe_to_remote_video_track(track); } let _ = Weak::into_raw(room); } @@ -176,6 +198,37 @@ impl Drop for LocalVideoTrack { pub struct RemoteVideoTrack(*const c_void); +impl RemoteVideoTrack { + pub fn add_renderer(&self, callback: F) + where + F: 'static + FnMut(CVImageBuffer), + { + extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) + where + F: FnMut(CVImageBuffer), + { + unsafe { + let buffer = CVImageBuffer::wrap_under_get_rule(frame); + let callback = &mut *(callback_data as *mut F); + callback(buffer); + } + } + + extern "C" fn on_drop(callback_data: *mut c_void) { + unsafe { + let _ = Box::from_raw(callback_data as *mut F); + } + } + + let callback_data = Box::into_raw(Box::new(callback)); + unsafe { + let renderer = + LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); + LKVideoTrackAddRenderer(self.0, renderer); + } + } +} + impl Drop for RemoteVideoTrack { fn drop(&mut self) { unsafe { LKRelease(self.0) } From 4e0380c9fb11d0dcf5d4d82b938d8d2eb5868924 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 14:51:35 +0200 Subject: [PATCH 47/89] Debounce frame assignment using a `watch` --- crates/capture/src/main.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ee58aadf1a716515c38a8a977ab5312dfee39a16..af2749ecd9622319358605a1ed6554b94d242a29 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -11,6 +11,7 @@ use gpui::{ use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; +use postage::watch; use simplelog::SimpleLogger; use std::sync::Arc; @@ -65,9 +66,6 @@ fn main() { .unwrap(); let track = LocalVideoTrack::screen_share_for_window(window.id); room1.publish_video_track(&track).await.unwrap(); - - std::mem::forget(track); - std::mem::forget(room1); }) .detach(); }); @@ -87,17 +85,23 @@ impl ScreenCaptureView { let mut remote_video_tracks = room.remote_video_tracks(); cx.spawn_weak(|this, mut cx| async move { if let Some(video_track) = remote_video_tracks.next().await { - video_track.add_renderer(move |frame| { + let (mut frames_tx, mut frames_rx) = watch::channel_with(None); + video_track.add_renderer(move |frame| *frames_tx.borrow_mut() = Some(frame)); + + while let Some(frame) = frames_rx.next().await { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { - this.image_buffer = Some(frame); + this.image_buffer = frame; cx.notify(); }); + } else { + break; } - }); + } } }) .detach(); + Self { image_buffer: None, _room: room, From ca618b02b6ed2ca204b2c068008c352fc4f97859 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 15:39:15 +0200 Subject: [PATCH 48/89] Render surfaces correctly when encoded in `420YpCbCr8BiPlanarFullRange` --- crates/gpui/src/platform/mac/renderer.rs | 75 ++++--- .../gpui/src/platform/mac/shaders/shaders.h | 185 +++++++++--------- .../src/platform/mac/shaders/shaders.metal | 48 +++++ crates/media/src/media.rs | 15 +- 4 files changed, 202 insertions(+), 121 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 21e67f3233050ba7ab50ea52382764528c91e4fc..dea9f0612e4110117901baf0031629fce405b81f 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -28,6 +28,7 @@ pub struct Renderer { shadow_pipeline_state: metal::RenderPipelineState, sprite_pipeline_state: metal::RenderPipelineState, image_pipeline_state: metal::RenderPipelineState, + surface_pipeline_state: metal::RenderPipelineState, path_atlas_pipeline_state: metal::RenderPipelineState, underline_pipeline_state: metal::RenderPipelineState, unit_vertices: metal::Buffer, @@ -116,6 +117,14 @@ impl Renderer { "image_fragment", pixel_format, ); + let surface_pipeline_state = build_pipeline_state( + &device, + &library, + "surface", + "surface_vertex", + "surface_fragment", + pixel_format, + ); let path_atlas_pipeline_state = build_path_atlas_pipeline_state( &device, &library, @@ -141,6 +150,7 @@ impl Renderer { shadow_pipeline_state, sprite_pipeline_state, image_pipeline_state, + surface_pipeline_state, path_atlas_pipeline_state, underline_pipeline_state, unit_vertices, @@ -798,14 +808,14 @@ impl Renderer { return; } - command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + command_encoder.set_render_pipeline_state(&self.surface_pipeline_state); command_encoder.set_vertex_buffer( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexVertices as u64, Some(&self.unit_vertices), 0, ); command_encoder.set_vertex_bytes( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexViewportSize as u64, mem::size_of::() as u64, [drawable_size.to_float2()].as_ptr() as *const c_void, ); @@ -817,64 +827,71 @@ impl Renderer { surface.image_buffer.height() as i32, ); let target_size = surface.bounds.size() * scale_factor; - let pixel_format = if surface.image_buffer.pixel_format_type() - == core_video::kCVPixelFormatType_32BGRA - { - MTLPixelFormat::BGRA8Unorm - } else { - MTLPixelFormat::R8Unorm - }; - let texture = self + assert_eq!( + surface.image_buffer.pixel_format_type(), + core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ); + + let y_texture = self .cv_texture_cache .create_texture_from_image( surface.image_buffer.as_concrete_TypeRef(), ptr::null(), - pixel_format, - source_size.x() as usize, - source_size.y() as usize, + MTLPixelFormat::R8Unorm, + surface.image_buffer.plane_width(0), + surface.image_buffer.plane_height(0), 0, ) .unwrap(); + let cb_cr_texture = self + .cv_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + MTLPixelFormat::RG8Unorm, + surface.image_buffer.plane_width(1), + surface.image_buffer.plane_height(1), + 1, + ) + .unwrap(); align_offset(offset); - let next_offset = *offset + mem::size_of::(); + let next_offset = *offset + mem::size_of::(); assert!( next_offset <= INSTANCE_BUFFER_SIZE, "instance buffer exhausted" ); command_encoder.set_vertex_buffer( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexSurfaces as u64, Some(&self.instances), *offset as u64, ); command_encoder.set_vertex_bytes( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexAtlasSize as u64, mem::size_of::() as u64, [source_size.to_float2()].as_ptr() as *const c_void, ); command_encoder.set_fragment_texture( - shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, - Some(texture.as_texture_ref()), + shaders::GPUISurfaceFragmentInputIndex_GPUISurfaceFragmentInputIndexYAtlas as u64, + Some(y_texture.as_texture_ref()), + ); + command_encoder.set_fragment_texture( + shaders::GPUISurfaceFragmentInputIndex_GPUISurfaceFragmentInputIndexCbCrAtlas + as u64, + Some(cb_cr_texture.as_texture_ref()), ); unsafe { - let buffer_contents = - (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + let buffer_contents = (self.instances.contents() as *mut u8).add(*offset) + as *mut shaders::GPUISurface; std::ptr::write( buffer_contents, - shaders::GPUIImage { + shaders::GPUISurface { origin: origin.to_float2(), target_size: target_size.to_float2(), source_size: source_size.to_float2(), - atlas_origin: Default::default(), - border_top: Default::default(), - border_right: Default::default(), - border_bottom: Default::default(), - border_left: Default::default(), - border_color: Default::default(), - corner_radius: Default::default(), }, ); } diff --git a/crates/gpui/src/platform/mac/shaders/shaders.h b/crates/gpui/src/platform/mac/shaders/shaders.h index 3f5096f37c867cfa2d52f8a8c79cda439418b785..29be2c9e1e789ade67c4c05b53e8300779cdc884 100644 --- a/crates/gpui/src/platform/mac/shaders/shaders.h +++ b/crates/gpui/src/platform/mac/shaders/shaders.h @@ -1,122 +1,125 @@ #include -typedef struct -{ - vector_float2 viewport_size; +typedef struct { + vector_float2 viewport_size; } GPUIUniforms; -typedef enum -{ - GPUIQuadInputIndexVertices = 0, - GPUIQuadInputIndexQuads = 1, - GPUIQuadInputIndexUniforms = 2, +typedef enum { + GPUIQuadInputIndexVertices = 0, + GPUIQuadInputIndexQuads = 1, + GPUIQuadInputIndexUniforms = 2, } GPUIQuadInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - vector_uchar4 background_color; - float border_top; - float border_right; - float border_bottom; - float border_left; - vector_uchar4 border_color; - float corner_radius; +typedef struct { + vector_float2 origin; + vector_float2 size; + vector_uchar4 background_color; + float border_top; + float border_right; + float border_bottom; + float border_left; + vector_uchar4 border_color; + float corner_radius; } GPUIQuad; -typedef enum -{ - GPUIShadowInputIndexVertices = 0, - GPUIShadowInputIndexShadows = 1, - GPUIShadowInputIndexUniforms = 2, +typedef enum { + GPUIShadowInputIndexVertices = 0, + GPUIShadowInputIndexShadows = 1, + GPUIShadowInputIndexUniforms = 2, } GPUIShadowInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - float corner_radius; - float sigma; - vector_uchar4 color; +typedef struct { + vector_float2 origin; + vector_float2 size; + float corner_radius; + float sigma; + vector_uchar4 color; } GPUIShadow; -typedef enum -{ - GPUISpriteVertexInputIndexVertices = 0, - GPUISpriteVertexInputIndexSprites = 1, - GPUISpriteVertexInputIndexViewportSize = 2, - GPUISpriteVertexInputIndexAtlasSize = 3, +typedef enum { + GPUISpriteVertexInputIndexVertices = 0, + GPUISpriteVertexInputIndexSprites = 1, + GPUISpriteVertexInputIndexViewportSize = 2, + GPUISpriteVertexInputIndexAtlasSize = 3, } GPUISpriteVertexInputIndex; -typedef enum -{ - GPUISpriteFragmentInputIndexAtlas = 0, +typedef enum { + GPUISpriteFragmentInputIndexAtlas = 0, } GPUISpriteFragmentInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 target_size; - vector_float2 source_size; - vector_float2 atlas_origin; - vector_uchar4 color; - uint8_t compute_winding; +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; + vector_float2 atlas_origin; + vector_uchar4 color; + uint8_t compute_winding; } GPUISprite; -typedef enum -{ - GPUIPathAtlasVertexInputIndexVertices = 0, - GPUIPathAtlasVertexInputIndexAtlasSize = 1, +typedef enum { + GPUIPathAtlasVertexInputIndexVertices = 0, + GPUIPathAtlasVertexInputIndexAtlasSize = 1, } GPUIPathAtlasVertexInputIndex; -typedef struct -{ - vector_float2 xy_position; - vector_float2 st_position; - vector_float2 clip_rect_origin; - vector_float2 clip_rect_size; +typedef struct { + vector_float2 xy_position; + vector_float2 st_position; + vector_float2 clip_rect_origin; + vector_float2 clip_rect_size; } GPUIPathVertex; -typedef enum -{ - GPUIImageVertexInputIndexVertices = 0, - GPUIImageVertexInputIndexImages = 1, - GPUIImageVertexInputIndexViewportSize = 2, - GPUIImageVertexInputIndexAtlasSize = 3, +typedef enum { + GPUIImageVertexInputIndexVertices = 0, + GPUIImageVertexInputIndexImages = 1, + GPUIImageVertexInputIndexViewportSize = 2, + GPUIImageVertexInputIndexAtlasSize = 3, } GPUIImageVertexInputIndex; -typedef enum -{ - GPUIImageFragmentInputIndexAtlas = 0, +typedef enum { + GPUIImageFragmentInputIndexAtlas = 0, } GPUIImageFragmentInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 target_size; - vector_float2 source_size; - vector_float2 atlas_origin; - float border_top; - float border_right; - float border_bottom; - float border_left; - vector_uchar4 border_color; - float corner_radius; +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; + vector_float2 atlas_origin; + float border_top; + float border_right; + float border_bottom; + float border_left; + vector_uchar4 border_color; + float corner_radius; } GPUIImage; -typedef enum -{ - GPUIUnderlineInputIndexVertices = 0, - GPUIUnderlineInputIndexUnderlines = 1, - GPUIUnderlineInputIndexUniforms = 2, +typedef enum { + GPUISurfaceVertexInputIndexVertices = 0, + GPUISurfaceVertexInputIndexSurfaces = 1, + GPUISurfaceVertexInputIndexViewportSize = 2, + GPUISurfaceVertexInputIndexAtlasSize = 3, +} GPUISurfaceVertexInputIndex; + +typedef enum { + GPUISurfaceFragmentInputIndexYAtlas = 0, + GPUISurfaceFragmentInputIndexCbCrAtlas = 1, +} GPUISurfaceFragmentInputIndex; + +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; +} GPUISurface; + +typedef enum { + GPUIUnderlineInputIndexVertices = 0, + GPUIUnderlineInputIndexUnderlines = 1, + GPUIUnderlineInputIndexUniforms = 2, } GPUIUnderlineInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - float thickness; - vector_uchar4 color; - uint8_t squiggly; +typedef struct { + vector_float2 origin; + vector_float2 size; + float thickness; + vector_uchar4 color; + uint8_t squiggly; } GPUIUnderline; diff --git a/crates/gpui/src/platform/mac/shaders/shaders.metal b/crates/gpui/src/platform/mac/shaders/shaders.metal index 2d79e69d569316f8a99e6c8375031006a9e9559d..795026e747e4ffa1f0083737c7d6b2158f79a1b9 100644 --- a/crates/gpui/src/platform/mac/shaders/shaders.metal +++ b/crates/gpui/src/platform/mac/shaders/shaders.metal @@ -263,6 +263,54 @@ fragment float4 image_fragment( return quad_sdf(input); } +vertex QuadFragmentInput surface_vertex( + uint unit_vertex_id [[vertex_id]], + uint image_id [[instance_id]], + constant float2 *unit_vertices [[buffer(GPUISurfaceVertexInputIndexVertices)]], + constant GPUISurface *images [[buffer(GPUISurfaceVertexInputIndexSurfaces)]], + constant float2 *viewport_size [[buffer(GPUISurfaceVertexInputIndexViewportSize)]], + constant float2 *atlas_size [[buffer(GPUISurfaceVertexInputIndexAtlasSize)]] +) { + float2 unit_vertex = unit_vertices[unit_vertex_id]; + GPUISurface image = images[image_id]; + float2 position = unit_vertex * image.target_size + image.origin; + float4 device_position = to_device_position(position, *viewport_size); + float2 atlas_position = (unit_vertex * image.source_size) / *atlas_size; + + return QuadFragmentInput { + device_position, + atlas_position, + image.origin, + image.target_size, + float4(0.), + 0., + 0., + 0., + 0., + float4(0.), + 0., + }; +} + +fragment float4 surface_fragment( + QuadFragmentInput input [[stage_in]], + texture2d y_atlas [[ texture(GPUISurfaceFragmentInputIndexYAtlas) ]], + texture2d cb_cr_atlas [[ texture(GPUISurfaceFragmentInputIndexCbCrAtlas) ]] +) { + constexpr sampler atlas_sampler(mag_filter::linear, min_filter::linear); + const float4x4 ycbcrToRGBTransform = float4x4( + float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), + float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), + float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), + float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f) + ); + float4 ycbcr = float4(y_atlas.sample(atlas_sampler, input.atlas_position).r, + cb_cr_atlas.sample(atlas_sampler, input.atlas_position).rg, 1.0); + + input.background_color = ycbcrToRGBTransform * ycbcr; + return quad_sdf(input); +} + struct PathAtlasVertexOutput { float4 position [[position]]; float2 st_position; diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index ebe3ef7f4d2daa2d6e59be5f9fa3af5428652f4f..fe69e684e70df437573ecd60a17e29b984291d6d 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -31,7 +31,10 @@ pub mod core_video { #![allow(non_snake_case)] use super::*; - pub use crate::bindings::kCVPixelFormatType_32BGRA; + pub use crate::bindings::{ + kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + }; use crate::bindings::{kCVReturnSuccess, CVReturn, OSType}; use anyhow::{anyhow, Result}; use core_foundation::{ @@ -68,6 +71,14 @@ pub mod core_video { unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } } + pub fn plane_width(&self, plane: usize) -> usize { + unsafe { CVPixelBufferGetWidthOfPlane(self.as_concrete_TypeRef(), plane) } + } + + pub fn plane_height(&self, plane: usize) -> usize { + unsafe { CVPixelBufferGetHeightOfPlane(self.as_concrete_TypeRef(), plane) } + } + pub fn pixel_format_type(&self) -> OSType { unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } } @@ -79,6 +90,8 @@ pub mod core_video { fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetWidthOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; + fn CVPixelBufferGetHeightOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; } From 21c91a29e7c18b88c2715b2984f3a60e34ff126c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 22 Aug 2022 18:17:14 +0200 Subject: [PATCH 49/89] Add the ability to hide the titlebar when creating windows Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform.rs | 15 ++++++++--- crates/gpui/src/platform/mac/window.rs | 36 +++++++++++++++++--------- crates/zed/src/zed.rs | 10 ++++--- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index 38e8348ac30e3c0fff472eb8df1009f8ec1ce67a..e31cc4621374b6f86d2dcb340e5f61429acf9fa6 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -136,8 +136,13 @@ pub trait WindowContext { #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, + pub titlebar: Option>, +} + +#[derive(Debug)] +pub struct TitlebarOptions<'a> { pub title: Option<&'a str>, - pub titlebar_appears_transparent: bool, + pub appears_transparent: bool, pub traffic_light_position: Option, } @@ -246,9 +251,11 @@ impl<'a> Default for WindowOptions<'a> { fn default() -> Self { Self { bounds: WindowBounds::Maximized, - title: Default::default(), - titlebar_appears_transparent: Default::default(), - traffic_light_position: Default::default(), + titlebar: Some(TitlebarOptions { + title: Default::default(), + appears_transparent: Default::default(), + traffic_light_position: Default::default(), + }), } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 9b52041d806bdbd60ec42139a2cca6619e9b53c2..5ceb8c96bee23852d2efb71b09632d45ed93982f 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -339,13 +339,19 @@ impl Window { WindowBounds::Fixed(rect) => rect, } .to_ns_rect(); - let mut style_mask = NSWindowStyleMask::NSClosableWindowMask - | NSWindowStyleMask::NSMiniaturizableWindowMask - | NSWindowStyleMask::NSResizableWindowMask - | NSWindowStyleMask::NSTitledWindowMask; - if options.titlebar_appears_transparent { - style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; + let mut style_mask; + if let Some(titlebar) = options.titlebar.as_ref() { + style_mask = NSWindowStyleMask::NSClosableWindowMask + | NSWindowStyleMask::NSMiniaturizableWindowMask + | NSWindowStyleMask::NSResizableWindowMask + | NSWindowStyleMask::NSTitledWindowMask; + + if titlebar.appears_transparent { + style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; + } + } else { + style_mask = NSWindowStyleMask::empty(); } let native_window: id = msg_send![WINDOW_CLASS, alloc]; @@ -409,7 +415,10 @@ impl Window { command_queue: device.new_command_queue(), last_fresh_keydown: None, layer, - traffic_light_position: options.traffic_light_position, + traffic_light_position: options + .titlebar + .as_ref() + .and_then(|titlebar| titlebar.traffic_light_position), previous_modifiers_changed_event: None, ime_state: ImeState::None, ime_text: None, @@ -425,12 +434,15 @@ impl Window { Rc::into_raw(window.0.clone()) as *const c_void, ); - if let Some(title) = options.title.as_ref() { - native_window.setTitle_(NSString::alloc(nil).init_str(title)); - } - if options.titlebar_appears_transparent { - native_window.setTitlebarAppearsTransparent_(YES); + if let Some(titlebar) = options.titlebar { + if let Some(title) = titlebar.title { + native_window.setTitle_(NSString::alloc(nil).init_str(title)); + } + if titlebar.appears_transparent { + native_window.setTitlebarAppearsTransparent_(YES); + } } + native_window.setAcceptsMouseMovedEvents_(YES); native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index a8f10a1579f09ecdaaced6763d2305307ff8fc88..59e9f8de9daa489bf11d3d002aae8fcbe69d29ce 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, ViewContext, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, }; use language::Rope; pub use lsp; @@ -330,9 +330,11 @@ pub fn initialize_workspace( pub fn build_window_options() -> WindowOptions<'static> { WindowOptions { bounds: WindowBounds::Maximized, - title: None, - titlebar_appears_transparent: true, - traffic_light_position: Some(vec2f(8., 8.)), + titlebar: Some(TitlebarOptions { + title: None, + appears_transparent: true, + traffic_light_position: Some(vec2f(8., 8.)), + }), } } From c03300df29250045175c7427efe40c3f611514d9 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 22 Aug 2022 18:17:48 +0200 Subject: [PATCH 50/89] WIP: Start on `App::add_status_bar_item` Co-Authored-By: Nathan Sobo --- crates/gpui/src/app.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index e9091d74c8001c59c03beb6c2e1f4101ae567035..9598540166e624a37a51a2318bd69cf7e8d300f4 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1926,6 +1926,29 @@ impl MutableAppContext { }) } + // pub fn add_status_bar_item( + // &mut self, + // build_item: F1, + // build_menu: F2, + // menu_bounds: Vector2F, + // ) where + // I: View, + // M: View, + // F1: FnOnce(&mut ViewContext) -> I, + // F2: FnOnce(&mut ViewContext) -> M, + // { + // self.add_window( + // WindowOptions { + // bounds: menu_bounds, + // titlebar: None, + // title: None, + // titlebar_appears_transparent: true, + // traffic_light_position: (), + // }, + // build_root_view, + // ) + // } + pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle where T: View, From 2b9fe0a2e6d32f025a093ef142caabcf2baeb117 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 19:07:11 +0200 Subject: [PATCH 51/89] WIP --- crates/capture/src/main.rs | 16 ++++++-- crates/gpui/src/app.rs | 23 ++--------- crates/gpui/src/platform.rs | 13 +++++-- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/platform.rs | 42 ++++++++++++--------- crates/gpui/src/platform/mac/status_item.rs | 25 ++++++++++++ crates/gpui/src/platform/test.rs | 18 +++++---- crates/zed/src/main.rs | 2 + 8 files changed, 89 insertions(+), 51 deletions(-) create mode 100644 crates/gpui/src/platform/mac/status_item.rs diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index af2749ecd9622319358605a1ed6554b94d242a29..c34f451e41e14ae539259c231c05e2e6684e0408 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -116,15 +116,25 @@ impl gpui::View for ScreenCaptureView { fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { let image_buffer = self.image_buffer.clone(); - Canvas::new(move |bounds, _, cx| { + let canvas = Canvas::new(move |bounds, _, cx| { if let Some(image_buffer) = image_buffer.clone() { cx.scene.push_surface(Surface { bounds, image_buffer, }); } - }) - .boxed() + }); + + if let Some(image_buffer) = self.image_buffer.as_ref() { + canvas + .constrained() + .with_width(image_buffer.width() as f32) + .with_height(image_buffer.height() as f32) + .aligned() + .boxed() + } else { + canvas.boxed() + } } } diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 9598540166e624a37a51a2318bd69cf7e8d300f4..4a749b551f4067f6bfd920222b1a17fdf26d4a8f 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1926,27 +1926,12 @@ impl MutableAppContext { }) } - // pub fn add_status_bar_item( - // &mut self, - // build_item: F1, - // build_menu: F2, - // menu_bounds: Vector2F, - // ) where + // pub fn add_status_bar_item(&mut self, build_item: F) + // where // I: View, - // M: View, - // F1: FnOnce(&mut ViewContext) -> I, - // F2: FnOnce(&mut ViewContext) -> M, + // F: FnOnce(&mut ViewContext) -> I, // { - // self.add_window( - // WindowOptions { - // bounds: menu_bounds, - // titlebar: None, - // title: None, - // titlebar_appears_transparent: true, - // traffic_light_position: (), - // }, - // build_root_view, - // ) + // mem::forget(self.platform.add_status_item()); // } pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index e31cc4621374b6f86d2dcb340e5f61429acf9fa6..ec11ee196a51ef5048f11d7cd77ef6c3c43f872f 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -39,6 +39,11 @@ pub trait Platform: Send + Sync { fn fonts(&self) -> Arc; fn activate(&self, ignoring_other_apps: bool); + fn hide(&self); + fn hide_other_apps(&self); + fn unhide_other_apps(&self); + fn quit(&self); + fn open_window( &self, id: usize, @@ -46,10 +51,8 @@ pub trait Platform: Send + Sync { executor: Rc, ) -> Box; fn key_window_id(&self) -> Option; - fn hide(&self); - fn hide_other_apps(&self); - fn unhide_other_apps(&self); - fn quit(&self); + + fn add_status_item(&self) -> Box; fn write_to_clipboard(&self, item: ClipboardItem); fn read_from_clipboard(&self) -> Option; @@ -133,6 +136,8 @@ pub trait WindowContext { fn present_scene(&mut self, scene: Scene); } +pub trait StatusItem {} + #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index c1a4d45561dae5307a0f628588c37b8977809a61..bd3c4eaf20c96750220a6d10376a79c34663e5af 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -7,6 +7,7 @@ mod image_cache; mod platform; mod renderer; mod sprite_cache; +mod status_item; mod window; use cocoa::base::{BOOL, NO, YES}; diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 113653478a8a235e47ab597229360dc5952b60cc..cda6521a8a708f3813265d30686fe934d9fdf24f 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -1,4 +1,6 @@ -use super::{event::key_to_native, BoolExt as _, Dispatcher, FontSystem, Window}; +use super::{ + event::key_to_native, status_item::StatusItem, BoolExt as _, Dispatcher, FontSystem, Window, +}; use crate::{ executor, keymap, platform::{self, CursorStyle}, @@ -439,23 +441,6 @@ impl platform::Platform for MacPlatform { } } - fn open_window( - &self, - id: usize, - options: platform::WindowOptions, - executor: Rc, - ) -> Box { - Box::new(Window::open(id, options, executor, self.fonts())) - } - - fn key_window_id(&self) -> Option { - Window::key_window_id() - } - - fn fonts(&self) -> Arc { - self.fonts.clone() - } - fn hide(&self) { unsafe { let app = NSApplication::sharedApplication(nil); @@ -497,6 +482,27 @@ impl platform::Platform for MacPlatform { } } + fn open_window( + &self, + id: usize, + options: platform::WindowOptions, + executor: Rc, + ) -> Box { + Box::new(Window::open(id, options, executor, self.fonts())) + } + + fn key_window_id(&self) -> Option { + Window::key_window_id() + } + + fn add_status_item(&self) -> Box { + Box::new(StatusItem::add()) + } + + fn fonts(&self) -> Arc { + self.fonts.clone() + } + fn write_to_clipboard(&self, item: ClipboardItem) { unsafe { self.pasteboard.clearContents(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs new file mode 100644 index 0000000000000000000000000000000000000000..6b32d5cb404422df54c9c13adccc00a7598e1e24 --- /dev/null +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -0,0 +1,25 @@ +use cocoa::{ + appkit::{NSSquareStatusItemLength, NSStatusBar}, + base::{id, nil}, +}; +use core_foundation::base::TCFType; +use core_graphics::color::CGColor; +use objc::{msg_send, sel, sel_impl}; + +pub struct StatusItem(id); + +impl StatusItem { + pub fn add() -> Self { + unsafe { + let status_bar = NSStatusBar::systemStatusBar(nil); + let native_item: id = + msg_send![status_bar, statusItemWithLength: NSSquareStatusItemLength]; + let button: id = msg_send![native_item, button]; + let layer: id = msg_send![button, layer]; + let _: () = msg_send![layer, setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef()]; + StatusItem(native_item) + } + } +} + +impl crate::StatusItem for StatusItem {} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 519ba931d10ca59fc1721a26b2566c9a8897b8b2..7fe7aca669a3d4737d99d0307d9f1b036455fd06 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -120,6 +120,14 @@ impl super::Platform for Platform { fn activate(&self, _ignoring_other_apps: bool) {} + fn hide(&self) {} + + fn hide_other_apps(&self) {} + + fn unhide_other_apps(&self) {} + + fn quit(&self) {} + fn open_window( &self, _: usize, @@ -136,13 +144,9 @@ impl super::Platform for Platform { None } - fn hide(&self) {} - - fn hide_other_apps(&self) {} - - fn unhide_other_apps(&self) {} - - fn quit(&self) {} + fn add_status_item(&self) -> Box { + todo!() + } fn write_to_clipboard(&self, item: ClipboardItem) { *self.current_clipboard_item.lock() = Some(item); diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index fc3616d59287f184d79c4895fda4bd6ce7368ea7..fafc954d2c33d5b803a1c064a72c1191fd69d833 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -87,6 +87,8 @@ fn main() { }); app.run(move |cx| { + std::mem::forget(cx.platform().add_status_item()); + let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From 1c810d7e8d2737be1abea3f797d74e0a1ce24a82 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 9 Sep 2022 15:07:41 +0200 Subject: [PATCH 52/89] WIP: Show status bar item with a backing metal layer Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform/mac/status_item.rs | 54 +++++++++++++++++---- crates/zed/src/main.rs | 1 - 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 6b32d5cb404422df54c9c13adccc00a7598e1e24..29bae982109501e7bc2de4ff643618b5e1c570e3 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,22 +1,58 @@ use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar}, - base::{id, nil}, + appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, + base::{id, nil, NO, YES}, + foundation::NSRect, + quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; use core_graphics::color::CGColor; -use objc::{msg_send, sel, sel_impl}; +use foreign_types::ForeignType; +use objc::{class, msg_send, rc::StrongPtr, sel, sel_impl}; -pub struct StatusItem(id); +pub struct StatusItem(StrongPtr); impl StatusItem { pub fn add() -> Self { + const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; + unsafe { let status_bar = NSStatusBar::systemStatusBar(nil); - let native_item: id = - msg_send![status_bar, statusItemWithLength: NSSquareStatusItemLength]; - let button: id = msg_send![native_item, button]; - let layer: id = msg_send![button, layer]; - let _: () = msg_send![layer, setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef()]; + let native_item = + StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); + native_item.button().setWantsLayer(true); + + let device: metal::Device = if let Some(device) = metal::Device::system_default() { + device + } else { + log::error!("unable to access a compatible graphics device"); + std::process::exit(1); + }; + + let layer: id = msg_send![class!(CAMetalLayer), layer]; + let _: () = msg_send![layer, setDevice: device.as_ptr()]; + let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; + let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; + let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; + let _: () = msg_send![layer, setPresentsWithTransaction: YES]; + let _: () = msg_send![ + layer, + setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE + | AutoresizingMask::HEIGHT_SIZABLE + ]; + let _: () = msg_send![ + layer, + setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef() + ]; + + let _: () = msg_send![native_item.button(), setLayer: layer]; + let native_item_window: id = msg_send![native_item.button(), window]; + + dbg!(native_item_window.frame().as_CGRect()); + // let rect_in_window: NSRect = msg_send![native_item.button(), convertRect: native_item.button().bounds() toView: nil]; + // let screen_rect: NSRect = + // msg_send![native_item_window, convertRectToScreen: rect_in_window]; + // dbg!(screen_rect.as_CGRect()); + StatusItem(native_item) } } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index fafc954d2c33d5b803a1c064a72c1191fd69d833..43a403ab7f3307dc107ce0aae9eb8aa058256947 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -88,7 +88,6 @@ fn main() { app.run(move |cx| { std::mem::forget(cx.platform().add_status_item()); - let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From e803dd9f72be0055c337e082bd500d9881a4f68c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 9 Sep 2022 15:24:28 +0200 Subject: [PATCH 53/89] Remove `platform::WindowContext` trait Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform.rs | 4 +-- crates/gpui/src/platform/mac/status_item.rs | 1 - crates/gpui/src/platform/mac/window.rs | 6 +--- crates/gpui/src/platform/test.rs | 34 ++++++++++----------- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index ec11ee196a51ef5048f11d7cd77ef6c3c43f872f..b2e199592053728b92e13ca24ad299451c5cd814 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -110,7 +110,7 @@ pub trait InputHandler { fn rect_for_range(&self, range_utf16: Range) -> Option; } -pub trait Window: WindowContext { +pub trait Window { fn as_any_mut(&mut self) -> &mut dyn Any; fn on_event(&mut self, callback: Box bool>); fn on_active_status_change(&mut self, callback: Box); @@ -127,9 +127,7 @@ pub trait Window: WindowContext { fn minimize(&self); fn zoom(&self); fn toggle_full_screen(&self); -} -pub trait WindowContext { fn size(&self) -> Vector2F; fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 29bae982109501e7bc2de4ff643618b5e1c570e3..a672e1e3237d30d6c2ba9072dbe512a32d6d3442 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,7 +1,6 @@ use cocoa::{ appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, base::{id, nil, NO, YES}, - foundation::NSRect, quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5ceb8c96bee23852d2efb71b09632d45ed93982f..ed1231588c4454c070eaaef4f892c1e862413159 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -6,7 +6,7 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - platform::{self, Event, WindowBounds, WindowContext}, + platform::{self, Event, WindowBounds}, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, MouseMovedEvent, Scene, }; @@ -649,9 +649,7 @@ impl platform::Window for Window { }) .detach(); } -} -impl platform::WindowContext for Window { fn size(&self) -> Vector2F { self.0.as_ref().borrow().size() } @@ -713,9 +711,7 @@ impl WindowState { } } } -} -impl platform::WindowContext for WindowState { fn size(&self) -> Vector2F { let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_window.contentView()) }.size; diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 7fe7aca669a3d4737d99d0307d9f1b036455fd06..f64307debfc936ddd4b52dd31e5aea0352f24be0 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -228,24 +228,6 @@ impl super::Dispatcher for Dispatcher { } } -impl super::WindowContext for Window { - fn size(&self) -> Vector2F { - self.size - } - - fn scale_factor(&self) -> f32 { - self.scale_factor - } - - fn titlebar_height(&self) -> f32 { - 24. - } - - fn present_scene(&mut self, scene: crate::Scene) { - self.current_scene = Some(scene); - } -} - impl super::Window for Window { fn as_any_mut(&mut self) -> &mut dyn Any { self @@ -300,6 +282,22 @@ impl super::Window for Window { fn zoom(&self) {} fn toggle_full_screen(&self) {} + + fn size(&self) -> Vector2F { + self.size + } + + fn scale_factor(&self) -> f32 { + self.scale_factor + } + + fn titlebar_height(&self) -> f32 { + 24. + } + + fn present_scene(&mut self, scene: crate::Scene) { + self.current_scene = Some(scene); + } } pub fn platform() -> Platform { From f50c6af00176abd573a965358c25f34b0e313e60 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 12 Sep 2022 10:07:13 +0200 Subject: [PATCH 54/89] Encapsulate metal layer into `Renderer` --- crates/gpui/src/platform/mac/renderer.rs | 92 ++++++++++++++++-------- crates/gpui/src/platform/mac/window.rs | 71 +++++------------- 2 files changed, 79 insertions(+), 84 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index dea9f0612e4110117901baf0031629fce405b81f..f9cc6e730d44637d417440ad29803313b9f549c9 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -8,12 +8,17 @@ use crate::{ platform, scene::{Glyph, Icon, Image, ImageGlyph, Layer, Quad, Scene, Shadow, Underline}, }; -use cocoa::foundation::NSUInteger; +use cocoa::{ + base::{NO, YES}, + foundation::{NSRect, NSUInteger}, + quartzcore::AutoresizingMask, +}; use core_foundation::base::TCFType; use foreign_types::ForeignTypeRef; use log::warn; use media::core_video::{self, CVMetalTextureCache}; -use metal::{MTLPixelFormat, MTLResourceOptions, NSRange}; +use metal::{CGFloat, CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; +use objc::{self, msg_send, sel, sel_impl}; use shaders::ToFloat2 as _; use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; @@ -21,6 +26,8 @@ const SHADERS_METALLIB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/shader const INSTANCE_BUFFER_SIZE: usize = 8192 * 1024; // This is an arbitrary decision. There's probably a more optimal value. pub struct Renderer { + layer: metal::MetalLayer, + command_queue: CommandQueue, sprite_cache: SpriteCache, image_cache: ImageCache, path_atlases: AtlasAllocator, @@ -48,12 +55,30 @@ pub struct Surface { } impl Renderer { - pub fn new( - device: metal::Device, - pixel_format: metal::MTLPixelFormat, - scale_factor: f32, - fonts: Arc, - ) -> Self { + pub fn new(fonts: Arc) -> Self { + const PIXEL_FORMAT: MTLPixelFormat = MTLPixelFormat::BGRA8Unorm; + + let device: metal::Device = if let Some(device) = metal::Device::system_default() { + device + } else { + log::error!("unable to access a compatible graphics device"); + std::process::exit(1); + }; + + let layer = metal::MetalLayer::new(); + layer.set_device(&device); + layer.set_pixel_format(PIXEL_FORMAT); + layer.set_presents_with_transaction(true); + unsafe { + let _: () = msg_send![&*layer, setAllowsNextDrawableTimeout: NO]; + let _: () = msg_send![&*layer, setNeedsDisplayOnBoundsChange: YES]; + let _: () = msg_send![ + &*layer, + setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE + | AutoresizingMask::HEIGHT_SIZABLE + ]; + } + let library = device .new_library_with_data(SHADERS_METALLIB) .expect("error building metal library"); @@ -76,13 +101,8 @@ impl Renderer { MTLResourceOptions::StorageModeManaged, ); - let sprite_cache = SpriteCache::new( - device.clone(), - vec2i(1024, 768), - scale_factor, - fonts.clone(), - ); - let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768), scale_factor, fonts); + let sprite_cache = SpriteCache::new(device.clone(), vec2i(1024, 768), 1., fonts.clone()); + let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768), 1., fonts); let path_atlases = AtlasAllocator::new(device.clone(), build_path_atlas_texture_descriptor()); let quad_pipeline_state = build_pipeline_state( @@ -91,7 +111,7 @@ impl Renderer { "quad", "quad_vertex", "quad_fragment", - pixel_format, + PIXEL_FORMAT, ); let shadow_pipeline_state = build_pipeline_state( &device, @@ -99,7 +119,7 @@ impl Renderer { "shadow", "shadow_vertex", "shadow_fragment", - pixel_format, + PIXEL_FORMAT, ); let sprite_pipeline_state = build_pipeline_state( &device, @@ -107,7 +127,7 @@ impl Renderer { "sprite", "sprite_vertex", "sprite_fragment", - pixel_format, + PIXEL_FORMAT, ); let image_pipeline_state = build_pipeline_state( &device, @@ -115,7 +135,7 @@ impl Renderer { "image", "image_vertex", "image_fragment", - pixel_format, + PIXEL_FORMAT, ); let surface_pipeline_state = build_pipeline_state( &device, @@ -123,7 +143,7 @@ impl Renderer { "surface", "surface_vertex", "surface_fragment", - pixel_format, + PIXEL_FORMAT, ); let path_atlas_pipeline_state = build_path_atlas_pipeline_state( &device, @@ -139,10 +159,12 @@ impl Renderer { "underline", "underline_vertex", "underline_fragment", - pixel_format, + PIXEL_FORMAT, ); let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()).unwrap(); Self { + layer, + command_queue: device.new_command_queue(), sprite_cache, image_cache, path_atlases, @@ -159,13 +181,21 @@ impl Renderer { } } - pub fn render( - &mut self, - scene: &Scene, - drawable_size: Vector2F, - command_buffer: &metal::CommandBufferRef, - output: &metal::TextureRef, - ) { + pub fn layer(&self) -> &metal::MetalLayerRef { + &*self.layer + } + + pub fn render(&mut self, scene: &Scene) { + let layer = self.layer.clone(); + let drawable = layer.next_drawable().unwrap(); + let command_queue = self.command_queue.clone(); + let command_buffer = command_queue.new_command_buffer(); + + let frame: NSRect = unsafe { msg_send![self.layer(), frame] }; + let scale_factor: CGFloat = unsafe { msg_send![self.layer(), contentsScale] }; + let drawable_size = + vec2f(frame.size.width as f32, frame.size.height as f32) * scale_factor as f32; + self.sprite_cache.set_scale_factor(scene.scale_factor()); self.image_cache.set_scale_factor(scene.scale_factor()); @@ -178,13 +208,17 @@ impl Renderer { &mut offset, drawable_size, command_buffer, - output, + drawable.texture(), ); self.instances.did_modify_range(NSRange { location: 0, length: offset as NSUInteger, }); self.image_cache.finish_frame(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drawable.present(); } fn render_path_atlases( diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index ed1231588c4454c070eaaef4f892c1e862413159..822e1700f1bf0403892108b531aad9c6cd9bf243 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -20,11 +20,10 @@ use cocoa::{ foundation::{ NSAutoreleasePool, NSInteger, NSNotFound, NSPoint, NSRect, NSSize, NSString, NSUInteger, }, - quartzcore::AutoresizingMask, }; use core_graphics::display::CGRect; use ctor::ctor; -use foreign_types::ForeignType as _; +use foreign_types::ForeignTypeRef; use objc::{ class, declare::ClassDecl, @@ -306,9 +305,7 @@ struct WindowState { executor: Rc, scene_to_render: Option, renderer: Renderer, - command_queue: metal::CommandQueue, last_fresh_keydown: Option, - layer: id, traffic_light_position: Option, previous_modifiers_changed_event: Option, //State tracking what the IME did after the last request @@ -329,8 +326,6 @@ impl Window { executor: Rc, fonts: Arc, ) -> Self { - const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; - unsafe { let pool = NSAutoreleasePool::new(nil); @@ -368,25 +363,6 @@ impl Window { native_window.setFrame_display_(screen.visibleFrame(), YES); } - let device: metal::Device = if let Some(device) = metal::Device::system_default() { - device - } else { - log::error!("unable to access a compatible graphics device"); - std::process::exit(1); - }; - - let layer: id = msg_send![class!(CAMetalLayer), layer]; - let _: () = msg_send![layer, setDevice: device.as_ptr()]; - let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; - let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; - let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; - let _: () = msg_send![layer, setPresentsWithTransaction: YES]; - let _: () = msg_send![ - layer, - setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE - | AutoresizingMask::HEIGHT_SIZABLE - ]; - let native_view: id = msg_send![VIEW_CLASS, alloc]; let native_view = NSView::init(native_view); assert!(!native_view.is_null()); @@ -406,15 +382,8 @@ impl Window { synthetic_drag_counter: 0, executor, scene_to_render: Default::default(), - renderer: Renderer::new( - device.clone(), - PIXEL_FORMAT, - get_scale_factor(native_window), - fonts, - ), - command_queue: device.new_command_queue(), + renderer: Renderer::new(fonts), last_fresh_keydown: None, - layer, traffic_light_position: options .titlebar .as_ref() @@ -1057,7 +1026,7 @@ extern "C" fn close_window(this: &Object, _: Sel) { extern "C" fn make_backing_layer(this: &Object, _: Sel) -> id { let window_state = unsafe { get_window_state(this) }; let window_state = window_state.as_ref().borrow(); - window_state.layer + window_state.renderer.layer().as_ptr() as id } extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { @@ -1072,8 +1041,14 @@ extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { height: size.y() as f64 * scale_factor, }; - let _: () = msg_send![window_state_borrow.layer, setContentsScale: scale_factor]; - let _: () = msg_send![window_state_borrow.layer, setDrawableSize: drawable_size]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setContentsScale: scale_factor + ]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setDrawableSize: drawable_size + ]; } if let Some(mut callback) = window_state_borrow.resize_callback.take() { @@ -1102,7 +1077,10 @@ extern "C" fn set_frame_size(this: &Object, _: Sel, size: NSSize) { }; unsafe { - let _: () = msg_send![window_state_borrow.layer, setDrawableSize: drawable_size]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setDrawableSize: drawable_size + ]; } drop(window_state_borrow); @@ -1118,25 +1096,8 @@ extern "C" fn display_layer(this: &Object, _: Sel, _: id) { unsafe { let window_state = get_window_state(this); let mut window_state = window_state.as_ref().borrow_mut(); - if let Some(scene) = window_state.scene_to_render.take() { - let drawable: &metal::MetalDrawableRef = msg_send![window_state.layer, nextDrawable]; - let command_queue = window_state.command_queue.clone(); - let command_buffer = command_queue.new_command_buffer(); - - let size = window_state.size(); - let scale_factor = window_state.scale_factor(); - - window_state.renderer.render( - &scene, - size * scale_factor, - command_buffer, - drawable.texture(), - ); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - drawable.present(); + window_state.renderer.render(&scene); }; } } From 6578af6f3b26ef01fd558e203c256a516c772c6f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 12 Sep 2022 12:03:03 +0200 Subject: [PATCH 55/89] WIP: Start rendering GPUI views to macOS status bar --- crates/gpui/src/app.rs | 202 ++++++++++++-------- crates/gpui/src/platform.rs | 4 +- crates/gpui/src/platform/mac/platform.rs | 4 +- crates/gpui/src/platform/mac/status_item.rs | 155 ++++++++++----- crates/gpui/src/platform/test.rs | 4 +- 5 files changed, 236 insertions(+), 133 deletions(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 4a749b551f4067f6bfd920222b1a17fdf26d4a8f..e7f0e260e371a6834b3294a40d2c6cba0e90a82d 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1920,19 +1920,134 @@ impl MutableAppContext { }, ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - this.open_platform_window(window_id, window_options); + + let mut window = + this.cx + .platform + .open_window(window_id, window_options, this.foreground.clone()); + let presenter = Rc::new(RefCell::new( + this.build_presenter(window_id, window.titlebar_height()), + )); + + { + let mut app = this.upgrade(); + let presenter = Rc::downgrade(&presenter); + window.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { + if cx.dispatch_keystroke(window_id, keystroke) { + return true; + } + } + + presenter.borrow_mut().dispatch_event(event, false, cx) + } else { + false + } + }) + })); + } + + { + let mut app = this.upgrade(); + window.on_active_status_change(Box::new(move |is_active| { + app.update(|cx| cx.window_changed_active_status(window_id, is_active)) + })); + } + + { + let mut app = this.upgrade(); + window.on_resize(Box::new(move || { + app.update(|cx| cx.window_was_resized(window_id)) + })); + } + + { + let mut app = this.upgrade(); + window.on_fullscreen(Box::new(move |is_fullscreen| { + app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) + })); + } + + { + let mut app = this.upgrade(); + window.on_close(Box::new(move || { + app.update(|cx| cx.remove_window(window_id)); + })); + } + + window.set_input_handler(Box::new(WindowInputHandler { + app: this.upgrade().0, + window_id, + })); + + let scene = presenter.borrow_mut().build_scene( + window.size(), + window.scale_factor(), + false, + this, + ); + window.present_scene(scene); + this.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), window)); (window_id, root_view) }) } - // pub fn add_status_bar_item(&mut self, build_item: F) - // where - // I: View, - // F: FnOnce(&mut ViewContext) -> I, - // { - // mem::forget(self.platform.add_status_item()); - // } + pub fn add_status_bar_item(&mut self, build_root_view: F) -> (usize, ViewHandle) + where + T: View, + F: FnOnce(&mut ViewContext) -> T, + { + self.update(|this| { + let window_id = post_inc(&mut this.next_window_id); + let root_view = this + .build_and_insert_view(window_id, ParentId::Root, |cx| Some(build_root_view(cx))) + .unwrap(); + this.cx.windows.insert( + window_id, + Window { + root_view: root_view.clone().into(), + focused_view_id: Some(root_view.id()), + is_active: false, + invalidation: None, + is_fullscreen: false, + }, + ); + root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); + + let mut status_item = this.cx.platform.add_status_item(); + let presenter = Rc::new(RefCell::new(this.build_presenter(window_id, 0.))); + + { + let mut app = this.upgrade(); + let presenter = Rc::downgrade(&presenter); + status_item.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + presenter.borrow_mut().dispatch_event(event, cx) + } else { + false + } + }) + })); + } + + let scene = presenter.borrow_mut().build_scene( + status_item.size(), + status_item.scale_factor(), + false, + this, + ); + status_item.present_scene(scene); + this.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), status_item)); + + (window_id, root_view) + }) + } pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle where @@ -1956,77 +2071,6 @@ impl MutableAppContext { self.flush_effects(); } - fn open_platform_window(&mut self, window_id: usize, window_options: WindowOptions) { - let mut window = - self.cx - .platform - .open_window(window_id, window_options, self.foreground.clone()); - let presenter = Rc::new(RefCell::new( - self.build_presenter(window_id, window.titlebar_height()), - )); - - { - let mut app = self.upgrade(); - let presenter = Rc::downgrade(&presenter); - window.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { - if cx.dispatch_keystroke(window_id, keystroke) { - return true; - } - } - - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false - } - }) - })); - } - - { - let mut app = self.upgrade(); - window.on_active_status_change(Box::new(move |is_active| { - app.update(|cx| cx.window_changed_active_status(window_id, is_active)) - })); - } - - { - let mut app = self.upgrade(); - window.on_resize(Box::new(move || { - app.update(|cx| cx.window_was_resized(window_id)) - })); - } - - { - let mut app = self.upgrade(); - window.on_fullscreen(Box::new(move |is_fullscreen| { - app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) - })); - } - - { - let mut app = self.upgrade(); - window.on_close(Box::new(move || { - app.update(|cx| cx.remove_window(window_id)); - })); - } - - window.set_input_handler(Box::new(WindowInputHandler { - app: self.upgrade().0, - window_id, - })); - - let scene = - presenter - .borrow_mut() - .build_scene(window.size(), window.scale_factor(), false, self); - window.present_scene(scene); - self.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), window)); - } - pub fn build_presenter(&mut self, window_id: usize, titlebar_height: f32) -> Presenter { Presenter::new( window_id, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index b2e199592053728b92e13ca24ad299451c5cd814..b739ace9472befef75bb988d441f08c4f4853c1a 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -52,7 +52,7 @@ pub trait Platform: Send + Sync { ) -> Box; fn key_window_id(&self) -> Option; - fn add_status_item(&self) -> Box; + fn add_status_item(&self) -> Box; fn write_to_clipboard(&self, item: ClipboardItem); fn read_from_clipboard(&self) -> Option; @@ -134,8 +134,6 @@ pub trait Window { fn present_scene(&mut self, scene: Scene); } -pub trait StatusItem {} - #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index cda6521a8a708f3813265d30686fe934d9fdf24f..d1a3b7a79792c151a59784ae455e5e177f5319b6 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -495,8 +495,8 @@ impl platform::Platform for MacPlatform { Window::key_window_id() } - fn add_status_item(&self) -> Box { - Box::new(StatusItem::add()) + fn add_status_item(&self) -> Box { + Box::new(StatusItem::add(self.fonts())) } fn fonts(&self) -> Arc { diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index a672e1e3237d30d6c2ba9072dbe512a32d6d3442..6a4d7549661191e56c06e30f87901c9989aa4dd1 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,60 +1,121 @@ +use crate::{ + geometry::vector::{vec2f, Vector2F}, + platform::{self, mac::renderer::Renderer}, + Event, FontSystem, Scene, +}; use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, - base::{id, nil, NO, YES}, - quartzcore::AutoresizingMask, + appkit::{ + NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSViewHeightSizable, + NSViewWidthSizable, NSWindow, + }, + base::{id, nil, YES}, + foundation::NSSize, }; -use core_foundation::base::TCFType; -use core_graphics::color::CGColor; -use foreign_types::ForeignType; -use objc::{class, msg_send, rc::StrongPtr, sel, sel_impl}; +use foreign_types::ForeignTypeRef; +use objc::{msg_send, rc::StrongPtr, sel, sel_impl}; +use std::{cell::RefCell, rc::Rc, sync::Arc}; -pub struct StatusItem(StrongPtr); +pub struct StatusItem(Rc>); -impl StatusItem { - pub fn add() -> Self { - const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; +struct StatusItemState { + native_item: StrongPtr, + renderer: Renderer, + event_callback: Option bool>>, +} +impl StatusItem { + pub fn add(fonts: Arc) -> Self { unsafe { + let renderer = Renderer::new(fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); - native_item.button().setWantsLayer(true); - - let device: metal::Device = if let Some(device) = metal::Device::system_default() { - device - } else { - log::error!("unable to access a compatible graphics device"); - std::process::exit(1); - }; - - let layer: id = msg_send![class!(CAMetalLayer), layer]; - let _: () = msg_send![layer, setDevice: device.as_ptr()]; - let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; - let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; - let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; - let _: () = msg_send![layer, setPresentsWithTransaction: YES]; - let _: () = msg_send![ - layer, - setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE - | AutoresizingMask::HEIGHT_SIZABLE - ]; - let _: () = msg_send![ - layer, - setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef() - ]; - - let _: () = msg_send![native_item.button(), setLayer: layer]; - let native_item_window: id = msg_send![native_item.button(), window]; - - dbg!(native_item_window.frame().as_CGRect()); - // let rect_in_window: NSRect = msg_send![native_item.button(), convertRect: native_item.button().bounds() toView: nil]; - // let screen_rect: NSRect = - // msg_send![native_item_window, convertRectToScreen: rect_in_window]; - // dbg!(screen_rect.as_CGRect()); - - StatusItem(native_item) + + let button = native_item.button(); + button.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); + button.setWantsBestResolutionOpenGLSurface_(YES); + button.setLayer(renderer.layer().as_ptr() as id); + + Self(Rc::new(RefCell::new(StatusItemState { + native_item, + renderer, + event_callback: None, + }))) } } } -impl crate::StatusItem for StatusItem {} +impl platform::Window for StatusItem { + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + fn on_event(&mut self, callback: Box bool>) { + self.0.borrow_mut().event_callback = Some(callback); + } + + fn on_active_status_change(&mut self, _: Box) {} + + fn on_resize(&mut self, _: Box) {} + + fn on_fullscreen(&mut self, _: Box) {} + + fn on_should_close(&mut self, _: Box bool>) {} + + fn on_close(&mut self, _: Box) {} + + fn set_input_handler(&mut self, _: Box) {} + + fn prompt( + &self, + _: crate::PromptLevel, + _: &str, + _: &[&str], + ) -> postage::oneshot::Receiver { + panic!() + } + + fn activate(&self) {} + + fn set_title(&mut self, _: &str) {} + + fn set_edited(&mut self, _: bool) {} + + fn show_character_palette(&self) {} + + fn minimize(&self) {} + + fn zoom(&self) {} + + fn toggle_full_screen(&self) {} + + fn size(&self) -> Vector2F { + self.0.borrow().size() + } + + fn scale_factor(&self) -> f32 { + self.0.borrow().scale_factor() + } + + fn titlebar_height(&self) -> f32 { + 0. + } + + fn present_scene(&mut self, scene: Scene) { + self.0.borrow_mut().renderer.render(&scene); + } +} + +impl StatusItemState { + fn size(&self) -> Vector2F { + let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_item.button()) }.size; + vec2f(width as f32, height as f32) + } + + fn scale_factor(&self) -> f32 { + unsafe { + let window: id = msg_send![self.native_item.button(), window]; + window.screen().backingScaleFactor() as f32 + } + } +} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index f64307debfc936ddd4b52dd31e5aea0352f24be0..0ce44f08173b7784f03640e134ac85307d655d2b 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -144,8 +144,8 @@ impl super::Platform for Platform { None } - fn add_status_item(&self) -> Box { - todo!() + fn add_status_item(&self) -> Box { + Box::new(Window::new(vec2f(24., 24.))) } fn write_to_clipboard(&self, item: ClipboardItem) { From 2acd215bb887945554bf5a9feaad9dceacab9a77 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 10:07:33 +0200 Subject: [PATCH 56/89] Wire up event handling for status items --- crates/gpui/src/platform/mac/status_item.rs | 161 +++++++++++++++++--- 1 file changed, 137 insertions(+), 24 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 6a4d7549661191e56c06e30f87901c9989aa4dd1..81a8ad036fd8c7bf61bdf17ed9ebed08a658eab5 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,15 +5,53 @@ use crate::{ }; use cocoa::{ appkit::{ - NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSViewHeightSizable, - NSViewWidthSizable, NSWindow, + NSApplication, NSButton, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, + NSViewHeightSizable, NSViewWidthSizable, NSWindow, }, base::{id, nil, YES}, - foundation::NSSize, + foundation::{NSSize, NSUInteger}, }; +use ctor::ctor; use foreign_types::ForeignTypeRef; -use objc::{msg_send, rc::StrongPtr, sel, sel_impl}; -use std::{cell::RefCell, rc::Rc, sync::Arc}; +use objc::{ + class, + declare::ClassDecl, + msg_send, + rc::StrongPtr, + runtime::{Class, Object, Sel}, + sel, sel_impl, +}; +use std::{ + cell::RefCell, + ffi::c_void, + ptr, + rc::{Rc, Weak}, + sync::Arc, +}; + +static mut HANDLER_CLASS: *const Class = ptr::null(); +const STATE_IVAR: &str = "state"; + +#[allow(non_upper_case_globals)] +const NSEventMaskAny: NSUInteger = NSUInteger::MAX; + +#[ctor] +unsafe fn build_classes() { + HANDLER_CLASS = { + let mut decl = ClassDecl::new("GPUIStatusItemEventHandler", class!(NSObject)).unwrap(); + decl.add_ivar::<*mut c_void>(STATE_IVAR); + decl.add_method( + sel!(dealloc), + dealloc_handler as extern "C" fn(&Object, Sel), + ); + decl.add_method( + sel!(handleEvent), + handle_event as extern "C" fn(&Object, Sel), + ); + + decl.register() + }; +} pub struct StatusItem(Rc>); @@ -21,6 +59,7 @@ struct StatusItemState { native_item: StrongPtr, renderer: Renderer, event_callback: Option bool>>, + _event_handler: StrongPtr, } impl StatusItem { @@ -36,11 +75,22 @@ impl StatusItem { button.setWantsBestResolutionOpenGLSurface_(YES); button.setLayer(renderer.layer().as_ptr() as id); - Self(Rc::new(RefCell::new(StatusItemState { - native_item, - renderer, - event_callback: None, - }))) + Self(Rc::new_cyclic(|state| { + let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); + let _: () = msg_send![*event_handler, init]; + (**event_handler) + .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); + button.setTarget_(*event_handler); + button.setAction_(sel!(handleEvent)); + let _: () = msg_send![button, sendActionOn: NSEventMaskAny]; + + RefCell::new(StatusItemState { + native_item, + renderer, + event_callback: None, + _event_handler: event_handler, + }) + })) } } } @@ -54,17 +104,29 @@ impl platform::Window for StatusItem { self.0.borrow_mut().event_callback = Some(callback); } - fn on_active_status_change(&mut self, _: Box) {} + fn on_active_status_change(&mut self, _: Box) { + unimplemented!() + } - fn on_resize(&mut self, _: Box) {} + fn on_resize(&mut self, _: Box) { + unimplemented!() + } - fn on_fullscreen(&mut self, _: Box) {} + fn on_fullscreen(&mut self, _: Box) { + unimplemented!() + } - fn on_should_close(&mut self, _: Box bool>) {} + fn on_should_close(&mut self, _: Box bool>) { + unimplemented!() + } - fn on_close(&mut self, _: Box) {} + fn on_close(&mut self, _: Box) { + unimplemented!() + } - fn set_input_handler(&mut self, _: Box) {} + fn set_input_handler(&mut self, _: Box) { + unimplemented!() + } fn prompt( &self, @@ -72,22 +134,36 @@ impl platform::Window for StatusItem { _: &str, _: &[&str], ) -> postage::oneshot::Receiver { - panic!() + unimplemented!() } - fn activate(&self) {} + fn activate(&self) { + unimplemented!() + } - fn set_title(&mut self, _: &str) {} + fn set_title(&mut self, _: &str) { + unimplemented!() + } - fn set_edited(&mut self, _: bool) {} + fn set_edited(&mut self, _: bool) { + unimplemented!() + } - fn show_character_palette(&self) {} + fn show_character_palette(&self) { + unimplemented!() + } - fn minimize(&self) {} + fn minimize(&self) { + unimplemented!() + } - fn zoom(&self) {} + fn zoom(&self) { + unimplemented!() + } - fn toggle_full_screen(&self) {} + fn toggle_full_screen(&self) { + unimplemented!() + } fn size(&self) -> Vector2F { self.0.borrow().size() @@ -119,3 +195,40 @@ impl StatusItemState { } } } + +extern "C" fn dealloc_handler(this: &Object, _: Sel) { + unsafe { + drop_state(this); + let _: () = msg_send![super(this, class!(NSObject)), dealloc]; + } +} + +extern "C" fn handle_event(this: &Object, _: Sel) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state_borrow = state.as_ref().borrow_mut(); + let app = NSApplication::sharedApplication(nil); + let native_event: id = msg_send![app, currentEvent]; + if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { + if let Some(mut callback) = state_borrow.event_callback.take() { + drop(state_borrow); + callback(event); + state.borrow_mut().event_callback = Some(callback); + } + } + } + } +} + +unsafe fn get_state(object: &Object) -> Weak> { + let raw: *mut c_void = *object.get_ivar(STATE_IVAR); + let weak1 = Weak::from_raw(raw as *mut RefCell); + let weak2 = weak1.clone(); + let _ = Weak::into_raw(weak1); + weak2 +} + +unsafe fn drop_state(object: &Object) { + let raw: *const c_void = *object.get_ivar(STATE_IVAR); + Weak::from_raw(raw as *const RefCell); +} From a9c2881831b7c924689539af46cefe6fd7fcf9eb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 12:08:56 +0200 Subject: [PATCH 57/89] Set contents scale and drawable size when creating status metal layer --- crates/gpui/src/platform/mac/renderer.rs | 8 ++------ crates/gpui/src/platform/mac/status_item.rs | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index f9cc6e730d44637d417440ad29803313b9f549c9..dc2d8a9d50485bb542af1fa229b634391376783c 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -187,15 +187,11 @@ impl Renderer { pub fn render(&mut self, scene: &Scene) { let layer = self.layer.clone(); + let drawable_size = layer.drawable_size(); let drawable = layer.next_drawable().unwrap(); let command_queue = self.command_queue.clone(); let command_buffer = command_queue.new_command_buffer(); - let frame: NSRect = unsafe { msg_send![self.layer(), frame] }; - let scale_factor: CGFloat = unsafe { msg_send![self.layer(), contentsScale] }; - let drawable_size = - vec2f(frame.size.width as f32, frame.size.height as f32) * scale_factor as f32; - self.sprite_cache.set_scale_factor(scene.scale_factor()); self.image_cache.set_scale_factor(scene.scale_factor()); @@ -206,7 +202,7 @@ impl Renderer { scene, path_sprites, &mut offset, - drawable_size, + vec2f(drawable_size.width as f32, drawable_size.height as f32), command_buffer, drawable.texture(), ); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 81a8ad036fd8c7bf61bdf17ed9ebed08a658eab5..bd8c83154b610b9d23c988e60fdac32f8baf643b 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,7 +1,7 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, platform::{self, mac::renderer::Renderer}, - Event, FontSystem, Scene, + Event, FontSystem, Scene, Window, }; use cocoa::{ appkit::{ @@ -75,7 +75,7 @@ impl StatusItem { button.setWantsBestResolutionOpenGLSurface_(YES); button.setLayer(renderer.layer().as_ptr() as id); - Self(Rc::new_cyclic(|state| { + let item = Self(Rc::new_cyclic(|state| { let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); let _: () = msg_send![*event_handler, init]; (**event_handler) @@ -90,7 +90,18 @@ impl StatusItem { event_callback: None, _event_handler: event_handler, }) - })) + })); + + { + let item = item.0.borrow(); + let layer = item.renderer.layer(); + let scale_factor = item.scale_factor(); + let size = item.size() * scale_factor; + layer.set_contents_scale(scale_factor.into()); + layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); + } + + item } } } From 11d47f5c728628820ea95b4795eab20b5b94cbdc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:05:50 +0200 Subject: [PATCH 58/89] Use a transparent layer for status bar This allows the compositor to blend the GPUI view with the background. --- crates/gpui/src/platform/mac/renderer.rs | 10 ++++++---- crates/gpui/src/platform/mac/status_item.rs | 2 +- crates/gpui/src/platform/mac/window.rs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index dc2d8a9d50485bb542af1fa229b634391376783c..ea094e998c112b4b875f129b2aef0c5d67fd8002 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -10,14 +10,14 @@ use crate::{ }; use cocoa::{ base::{NO, YES}, - foundation::{NSRect, NSUInteger}, + foundation::NSUInteger, quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; use foreign_types::ForeignTypeRef; use log::warn; use media::core_video::{self, CVMetalTextureCache}; -use metal::{CGFloat, CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; +use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; use objc::{self, msg_send, sel, sel_impl}; use shaders::ToFloat2 as _; use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; @@ -55,7 +55,7 @@ pub struct Surface { } impl Renderer { - pub fn new(fonts: Arc) -> Self { + pub fn new(is_opaque: bool, fonts: Arc) -> Self { const PIXEL_FORMAT: MTLPixelFormat = MTLPixelFormat::BGRA8Unorm; let device: metal::Device = if let Some(device) = metal::Device::system_default() { @@ -69,6 +69,7 @@ impl Renderer { layer.set_device(&device); layer.set_pixel_format(PIXEL_FORMAT); layer.set_presents_with_transaction(true); + layer.set_opaque(is_opaque); unsafe { let _: () = msg_send![&*layer, setAllowsNextDrawableTimeout: NO]; let _: () = msg_send![&*layer, setNeedsDisplayOnBoundsChange: YES]; @@ -363,7 +364,8 @@ impl Renderer { color_attachment.set_texture(Some(output)); color_attachment.set_load_action(metal::MTLLoadAction::Clear); color_attachment.set_store_action(metal::MTLStoreAction::Store); - color_attachment.set_clear_color(metal::MTLClearColor::new(0., 0., 0., 1.)); + let alpha = if self.layer.is_opaque() { 1. } else { 0. }; + color_attachment.set_clear_color(metal::MTLClearColor::new(0., 0., 0., alpha)); let command_encoder = command_buffer.new_render_command_encoder(render_pass_descriptor); command_encoder.set_viewport(metal::MTLViewport { diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index bd8c83154b610b9d23c988e60fdac32f8baf643b..9d98c4a1f7e397685657f4213b7bdc62e1ccf687 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -65,7 +65,7 @@ struct StatusItemState { impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { - let renderer = Renderer::new(fonts); + let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 822e1700f1bf0403892108b531aad9c6cd9bf243..4abee8b96163a8caed3d8dc7dd8c68a3f74e6b64 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -382,7 +382,7 @@ impl Window { synthetic_drag_counter: 0, executor, scene_to_render: Default::default(), - renderer: Renderer::new(fonts), + renderer: Renderer::new(true, fonts), last_fresh_keydown: None, traffic_light_position: options .titlebar From a34eaa36061d030353502e8048ca401067238d0c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:49:49 +0200 Subject: [PATCH 59/89] Pass `reused` parameter to `Presenter::dispatch_event` for status items --- crates/gpui/src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index e7f0e260e371a6834b3294a40d2c6cba0e90a82d..a4d067fb4f1a857bfd267d02fdcfa2e9e4b18e33 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -2027,7 +2027,7 @@ impl MutableAppContext { status_item.on_event(Box::new(move |event| { app.update(|cx| { if let Some(presenter) = presenter.upgrade() { - presenter.borrow_mut().dispatch_event(event, cx) + presenter.borrow_mut().dispatch_event(event, false, cx) } else { false } From b3dd09a0f2df44ded01ff30f83e1ac2f0c365abc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:50:18 +0200 Subject: [PATCH 60/89] :art: --- crates/gpui/src/platform/mac/status_item.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 9d98c4a1f7e397685657f4213b7bdc62e1ccf687..9c08917927999401211e3416c7f383889f60eb74 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,8 +5,8 @@ use crate::{ }; use cocoa::{ appkit::{ - NSApplication, NSButton, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, - NSViewHeightSizable, NSViewWidthSizable, NSWindow, + NSApplication, NSButton, NSEventMask, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, + NSView, NSViewHeightSizable, NSViewWidthSizable, NSWindow, }, base::{id, nil, YES}, foundation::{NSSize, NSUInteger}, @@ -32,9 +32,6 @@ use std::{ static mut HANDLER_CLASS: *const Class = ptr::null(); const STATE_IVAR: &str = "state"; -#[allow(non_upper_case_globals)] -const NSEventMaskAny: NSUInteger = NSUInteger::MAX; - #[ctor] unsafe fn build_classes() { HANDLER_CLASS = { @@ -82,7 +79,7 @@ impl StatusItem { .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); button.setTarget_(*event_handler); button.setAction_(sel!(handleEvent)); - let _: () = msg_send![button, sendActionOn: NSEventMaskAny]; + let _: () = msg_send![button, sendActionOn: NSEventMask::NSAnyEventMask]; RefCell::new(StatusItemState { native_item, From a102b3ba4b1acb3d38f48611401e1fd77fc07670 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:51:00 +0200 Subject: [PATCH 61/89] Start on a real status bar item implementation --- Cargo.lock | 25 +++++++++++++ assets/icons/zed_32.svg | 22 ++++++++++++ crates/contacts_status_item/Cargo.toml | 32 +++++++++++++++++ .../src/contacts_status_item.rs | 36 +++++++++++++++++++ crates/zed/Cargo.toml | 1 + crates/zed/src/main.rs | 4 ++- 6 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 assets/icons/zed_32.svg create mode 100644 crates/contacts_status_item/Cargo.toml create mode 100644 crates/contacts_status_item/src/contacts_status_item.rs diff --git a/Cargo.lock b/Cargo.lock index 7486a62082b2a420b1de6575592202cc574ab6bc..05701b7c56009e791c54b44d3e1917841b9394db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1126,6 +1126,30 @@ dependencies = [ "workspace", ] +[[package]] +name = "contacts_status_item" +version = "0.1.0" +dependencies = [ + "anyhow", + "client", + "collections", + "editor", + "futures", + "fuzzy", + "gpui", + "language", + "log", + "menu", + "picker", + "postage", + "project", + "serde", + "settings", + "theme", + "util", + "workspace", +] + [[package]] name = "context_menu" version = "0.1.0" @@ -7122,6 +7146,7 @@ dependencies = [ "collections", "command_palette", "contacts_panel", + "contacts_status_item", "context_menu", "ctor", "diagnostics", diff --git a/assets/icons/zed_32.svg b/assets/icons/zed_32.svg new file mode 100644 index 0000000000000000000000000000000000000000..f8c0c537c394d4b6287dec7e06bf169248a35bfe --- /dev/null +++ b/assets/icons/zed_32.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/contacts_status_item/Cargo.toml b/crates/contacts_status_item/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..df115a384220553afd75c37b6276931fe6794fb3 --- /dev/null +++ b/crates/contacts_status_item/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "contacts_status_item" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/contacts_status_item.rs" +doctest = false + +[dependencies] +client = { path = "../client" } +collections = { path = "../collections" } +editor = { path = "../editor" } +fuzzy = { path = "../fuzzy" } +gpui = { path = "../gpui" } +menu = { path = "../menu" } +picker = { path = "../picker" } +project = { path = "../project" } +settings = { path = "../settings" } +theme = { path = "../theme" } +util = { path = "../util" } +workspace = { path = "../workspace" } +anyhow = "1.0" +futures = "0.3" +log = "0.4" +postage = { version = "0.4.1", features = ["futures-traits"] } +serde = { version = "1.0", features = ["derive", "rc"] } + +[dev-dependencies] +language = { path = "../language", features = ["test-support"] } +project = { path = "../project", features = ["test-support"] } +workspace = { path = "../workspace", features = ["test-support"] } diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs new file mode 100644 index 0000000000000000000000000000000000000000..928eecb8fe27d128a8d7b5074d8ffa5989eb0a5a --- /dev/null +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -0,0 +1,36 @@ +use gpui::{color::Color, elements::*, Entity, RenderContext, View}; + +pub struct ContactsStatusItem; + +impl Entity for ContactsStatusItem { + type Event = (); +} + +impl View for ContactsStatusItem { + fn ui_name() -> &'static str { + "ContactsStatusItem" + } + + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + MouseEventHandler::new::(0, cx, |state, cx| { + Svg::new("icons/zed_32.svg") + .with_color(if state.clicked.is_some() { + Color::red() + } else { + Color::blue() + }) + .boxed() + }) + .on_down(gpui::MouseButton::Left, |_, cx| {}) + .on_up(gpui::MouseButton::Left, |_, cx| {}) + .contained() + .with_background_color(Color::green()) + .boxed() + } +} + +impl ContactsStatusItem { + pub fn new() -> Self { + Self + } +} diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index 6d2a617d4c81c3f7db5905b546e24df2287f9771..446139eaafbd25c8f4c4374b0f6538ee44586b58 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -27,6 +27,7 @@ context_menu = { path = "../context_menu" } client = { path = "../client" } clock = { path = "../clock" } contacts_panel = { path = "../contacts_panel" } +contacts_status_item = { path = "../contacts_status_item" } diagnostics = { path = "../diagnostics" } editor = { path = "../editor" } file_finder = { path = "../file_finder" } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index 43a403ab7f3307dc107ce0aae9eb8aa058256947..287ac721fdc7e7d30dca312ce74fa42140e59b9d 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -14,6 +14,7 @@ use client::{ http::{self, HttpClient}, UserStore, ZED_SECRET_CLIENT_TOKEN, }; +use contacts_status_item::ContactsStatusItem; use fs::OpenOptions; use futures::{ channel::{mpsc, oneshot}, @@ -87,7 +88,8 @@ fn main() { }); app.run(move |cx| { - std::mem::forget(cx.platform().add_status_item()); + cx.add_status_bar_item(|_| ContactsStatusItem::new()); + let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From 97ccb16c97bd9da0fdb2e04b44e9a914adc967b2 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 16:20:34 +0200 Subject: [PATCH 62/89] Rework status bar item to use a custom view --- .../src/contacts_status_item.rs | 7 +- crates/gpui/src/platform/mac/status_item.rs | 102 ++++++++++++------ 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 928eecb8fe27d128a8d7b5074d8ffa5989eb0a5a..5905c9ad12854fccaf1c4333497afac410ede006 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -21,8 +21,11 @@ impl View for ContactsStatusItem { }) .boxed() }) - .on_down(gpui::MouseButton::Left, |_, cx| {}) - .on_up(gpui::MouseButton::Left, |_, cx| {}) + .on_down(gpui::MouseButton::Left, |_, _| {}) + .on_down_out(gpui::MouseButton::Left, |_, _| {}) + .on_up(gpui::MouseButton::Left, |_, _| {}) + .on_up_out(gpui::MouseButton::Left, |_, _| {}) + .aligned() .contained() .with_background_color(Color::green()) .boxed() diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 9c08917927999401211e3416c7f383889f60eb74..c369ab424523938b45c515f6d19d8e77a58af3e3 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,15 +1,12 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, platform::{self, mac::renderer::Renderer}, - Event, FontSystem, Scene, Window, + Event, FontSystem, Scene, }; use cocoa::{ - appkit::{ - NSApplication, NSButton, NSEventMask, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, - NSView, NSViewHeightSizable, NSViewWidthSizable, NSWindow, - }, + appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSSize, NSUInteger}, + foundation::{NSAutoreleasePool, NSPoint, NSRect, NSSize}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -29,21 +26,56 @@ use std::{ sync::Arc, }; -static mut HANDLER_CLASS: *const Class = ptr::null(); +static mut VIEW_CLASS: *const Class = ptr::null(); const STATE_IVAR: &str = "state"; #[ctor] unsafe fn build_classes() { - HANDLER_CLASS = { - let mut decl = ClassDecl::new("GPUIStatusItemEventHandler", class!(NSObject)).unwrap(); + VIEW_CLASS = { + let mut decl = ClassDecl::new("GPUIStatusItemView", class!(NSView)).unwrap(); decl.add_ivar::<*mut c_void>(STATE_IVAR); + + decl.add_method(sel!(dealloc), dealloc_view as extern "C" fn(&Object, Sel)); + + decl.add_method( + sel!(mouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(mouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(rightMouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(rightMouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(otherMouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(otherMouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); decl.add_method( - sel!(dealloc), - dealloc_handler as extern "C" fn(&Object, Sel), + sel!(mouseMoved:), + handle_view_event as extern "C" fn(&Object, Sel, id), ); decl.add_method( - sel!(handleEvent), - handle_event as extern "C" fn(&Object, Sel), + sel!(mouseDragged:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(scrollWheel:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(flagsChanged:), + handle_view_event as extern "C" fn(&Object, Sel, id), ); decl.register() @@ -56,36 +88,39 @@ struct StatusItemState { native_item: StrongPtr, renderer: Renderer, event_callback: Option bool>>, - _event_handler: StrongPtr, } impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { + let pool = NSAutoreleasePool::new(nil); + let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); let button = native_item.button(); - button.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); - button.setWantsBestResolutionOpenGLSurface_(YES); - button.setLayer(renderer.layer().as_ptr() as id); + let _: () = msg_send![button, setHidden: YES]; let item = Self(Rc::new_cyclic(|state| { - let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); - let _: () = msg_send![*event_handler, init]; - (**event_handler) - .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); - button.setTarget_(*event_handler); - button.setAction_(sel!(handleEvent)); - let _: () = msg_send![button, sendActionOn: NSEventMask::NSAnyEventMask]; + let parent_view = button.superview().superview(); + + let view: id = msg_send![VIEW_CLASS, alloc]; + NSView::initWithFrame_( + view, + NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), + ); + view.setWantsBestResolutionOpenGLSurface_(YES); + view.setLayer(renderer.layer().as_ptr() as id); + view.setWantsLayer(true); + (*view).set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); + parent_view.addSubview_(view.autorelease()); RefCell::new(StatusItemState { native_item, renderer, event_callback: None, - _event_handler: event_handler, }) })); @@ -98,6 +133,8 @@ impl StatusItem { layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } + pool.drain(); + item } } @@ -192,8 +229,11 @@ impl platform::Window for StatusItem { impl StatusItemState { fn size(&self) -> Vector2F { - let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_item.button()) }.size; - vec2f(width as f32, height as f32) + unsafe { + let NSSize { width, height, .. } = + NSWindow::frame(self.native_item.button().superview().superview()).size; + vec2f(width as f32, height as f32) + } } fn scale_factor(&self) -> f32 { @@ -204,19 +244,17 @@ impl StatusItemState { } } -extern "C" fn dealloc_handler(this: &Object, _: Sel) { +extern "C" fn dealloc_view(this: &Object, _: Sel) { unsafe { drop_state(this); - let _: () = msg_send![super(this, class!(NSObject)), dealloc]; + let _: () = msg_send![super(this, class!(NSView)), dealloc]; } } -extern "C" fn handle_event(this: &Object, _: Sel) { +extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); - let app = NSApplication::sharedApplication(nil); - let native_event: id = msg_send![app, currentEvent]; if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { if let Some(mut callback) = state_borrow.event_callback.take() { drop(state_borrow); From 0f9ff575689276de5b7b4cb07c0ce776b5d89a6c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 17:00:11 +0200 Subject: [PATCH 63/89] Show the correct icon in status bar --- assets/icons/zed_22.svg | 4 ++++ assets/icons/zed_32.svg | 22 ------------------- .../src/contacts_status_item.rs | 22 +++---------------- 3 files changed, 7 insertions(+), 41 deletions(-) create mode 100644 assets/icons/zed_22.svg delete mode 100644 assets/icons/zed_32.svg diff --git a/assets/icons/zed_22.svg b/assets/icons/zed_22.svg new file mode 100644 index 0000000000000000000000000000000000000000..68e7dc8e57c966d5723920bc027c313161b95af8 --- /dev/null +++ b/assets/icons/zed_22.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/assets/icons/zed_32.svg b/assets/icons/zed_32.svg deleted file mode 100644 index f8c0c537c394d4b6287dec7e06bf169248a35bfe..0000000000000000000000000000000000000000 --- a/assets/icons/zed_32.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 5905c9ad12854fccaf1c4333497afac410ede006..c7cd9c3a4a6c7e9e63b3c480d8715f5726ce0717 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,4 +1,4 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View}; +use gpui::{elements::*, Entity, RenderContext, View}; pub struct ContactsStatusItem; @@ -11,24 +11,8 @@ impl View for ContactsStatusItem { "ContactsStatusItem" } - fn render(&mut self, cx: &mut RenderContext) -> ElementBox { - MouseEventHandler::new::(0, cx, |state, cx| { - Svg::new("icons/zed_32.svg") - .with_color(if state.clicked.is_some() { - Color::red() - } else { - Color::blue() - }) - .boxed() - }) - .on_down(gpui::MouseButton::Left, |_, _| {}) - .on_down_out(gpui::MouseButton::Left, |_, _| {}) - .on_up(gpui::MouseButton::Left, |_, _| {}) - .on_up_out(gpui::MouseButton::Left, |_, _| {}) - .aligned() - .contained() - .with_background_color(Color::green()) - .boxed() + fn render(&mut self, _: &mut RenderContext) -> ElementBox { + Svg::new("icons/zed_22.svg").aligned().boxed() } } From f67e2bea29f286d41b36ffccfde0fa9cbfb04bfa Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 11:47:43 +0200 Subject: [PATCH 64/89] Refresh windows when OS appearance changes --- .../src/contacts_status_item.rs | 13 +- crates/diagnostics/src/diagnostics.rs | 2 +- crates/editor/src/element.rs | 4 +- crates/gpui/src/app.rs | 47 +++++- crates/gpui/src/elements/list.rs | 4 +- crates/gpui/src/elements/text.rs | 2 +- crates/gpui/src/platform.rs | 28 +++- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/appearance.rs | 37 +++++ crates/gpui/src/platform/mac/platform.rs | 5 + crates/gpui/src/platform/mac/status_item.rs | 146 ++++++++++++++---- crates/gpui/src/platform/mac/window.rs | 47 +++++- crates/gpui/src/platform/test.rs | 6 + crates/gpui/src/presenter.rs | 29 +++- 14 files changed, 304 insertions(+), 67 deletions(-) create mode 100644 crates/gpui/src/platform/mac/appearance.rs diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index c7cd9c3a4a6c7e9e63b3c480d8715f5726ce0717..f3363f18d357a7e24902105fe047c5148cba9804 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,4 +1,4 @@ -use gpui::{elements::*, Entity, RenderContext, View}; +use gpui::{color::Color, elements::*, Appearance, Entity, RenderContext, View}; pub struct ContactsStatusItem; @@ -11,8 +11,15 @@ impl View for ContactsStatusItem { "ContactsStatusItem" } - fn render(&mut self, _: &mut RenderContext) -> ElementBox { - Svg::new("icons/zed_22.svg").aligned().boxed() + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + let color = match cx.appearance { + Appearance::Light | Appearance::VibrantLight => Color::black(), + Appearance::Dark | Appearance::VibrantDark => Color::white(), + }; + Svg::new("icons/zed_22.svg") + .with_color(color) + .aligned() + .boxed() } } diff --git a/crates/diagnostics/src/diagnostics.rs b/crates/diagnostics/src/diagnostics.rs index 271843dc69a5f572264ee02ab40c921d856759d8..25db51e5e8ceef6195f1d5d31d7459b95e2c103e 100644 --- a/crates/diagnostics/src/diagnostics.rs +++ b/crates/diagnostics/src/diagnostics.rs @@ -1149,7 +1149,7 @@ mod tests { editor: &ViewHandle, cx: &mut MutableAppContext, ) -> Vec<(u32, String)> { - let mut presenter = cx.build_presenter(editor.id(), 0.); + let mut presenter = cx.build_presenter(editor.id(), 0., Default::default()); let mut cx = presenter.build_layout_context(Default::default(), false, cx); cx.render(editor, |editor, cx| { let snapshot = editor.snapshot(cx); diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index d9d0c3990cce3960d5c88749bf0a0aefe2609d13..5146be120a00424e25fe27a648a983ad1aff0600 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -2044,7 +2044,7 @@ mod tests { let layouts = editor.update(cx, |editor, cx| { let snapshot = editor.snapshot(cx); - let mut presenter = cx.build_presenter(window_id, 30.); + let mut presenter = cx.build_presenter(window_id, 30., Default::default()); let layout_cx = presenter.build_layout_context(Vector2F::zero(), false, cx); element.layout_line_numbers(0..6, &Default::default(), &snapshot, &layout_cx) }); @@ -2083,7 +2083,7 @@ mod tests { ); let mut scene = Scene::new(1.0); - let mut presenter = cx.build_presenter(window_id, 30.); + let mut presenter = cx.build_presenter(window_id, 30., Default::default()); let mut layout_cx = presenter.build_layout_context(Vector2F::zero(), false, cx); let (size, mut state) = element.layout( SizeConstraint::new(vec2f(500., 500.), vec2f(500., 500.)), diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index a4d067fb4f1a857bfd267d02fdcfa2e9e4b18e33..0baff67e94fd96479218baa4e7be8da063cac1a2 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -9,8 +9,8 @@ use crate::{ platform::{self, KeyDownEvent, Platform, PromptLevel, WindowOptions}, presenter::Presenter, util::post_inc, - AssetCache, AssetSource, ClipboardItem, FontCache, InputHandler, MouseButton, MouseRegionId, - PathPromptOptions, TextLayoutCache, + Appearance, AssetCache, AssetSource, ClipboardItem, FontCache, InputHandler, MouseButton, + MouseRegionId, PathPromptOptions, TextLayoutCache, }; pub use action::*; use anyhow::{anyhow, Context, Result}; @@ -579,6 +579,7 @@ impl TestAppContext { hovered_region_ids: Default::default(), clicked_region_ids: None, refreshing: false, + appearance: Appearance::Light, }; f(view, &mut render_cx) }) @@ -1260,6 +1261,7 @@ impl MutableAppContext { &mut self, window_id: usize, titlebar_height: f32, + appearance: Appearance, ) -> HashMap { self.start_frame(); #[allow(clippy::needless_collect)] @@ -1287,6 +1289,7 @@ impl MutableAppContext { hovered_region_ids: Default::default(), clicked_region_ids: None, refreshing: false, + appearance, }) .unwrap(), ) @@ -1925,9 +1928,11 @@ impl MutableAppContext { this.cx .platform .open_window(window_id, window_options, this.foreground.clone()); - let presenter = Rc::new(RefCell::new( - this.build_presenter(window_id, window.titlebar_height()), - )); + let presenter = Rc::new(RefCell::new(this.build_presenter( + window_id, + window.titlebar_height(), + window.appearance(), + ))); { let mut app = this.upgrade(); @@ -1977,6 +1982,12 @@ impl MutableAppContext { })); } + { + let mut app = this.upgrade(); + window + .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + window.set_input_handler(Box::new(WindowInputHandler { app: this.upgrade().0, window_id, @@ -2019,7 +2030,11 @@ impl MutableAppContext { root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); let mut status_item = this.cx.platform.add_status_item(); - let presenter = Rc::new(RefCell::new(this.build_presenter(window_id, 0.))); + let presenter = Rc::new(RefCell::new(this.build_presenter( + window_id, + 0., + status_item.appearance(), + ))); { let mut app = this.upgrade(); @@ -2035,6 +2050,12 @@ impl MutableAppContext { })); } + { + let mut app = this.upgrade(); + status_item + .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + let scene = presenter.borrow_mut().build_scene( status_item.size(), status_item.scale_factor(), @@ -2071,10 +2092,16 @@ impl MutableAppContext { self.flush_effects(); } - pub fn build_presenter(&mut self, window_id: usize, titlebar_height: f32) -> Presenter { + pub fn build_presenter( + &mut self, + window_id: usize, + titlebar_height: f32, + appearance: Appearance, + ) -> Presenter { Presenter::new( window_id, titlebar_height, + appearance, self.cx.font_cache.clone(), TextLayoutCache::new(self.cx.platform.fonts()), self.assets.clone(), @@ -2412,7 +2439,7 @@ impl MutableAppContext { { { let mut presenter = presenter.borrow_mut(); - presenter.invalidate(&mut invalidation, self); + presenter.invalidate(&mut invalidation, window.appearance(), self); let scene = presenter.build_scene(window.size(), window.scale_factor(), false, self); window.present_scene(scene); @@ -2476,6 +2503,7 @@ impl MutableAppContext { let mut presenter = presenter.borrow_mut(); presenter.refresh( invalidation.as_mut().unwrap_or(&mut Default::default()), + window.appearance(), self, ); let scene = presenter.build_scene(window.size(), window.scale_factor(), true, self); @@ -4082,6 +4110,7 @@ pub struct RenderParams { pub hovered_region_ids: HashSet, pub clicked_region_ids: Option<(Vec, MouseButton)>, pub refreshing: bool, + pub appearance: Appearance, } pub struct RenderContext<'a, T: View> { @@ -4092,6 +4121,7 @@ pub struct RenderContext<'a, T: View> { pub(crate) clicked_region_ids: Option<(Vec, MouseButton)>, pub app: &'a mut MutableAppContext, pub titlebar_height: f32, + pub appearance: Appearance, pub refreshing: bool, } @@ -4112,6 +4142,7 @@ impl<'a, V: View> RenderContext<'a, V> { hovered_region_ids: params.hovered_region_ids.clone(), clicked_region_ids: params.clicked_region_ids.clone(), refreshing: params.refreshing, + appearance: params.appearance, } } diff --git a/crates/gpui/src/elements/list.rs b/crates/gpui/src/elements/list.rs index df480005b58151f978e4fb370f413aff163c7e03..57436c256bd9ee241dc78d07206ba8c3655e2c54 100644 --- a/crates/gpui/src/elements/list.rs +++ b/crates/gpui/src/elements/list.rs @@ -659,7 +659,7 @@ mod tests { #[crate::test(self)] fn test_layout(cx: &mut crate::MutableAppContext) { - let mut presenter = cx.build_presenter(0, 0.); + let mut presenter = cx.build_presenter(0, 0., Default::default()); let (_, view) = cx.add_window(Default::default(), |_| TestView); let constraint = SizeConstraint::new(vec2f(0., 0.), vec2f(100., 40.)); @@ -759,7 +759,7 @@ mod tests { .unwrap_or(10); let (_, view) = cx.add_window(Default::default(), |_| TestView); - let mut presenter = cx.build_presenter(0, 0.); + let mut presenter = cx.build_presenter(0, 0., Default::default()); let mut next_id = 0; let elements = Rc::new(RefCell::new( (0..rng.gen_range(0..=20)) diff --git a/crates/gpui/src/elements/text.rs b/crates/gpui/src/elements/text.rs index 92e38894c1d6170f65362f7e5ea1c71c0898ea14..45dc2dbae3343051e8b417fc6a12393757e125f7 100644 --- a/crates/gpui/src/elements/text.rs +++ b/crates/gpui/src/elements/text.rs @@ -291,7 +291,7 @@ mod tests { #[crate::test(self)] fn test_soft_wrapping_with_carriage_returns(cx: &mut MutableAppContext) { let (window_id, _) = cx.add_window(Default::default(), |_| TestView); - let mut presenter = cx.build_presenter(window_id, Default::default()); + let mut presenter = cx.build_presenter(window_id, Default::default(), Default::default()); fonts::with_font_cache(cx.font_cache().clone(), || { let mut text = Text::new("Hello\r\n".into(), Default::default()).with_soft_wrap(true); let (_, state) = text.layout( diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index b739ace9472befef75bb988d441f08c4f4853c1a..d2ae7d8d178efe35f92fc483eaae8f770620ea2b 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -132,6 +132,8 @@ pub trait Window { fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; fn present_scene(&mut self, scene: Scene); + fn appearance(&self) -> Appearance; + fn on_appearance_changed(&mut self, callback: Box); } #[derive(Debug)] @@ -147,6 +149,20 @@ pub struct TitlebarOptions<'a> { pub traffic_light_position: Option, } +#[derive(Copy, Clone, Debug)] +pub enum Appearance { + Light, + VibrantLight, + Dark, + VibrantDark, +} + +impl Default for Appearance { + fn default() -> Self { + Self::Light + } +} + #[derive(Debug)] pub enum WindowBounds { Maximized, @@ -173,6 +189,12 @@ pub enum CursorStyle { IBeam, } +impl Default for CursorStyle { + fn default() -> Self { + Self::Arrow + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct AppVersion { major: usize, @@ -180,12 +202,6 @@ pub struct AppVersion { patch: usize, } -impl Default for CursorStyle { - fn default() -> Self { - Self::Arrow - } -} - impl FromStr for AppVersion { type Err = anyhow::Error; diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index bd3c4eaf20c96750220a6d10376a79c34663e5af..90b378e4a644dacb22f6550ea4783db98caa7237 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,3 +1,4 @@ +mod appearance; mod atlas; mod dispatcher; mod event; diff --git a/crates/gpui/src/platform/mac/appearance.rs b/crates/gpui/src/platform/mac/appearance.rs new file mode 100644 index 0000000000000000000000000000000000000000..c0b8258a0e2a388075579a403deecba5b46ca85e --- /dev/null +++ b/crates/gpui/src/platform/mac/appearance.rs @@ -0,0 +1,37 @@ +use std::ffi::CStr; + +use cocoa::{ + appkit::{NSAppearanceNameVibrantDark, NSAppearanceNameVibrantLight}, + base::id, + foundation::NSString, +}; +use objc::{msg_send, sel, sel_impl}; + +use crate::Appearance; + +impl Appearance { + pub unsafe fn from_native(appearance: id) -> Self { + let name: id = msg_send![appearance, name]; + if name == NSAppearanceNameVibrantLight { + Self::VibrantLight + } else if name == NSAppearanceNameVibrantDark { + Self::VibrantDark + } else if name == NSAppearanceNameAqua { + Self::Light + } else if name == NSAppearanceNameDarkAqua { + Self::Dark + } else { + println!( + "unknown appearance: {:?}", + CStr::from_ptr(name.UTF8String()) + ); + Self::Light + } + } +} + +#[link(name = "AppKit", kind = "framework")] +extern "C" { + pub static NSAppearanceNameAqua: id; + pub static NSAppearanceNameDarkAqua: id; +} diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index d1a3b7a79792c151a59784ae455e5e177f5319b6..29e6b58e59e37ad2e2b9f4c575ecc3fe29c0c579 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -52,6 +52,11 @@ use time::UtcOffset; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; +#[allow(non_upper_case_globals)] +pub const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; +#[allow(non_upper_case_globals)] +pub const NSKeyValueObservingOptionNew: NSInteger = 1; + const MAC_PLATFORM_IVAR: &str = "platform"; static mut APP_CLASS: *const Class = ptr::null(); static mut APP_DELEGATE_CLASS: *const Class = ptr::null(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index c369ab424523938b45c515f6d19d8e77a58af3e3..a6570e489025318cbcd2de42eed907895c4daf71 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,12 +1,18 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, - platform::{self, mac::renderer::Renderer}, + platform::{ + self, + mac::{ + platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + renderer::Renderer, + }, + }, Event, FontSystem, Scene, }; use cocoa::{ appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSAutoreleasePool, NSPoint, NSRect, NSSize}, + foundation::{NSPoint, NSRect, NSSize, NSString}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -15,7 +21,7 @@ use objc::{ declare::ClassDecl, msg_send, rc::StrongPtr, - runtime::{Class, Object, Sel}, + runtime::{Class, Object, Protocol, Sel}, sel, sel_impl, }; use std::{ @@ -77,6 +83,20 @@ unsafe fn build_classes() { sel!(flagsChanged:), handle_view_event as extern "C" fn(&Object, Sel, id), ); + decl.add_method( + sel!(makeBackingLayer), + make_backing_layer as extern "C" fn(&Object, Sel) -> id, + ); + decl.add_method( + sel!(observeValueForKeyPath:ofObject:change:context:), + appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + ); + + decl.add_protocol(Protocol::get("CALayerDelegate").unwrap()); + decl.add_method( + sel!(displayLayer:), + display_layer as extern "C" fn(&Object, Sel, id), + ); decl.register() }; @@ -86,15 +106,16 @@ pub struct StatusItem(Rc>); struct StatusItemState { native_item: StrongPtr, + native_view: StrongPtr, renderer: Renderer, + scene: Option, event_callback: Option bool>>, + appearance_changed_callback: Option>, } impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { - let pool = NSAutoreleasePool::new(nil); - let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = @@ -103,39 +124,51 @@ impl StatusItem { let button = native_item.button(); let _: () = msg_send![button, setHidden: YES]; - let item = Self(Rc::new_cyclic(|state| { - let parent_view = button.superview().superview(); - - let view: id = msg_send![VIEW_CLASS, alloc]; - NSView::initWithFrame_( - view, - NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), - ); - view.setWantsBestResolutionOpenGLSurface_(YES); - view.setLayer(renderer.layer().as_ptr() as id); - view.setWantsLayer(true); - (*view).set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); - parent_view.addSubview_(view.autorelease()); - - RefCell::new(StatusItemState { - native_item, - renderer, - event_callback: None, - }) + let native_view = msg_send![VIEW_CLASS, alloc]; + let state = Rc::new(RefCell::new(StatusItemState { + native_item, + native_view: StrongPtr::new(native_view), + renderer, + scene: None, + event_callback: None, + appearance_changed_callback: None, })); + let parent_view = button.superview().superview(); + NSView::initWithFrame_( + native_view, + NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), + ); + (*native_view).set_ivar( + STATE_IVAR, + Weak::into_raw(Rc::downgrade(&state)) as *const c_void, + ); + native_view.setWantsBestResolutionOpenGLSurface_(YES); + native_view.setWantsLayer(true); + let _: () = msg_send![ + native_view, + setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize + ]; + let _: () = msg_send![ + button, + addObserver: native_view + forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") + options: NSKeyValueObservingOptionNew + context: nil + ]; + + parent_view.addSubview_(native_view); + { - let item = item.0.borrow(); - let layer = item.renderer.layer(); - let scale_factor = item.scale_factor(); - let size = item.size() * scale_factor; + let state = state.borrow(); + let layer = state.renderer.layer(); + let scale_factor = state.scale_factor(); + let size = state.size() * scale_factor; layer.set_contents_scale(scale_factor.into()); layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } - pool.drain(); - - item + Self(state) } } } @@ -149,6 +182,10 @@ impl platform::Window for StatusItem { self.0.borrow_mut().event_callback = Some(callback); } + fn on_appearance_changed(&mut self, callback: Box) { + self.0.borrow_mut().appearance_changed_callback = Some(callback); + } + fn on_active_status_change(&mut self, _: Box) { unimplemented!() } @@ -223,7 +260,18 @@ impl platform::Window for StatusItem { } fn present_scene(&mut self, scene: Scene) { - self.0.borrow_mut().renderer.render(&scene); + self.0.borrow_mut().scene = Some(scene); + unsafe { + let _: () = msg_send![*self.0.borrow().native_view, setNeedsDisplay: YES]; + } + } + + fn appearance(&self) -> crate::Appearance { + unsafe { + let appearance: id = + msg_send![self.0.borrow().native_item.button(), effectiveAppearance]; + crate::Appearance::from_native(appearance) + } } } @@ -247,6 +295,7 @@ impl StatusItemState { extern "C" fn dealloc_view(this: &Object, _: Sel) { unsafe { drop_state(this); + let _: () = msg_send![super(this, class!(NSView)), dealloc]; } } @@ -266,6 +315,39 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { } } +extern "C" fn make_backing_layer(this: &Object, _: Sel) -> id { + if let Some(state) = unsafe { get_state(this).upgrade() } { + let state = state.borrow(); + state.renderer.layer().as_ptr() as id + } else { + nil + } +} + +extern "C" fn display_layer(this: &Object, _: Sel, _: id) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state = state.borrow_mut(); + if let Some(scene) = state.scene.take() { + state.renderer.render(&scene); + } + } + } +} + +extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state_borrow = state.as_ref().borrow_mut(); + if let Some(mut callback) = state_borrow.appearance_changed_callback.take() { + drop(state_borrow); + callback(); + state.borrow_mut().appearance_changed_callback = Some(callback); + } + } + } +} + unsafe fn get_state(object: &Object) -> Weak> { let raw: *mut c_void = *object.get_ivar(STATE_IVAR); let weak1 = Weak::from_raw(raw as *mut RefCell); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 4abee8b96163a8caed3d8dc7dd8c68a3f74e6b64..5cb45719eb300b9099bd7478aeaba8dc289bc5ee 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1,4 +1,3 @@ -use super::{geometry::RectFExt, renderer::Renderer}; use crate::{ executor, geometry::{ @@ -6,7 +5,12 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - platform::{self, Event, WindowBounds}, + mac::platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + platform::{ + self, + mac::{geometry::RectFExt, renderer::Renderer}, + Event, WindowBounds, + }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, MouseMovedEvent, Scene, }; @@ -102,9 +106,6 @@ unsafe impl objc::Encode for NSRange { } } -#[allow(non_upper_case_globals)] -const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; - #[ctor] unsafe fn build_classes() { WINDOW_CLASS = { @@ -264,6 +265,10 @@ unsafe fn build_classes() { attributed_substring_for_proposed_range as extern "C" fn(&Object, Sel, NSRange, *mut c_void) -> id, ); + decl.add_method( + sel!(observeValueForKeyPath:ofObject:change:context:), + appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + ); // Suppress beep on keystrokes with modifier keys. decl.add_method( @@ -298,6 +303,7 @@ struct WindowState { fullscreen_callback: Option>, should_close_callback: Option bool>>, close_callback: Option>, + appearance_changed_callback: Option>, input_handler: Option>, pending_key_down: Option<(KeyDownEvent, Option)>, performed_key_equivalent: bool, @@ -376,6 +382,7 @@ impl Window { close_callback: None, activate_callback: None, fullscreen_callback: None, + appearance_changed_callback: None, input_handler: None, pending_key_down: None, performed_key_equivalent: false, @@ -433,6 +440,13 @@ impl Window { native_window.center(); native_window.makeKeyAndOrderFront_(nil); + let _: () = msg_send![ + native_window, + addObserver: native_view + forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") + options: NSKeyValueObservingOptionNew + context: nil + ]; window.0.borrow().move_traffic_light(); pool.drain(); @@ -634,6 +648,17 @@ impl platform::Window for Window { fn titlebar_height(&self) -> f32 { self.0.as_ref().borrow().titlebar_height() } + + fn appearance(&self) -> crate::Appearance { + unsafe { + let appearance: id = msg_send![self.0.borrow().native_window, effectiveAppearance]; + crate::Appearance::from_native(appearance) + } + } + + fn on_appearance_changed(&mut self, callback: Box) { + self.0.borrow_mut().appearance_changed_callback = Some(callback); + } } impl WindowState { @@ -1270,6 +1295,18 @@ extern "C" fn do_command_by_selector(this: &Object, _: Sel, _: Sel) { } } +extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { + unsafe { + let state = get_window_state(this); + let mut state_borrow = state.as_ref().borrow_mut(); + if let Some(mut callback) = state_borrow.appearance_changed_callback.take() { + drop(state_borrow); + callback(); + state.borrow_mut().appearance_changed_callback = Some(callback); + } + } +} + async fn synthetic_drag( window_state: Weak>, drag_id: usize, diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 0ce44f08173b7784f03640e134ac85307d655d2b..9a4f3638bec50963d16020b5339149259bbe13a9 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -298,6 +298,12 @@ impl super::Window for Window { fn present_scene(&mut self, scene: crate::Scene) { self.current_scene = Some(scene); } + + fn appearance(&self) -> crate::Appearance { + crate::Appearance::Light + } + + fn on_appearance_changed(&mut self, _: Box) {} } pub fn platform() -> Platform { diff --git a/crates/gpui/src/presenter.rs b/crates/gpui/src/presenter.rs index 859ba463754cade5e4af05847cb4d0445adba9ee..2943fda647914452fe2a307f06a8c0955a3f9eaa 100644 --- a/crates/gpui/src/presenter.rs +++ b/crates/gpui/src/presenter.rs @@ -11,10 +11,10 @@ use crate::{ HoverRegionEvent, MouseRegionEvent, MoveRegionEvent, UpOutRegionEvent, UpRegionEvent, }, text_layout::TextLayoutCache, - Action, AnyModelHandle, AnyViewHandle, AnyWeakModelHandle, AssetCache, ElementBox, Entity, - FontSystem, ModelHandle, MouseButton, MouseMovedEvent, MouseRegion, MouseRegionId, ParentId, - ReadModel, ReadView, RenderContext, RenderParams, Scene, UpgradeModelHandle, UpgradeViewHandle, - View, ViewHandle, WeakModelHandle, WeakViewHandle, + Action, AnyModelHandle, AnyViewHandle, AnyWeakModelHandle, Appearance, AssetCache, ElementBox, + Entity, FontSystem, ModelHandle, MouseButton, MouseMovedEvent, MouseRegion, MouseRegionId, + ParentId, ReadModel, ReadView, RenderContext, RenderParams, Scene, UpgradeModelHandle, + UpgradeViewHandle, View, ViewHandle, WeakModelHandle, WeakViewHandle, }; use collections::{HashMap, HashSet}; use pathfinder_geometry::vector::{vec2f, Vector2F}; @@ -40,12 +40,14 @@ pub struct Presenter { clicked_button: Option, mouse_position: Vector2F, titlebar_height: f32, + appearance: Appearance, } impl Presenter { pub fn new( window_id: usize, titlebar_height: f32, + appearance: Appearance, font_cache: Arc, text_layout_cache: TextLayoutCache, asset_cache: Arc, @@ -53,7 +55,7 @@ impl Presenter { ) -> Self { Self { window_id, - rendered_views: cx.render_views(window_id, titlebar_height), + rendered_views: cx.render_views(window_id, titlebar_height, appearance), cursor_regions: Default::default(), mouse_regions: Default::default(), font_cache, @@ -65,15 +67,18 @@ impl Presenter { clicked_button: None, mouse_position: vec2f(0., 0.), titlebar_height, + appearance, } } pub fn invalidate( &mut self, invalidation: &mut WindowInvalidation, + appearance: Appearance, cx: &mut MutableAppContext, ) { cx.start_frame(); + self.appearance = appearance; for view_id in &invalidation.removed { invalidation.updated.remove(view_id); self.rendered_views.remove(view_id); @@ -96,14 +101,20 @@ impl Presenter { ) }), refreshing: false, + appearance, }) .unwrap(), ); } } - pub fn refresh(&mut self, invalidation: &mut WindowInvalidation, cx: &mut MutableAppContext) { - self.invalidate(invalidation, cx); + pub fn refresh( + &mut self, + invalidation: &mut WindowInvalidation, + appearance: Appearance, + cx: &mut MutableAppContext, + ) { + self.invalidate(invalidation, appearance, cx); for (view_id, view) in &mut self.rendered_views { if !invalidation.updated.contains(view_id) { *view = cx @@ -122,6 +133,7 @@ impl Presenter { ) }), refreshing: true, + appearance, }) .unwrap(); } @@ -194,6 +206,7 @@ impl Presenter { ) }), titlebar_height: self.titlebar_height, + appearance: self.appearance, window_size, app: cx, } @@ -545,6 +558,7 @@ pub struct LayoutContext<'a> { pub refreshing: bool, pub window_size: Vector2F, titlebar_height: f32, + appearance: Appearance, hovered_region_ids: HashSet, clicked_region_ids: Option<(Vec, MouseButton)>, } @@ -619,6 +633,7 @@ impl<'a> LayoutContext<'a> { hovered_region_ids: self.hovered_region_ids.clone(), clicked_region_ids: self.clicked_region_ids.clone(), refreshing: self.refreshing, + appearance: self.appearance, }; f(view, &mut render_cx) }) From 9b8492a3ba7c5befc22b08040528f0cdeb83a2a0 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 11:58:05 +0200 Subject: [PATCH 65/89] Extract a common `App::register_platform_window` --- crates/gpui/src/app.rs | 189 +++++++++----------- crates/gpui/src/platform/mac/status_item.rs | 24 +-- 2 files changed, 86 insertions(+), 127 deletions(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 0baff67e94fd96479218baa4e7be8da063cac1a2..c8e8efec7980c475947b3919d98df08ad972dda7 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1924,84 +1924,11 @@ impl MutableAppContext { ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - let mut window = + let window = this.cx .platform .open_window(window_id, window_options, this.foreground.clone()); - let presenter = Rc::new(RefCell::new(this.build_presenter( - window_id, - window.titlebar_height(), - window.appearance(), - ))); - - { - let mut app = this.upgrade(); - let presenter = Rc::downgrade(&presenter); - window.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { - if cx.dispatch_keystroke(window_id, keystroke) { - return true; - } - } - - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false - } - }) - })); - } - - { - let mut app = this.upgrade(); - window.on_active_status_change(Box::new(move |is_active| { - app.update(|cx| cx.window_changed_active_status(window_id, is_active)) - })); - } - - { - let mut app = this.upgrade(); - window.on_resize(Box::new(move || { - app.update(|cx| cx.window_was_resized(window_id)) - })); - } - - { - let mut app = this.upgrade(); - window.on_fullscreen(Box::new(move |is_fullscreen| { - app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) - })); - } - - { - let mut app = this.upgrade(); - window.on_close(Box::new(move || { - app.update(|cx| cx.remove_window(window_id)); - })); - } - - { - let mut app = this.upgrade(); - window - .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); - } - - window.set_input_handler(Box::new(WindowInputHandler { - app: this.upgrade().0, - window_id, - })); - - let scene = presenter.borrow_mut().build_scene( - window.size(), - window.scale_factor(), - false, - this, - ); - window.present_scene(scene); - this.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), window)); + this.register_platform_window(window_id, window); (window_id, root_view) }) @@ -2029,45 +1956,89 @@ impl MutableAppContext { ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - let mut status_item = this.cx.platform.add_status_item(); - let presenter = Rc::new(RefCell::new(this.build_presenter( - window_id, - 0., - status_item.appearance(), - ))); + let status_item = this.cx.platform.add_status_item(); + this.register_platform_window(window_id, status_item); - { - let mut app = this.upgrade(); - let presenter = Rc::downgrade(&presenter); - status_item.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false + (window_id, root_view) + }) + } + + fn register_platform_window( + &mut self, + window_id: usize, + mut window: Box, + ) { + let presenter = Rc::new(RefCell::new(self.build_presenter( + window_id, + window.titlebar_height(), + window.appearance(), + ))); + + { + let mut app = self.upgrade(); + let presenter = Rc::downgrade(&presenter); + window.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { + if cx.dispatch_keystroke(window_id, keystroke) { + return true; + } } - }) - })); - } - { - let mut app = this.upgrade(); - status_item - .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); - } + presenter.borrow_mut().dispatch_event(event, false, cx) + } else { + false + } + }) + })); + } - let scene = presenter.borrow_mut().build_scene( - status_item.size(), - status_item.scale_factor(), - false, - this, - ); - status_item.present_scene(scene); - this.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), status_item)); + { + let mut app = self.upgrade(); + window.on_active_status_change(Box::new(move |is_active| { + app.update(|cx| cx.window_changed_active_status(window_id, is_active)) + })); + } - (window_id, root_view) - }) + { + let mut app = self.upgrade(); + window.on_resize(Box::new(move || { + app.update(|cx| cx.window_was_resized(window_id)) + })); + } + + { + let mut app = self.upgrade(); + window.on_fullscreen(Box::new(move |is_fullscreen| { + app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) + })); + } + + { + let mut app = self.upgrade(); + window.on_close(Box::new(move || { + app.update(|cx| cx.remove_window(window_id)); + })); + } + + { + let mut app = self.upgrade(); + window.on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + + window.set_input_handler(Box::new(WindowInputHandler { + app: self.upgrade().0, + window_id, + })); + + let scene = + presenter + .borrow_mut() + .build_scene(window.size(), window.scale_factor(), false, self); + window.present_scene(scene); + self.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), window)); } pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index a6570e489025318cbcd2de42eed907895c4daf71..b99b707bff5f64cbe2a74085b2c69896d6a41132 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -186,29 +186,17 @@ impl platform::Window for StatusItem { self.0.borrow_mut().appearance_changed_callback = Some(callback); } - fn on_active_status_change(&mut self, _: Box) { - unimplemented!() - } + fn on_active_status_change(&mut self, _: Box) {} - fn on_resize(&mut self, _: Box) { - unimplemented!() - } + fn on_resize(&mut self, _: Box) {} - fn on_fullscreen(&mut self, _: Box) { - unimplemented!() - } + fn on_fullscreen(&mut self, _: Box) {} - fn on_should_close(&mut self, _: Box bool>) { - unimplemented!() - } + fn on_should_close(&mut self, _: Box bool>) {} - fn on_close(&mut self, _: Box) { - unimplemented!() - } + fn on_close(&mut self, _: Box) {} - fn set_input_handler(&mut self, _: Box) { - unimplemented!() - } + fn set_input_handler(&mut self, _: Box) {} fn prompt( &self, From d10f6f60adad0b64f511531f3d5bd4ea0b6cf0b8 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 15:43:51 +0200 Subject: [PATCH 66/89] Toggle contacts popover when clicking on status bar icon --- .../src/contacts_popover.rs | 26 +++++++ .../src/contacts_status_item.rs | 63 ++++++++++++++-- crates/gpui/src/app.rs | 29 ++++++-- crates/gpui/src/platform.rs | 5 +- crates/gpui/src/platform/mac/status_item.rs | 45 +++++++++--- crates/gpui/src/platform/mac/window.rs | 71 ++++++++++++++----- crates/gpui/src/platform/test.rs | 11 ++- crates/zed/src/main.rs | 1 + crates/zed/src/zed.rs | 1 + 9 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 crates/contacts_status_item/src/contacts_popover.rs diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs new file mode 100644 index 0000000000000000000000000000000000000000..0365a2ee2a62babdc62f000cc5afd451ce384e20 --- /dev/null +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -0,0 +1,26 @@ +use gpui::{color::Color, elements::*, Entity, RenderContext, View}; + +pub struct ContactsPopover; + +impl Entity for ContactsPopover { + type Event = (); +} + +impl View for ContactsPopover { + fn ui_name() -> &'static str { + "ContactsPopover" + } + + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + Empty::new() + .contained() + .with_background_color(Color::red()) + .boxed() + } +} + +impl ContactsPopover { + pub fn new() -> Self { + Self + } +} diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index f3363f18d357a7e24902105fe047c5148cba9804..70e8083a76b63a5e98ed6ca3fee559ac33f98c00 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,6 +1,24 @@ -use gpui::{color::Color, elements::*, Appearance, Entity, RenderContext, View}; +mod contacts_popover; -pub struct ContactsStatusItem; +use contacts_popover::ContactsPopover; +use gpui::{ + actions, + color::Color, + elements::*, + geometry::{rect::RectF, vector::vec2f}, + Appearance, Entity, MouseButton, MutableAppContext, RenderContext, View, ViewContext, + ViewHandle, +}; + +actions!(contacts_status_item, [ToggleContactsPopover]); + +pub fn init(cx: &mut MutableAppContext) { + cx.add_action(ContactsStatusItem::toggle_contacts_popover); +} + +pub struct ContactsStatusItem { + popover: Option>, +} impl Entity for ContactsStatusItem { type Event = (); @@ -16,15 +34,46 @@ impl View for ContactsStatusItem { Appearance::Light | Appearance::VibrantLight => Color::black(), Appearance::Dark | Appearance::VibrantDark => Color::white(), }; - Svg::new("icons/zed_22.svg") - .with_color(color) - .aligned() - .boxed() + MouseEventHandler::new::(0, cx, |_, _| { + Svg::new("icons/zed_22.svg") + .with_color(color) + .aligned() + .boxed() + }) + .on_click(MouseButton::Left, |_, cx| { + cx.dispatch_action(ToggleContactsPopover); + }) + .boxed() } } impl ContactsStatusItem { pub fn new() -> Self { - Self + Self { popover: None } + } + + fn toggle_contacts_popover(&mut self, _: &ToggleContactsPopover, cx: &mut ViewContext) { + match self.popover.take() { + Some(popover) => { + cx.remove_window(popover.window_id()); + } + None => { + let window_bounds = cx.window_bounds(); + let size = vec2f(360., 460.); + let origin = window_bounds.lower_left() + + vec2f(window_bounds.width() / 2. - size.x() / 2., 0.); + self.popover = Some( + cx.add_window( + gpui::WindowOptions { + bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), + titlebar: None, + center: false, + }, + |_| ContactsPopover::new(), + ) + .1, + ); + } + } } } diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index c8e8efec7980c475947b3919d98df08ad972dda7..3ea88995f16a7215a0343e45b05ef5fd03e89e8f 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1244,6 +1244,10 @@ impl MutableAppContext { .map_or(false, |window| window.is_fullscreen) } + pub fn window_bounds(&self, window_id: usize) -> RectF { + self.presenters_and_platform_windows[&window_id].1.bounds() + } + pub fn render_view(&mut self, params: RenderParams) -> Result { let window_id = params.window_id; let view_id = params.view_id; @@ -2032,10 +2036,12 @@ impl MutableAppContext { window_id, })); - let scene = - presenter - .borrow_mut() - .build_scene(window.size(), window.scale_factor(), false, self); + let scene = presenter.borrow_mut().build_scene( + window.content_size(), + window.scale_factor(), + false, + self, + ); window.present_scene(scene); self.presenters_and_platform_windows .insert(window_id, (presenter.clone(), window)); @@ -2411,8 +2417,12 @@ impl MutableAppContext { { let mut presenter = presenter.borrow_mut(); presenter.invalidate(&mut invalidation, window.appearance(), self); - let scene = - presenter.build_scene(window.size(), window.scale_factor(), false, self); + let scene = presenter.build_scene( + window.content_size(), + window.scale_factor(), + false, + self, + ); window.present_scene(scene); } self.presenters_and_platform_windows @@ -2477,7 +2487,8 @@ impl MutableAppContext { window.appearance(), self, ); - let scene = presenter.build_scene(window.size(), window.scale_factor(), true, self); + let scene = + presenter.build_scene(window.content_size(), window.scale_factor(), true, self); window.present_scene(scene); } self.presenters_and_platform_windows = presenters; @@ -3749,6 +3760,10 @@ impl<'a, T: View> ViewContext<'a, T> { self.app.toggle_window_full_screen(self.window_id) } + pub fn window_bounds(&self) -> RectF { + self.app.window_bounds(self.window_id) + } + pub fn prompt( &self, level: PromptLevel, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index d2ae7d8d178efe35f92fc483eaae8f770620ea2b..f2fd09d30b6c91977a2efcb163b9b180f550753a 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -128,7 +128,8 @@ pub trait Window { fn zoom(&self); fn toggle_full_screen(&self); - fn size(&self) -> Vector2F; + fn bounds(&self) -> RectF; + fn content_size(&self) -> Vector2F; fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; fn present_scene(&mut self, scene: Scene); @@ -140,6 +141,7 @@ pub trait Window { pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, + pub center: bool, } #[derive(Debug)] @@ -273,6 +275,7 @@ impl<'a> Default for WindowOptions<'a> { appears_transparent: Default::default(), traffic_light_position: Default::default(), }), + center: false, } } } diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index b99b707bff5f64cbe2a74085b2c69896d6a41132..1ed39894a105fa8d662be96323c315f6ae1270ad 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,5 +1,8 @@ use crate::{ - geometry::vector::{vec2f, Vector2F}, + geometry::{ + rect::RectF, + vector::{vec2f, Vector2F}, + }, platform::{ self, mac::{ @@ -10,7 +13,7 @@ use crate::{ Event, FontSystem, Scene, }; use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, + appkit::{NSScreen, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, foundation::{NSPoint, NSRect, NSSize, NSString}, }; @@ -163,7 +166,7 @@ impl StatusItem { let state = state.borrow(); let layer = state.renderer.layer(); let scale_factor = state.scale_factor(); - let size = state.size() * scale_factor; + let size = state.content_size() * scale_factor; layer.set_contents_scale(scale_factor.into()); layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } @@ -235,8 +238,12 @@ impl platform::Window for StatusItem { unimplemented!() } - fn size(&self) -> Vector2F { - self.0.borrow().size() + fn bounds(&self) -> RectF { + self.0.borrow().bounds() + } + + fn content_size(&self) -> Vector2F { + self.0.borrow().content_size() } fn scale_factor(&self) -> f32 { @@ -264,10 +271,28 @@ impl platform::Window for StatusItem { } impl StatusItemState { - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + unsafe { + let window: id = msg_send![self.native_item.button(), window]; + let screen_frame = window.screen().visibleFrame(); + let window_frame = NSWindow::frame(window); + let origin = vec2f( + window_frame.origin.x as f32, + (window_frame.origin.y - screen_frame.size.height - window_frame.size.height) + as f32, + ); + let size = vec2f( + window_frame.size.width as f32, + window_frame.size.height as f32, + ); + RectF::new(origin, size) + } + } + + fn content_size(&self) -> Vector2F { unsafe { let NSSize { width, height, .. } = - NSWindow::frame(self.native_item.button().superview().superview()).size; + NSView::frame(self.native_item.button().superview().superview()).size; vec2f(width as f32, height as f32) } } @@ -275,7 +300,7 @@ impl StatusItemState { fn scale_factor(&self) -> f32 { unsafe { let window: id = msg_send![self.native_item.button(), window]; - window.screen().backingScaleFactor() as f32 + NSScreen::backingScaleFactor(window.screen()) as f32 } } } @@ -292,7 +317,9 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); - if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { + if let Some(event) = + Event::from_native(native_event, Some(state_borrow.content_size().y())) + { if let Some(mut callback) = state_borrow.event_callback.take() { drop(state_borrow); callback(event); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5cb45719eb300b9099bd7478aeaba8dc289bc5ee..00688f1ea54f3a612fb0b816d8ec6424388d8f22 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -335,12 +335,6 @@ impl Window { unsafe { let pool = NSAutoreleasePool::new(nil); - let frame = match options.bounds { - WindowBounds::Maximized => RectF::new(Default::default(), vec2f(1024., 768.)), - WindowBounds::Fixed(rect) => rect, - } - .to_ns_rect(); - let mut style_mask; if let Some(titlebar) = options.titlebar.as_ref() { style_mask = NSWindowStyleMask::NSClosableWindowMask @@ -357,16 +351,31 @@ impl Window { let native_window: id = msg_send![WINDOW_CLASS, alloc]; let native_window = native_window.initWithContentRect_styleMask_backing_defer_( - frame, + RectF::new(Default::default(), vec2f(1024., 768.)).to_ns_rect(), style_mask, NSBackingStoreBuffered, NO, ); assert!(!native_window.is_null()); - if matches!(options.bounds, WindowBounds::Maximized) { - let screen = native_window.screen(); - native_window.setFrame_display_(screen.visibleFrame(), YES); + let screen = native_window.screen(); + match options.bounds { + WindowBounds::Maximized => { + native_window.setFrame_display_(screen.visibleFrame(), YES); + } + WindowBounds::Fixed(top_left_bounds) => { + let frame = screen.visibleFrame(); + let bottom_left_bounds = RectF::new( + vec2f( + top_left_bounds.origin_x(), + frame.size.height as f32 + - top_left_bounds.origin_y() + - top_left_bounds.height(), + ), + top_left_bounds.size(), + ); + native_window.setFrame_display_(bottom_left_bounds.to_ns_rect(), YES); + } } let native_view: id = msg_send![VIEW_CLASS, alloc]; @@ -438,7 +447,10 @@ impl Window { native_window.setContentView_(native_view.autorelease()); native_window.makeFirstResponder_(native_view); - native_window.center(); + if options.center { + native_window.center(); + } + native_window.makeKeyAndOrderFront_(nil); let _: () = msg_send![ native_window, @@ -633,8 +645,12 @@ impl platform::Window for Window { .detach(); } - fn size(&self) -> Vector2F { - self.0.as_ref().borrow().size() + fn bounds(&self) -> RectF { + self.0.as_ref().borrow().bounds() + } + + fn content_size(&self) -> Vector2F { + self.0.as_ref().borrow().content_size() } fn scale_factor(&self) -> f32 { @@ -706,7 +722,24 @@ impl WindowState { } } - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + unsafe { + let screen_frame = self.native_window.screen().visibleFrame(); + let window_frame = NSWindow::frame(self.native_window); + let origin = vec2f( + window_frame.origin.x as f32, + (window_frame.origin.y - screen_frame.size.height - window_frame.size.height) + as f32, + ); + let size = vec2f( + window_frame.size.width as f32, + window_frame.size.height as f32, + ); + RectF::new(origin, size) + } + } + + fn content_size(&self) -> Vector2F { let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_window.contentView()) }.size; vec2f(width as f32, height as f32) @@ -783,7 +816,8 @@ extern "C" fn handle_key_event(this: &Object, native_event: id, key_equivalent: let mut window_state_borrow = window_state.as_ref().borrow_mut(); - let event = unsafe { Event::from_native(native_event, Some(window_state_borrow.size().y())) }; + let event = + unsafe { Event::from_native(native_event, Some(window_state_borrow.content_size().y())) }; if let Some(event) = event { if key_equivalent { @@ -875,7 +909,8 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { let weak_window_state = Rc::downgrade(&window_state); let mut window_state_borrow = window_state.as_ref().borrow_mut(); - let event = unsafe { Event::from_native(native_event, Some(window_state_borrow.size().y())) }; + let event = + unsafe { Event::from_native(native_event, Some(window_state_borrow.content_size().y())) }; if let Some(event) = event { match &event { Event::MouseMoved( @@ -1060,7 +1095,7 @@ extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { unsafe { let scale_factor = window_state_borrow.scale_factor() as f64; - let size = window_state_borrow.size(); + let size = window_state_borrow.content_size(); let drawable_size: NSSize = NSSize { width: size.x() as f64 * scale_factor, height: size.y() as f64 * scale_factor, @@ -1087,7 +1122,7 @@ extern "C" fn set_frame_size(this: &Object, _: Sel, size: NSSize) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.as_ref().borrow(); - if window_state_borrow.size() == vec2f(size.width as f32, size.height as f32) { + if window_state_borrow.content_size() == vec2f(size.width as f32, size.height as f32) { return; } diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 9a4f3638bec50963d16020b5339149259bbe13a9..9a458a1dd96609c70c6acc4ffed2e69b6e44faa3 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -1,6 +1,9 @@ use super::{AppVersion, CursorStyle, WindowBounds}; use crate::{ - geometry::vector::{vec2f, Vector2F}, + geometry::{ + rect::RectF, + vector::{vec2f, Vector2F}, + }, keymap, Action, ClipboardItem, }; use anyhow::{anyhow, Result}; @@ -283,7 +286,11 @@ impl super::Window for Window { fn toggle_full_screen(&self) {} - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + RectF::new(Default::default(), self.size) + } + + fn content_size(&self) -> Vector2F { self.size } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index 287ac721fdc7e7d30dca312ce74fa42140e59b9d..868ad15d675e0c8af470cccdd16a8cfaeff3d1de 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -105,6 +105,7 @@ fn main() { watch_settings_file(default_settings, settings_file, themes.clone(), cx); watch_keymap_file(keymap_file, cx); + contacts_status_item::init(cx); context_menu::init(cx); project::Project::init(&client); client::Channel::init(&client); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index 59e9f8de9daa489bf11d3d002aae8fcbe69d29ce..a14e85e93b8c0cabfdc69e92a41d61c59ef039cb 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -335,6 +335,7 @@ pub fn build_window_options() -> WindowOptions<'static> { appears_transparent: true, traffic_light_position: Some(vec2f(8., 8.)), }), + center: false, } } From c1f448d8a88aef17163d05bce4fc092de103ea57 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 15:49:08 +0200 Subject: [PATCH 67/89] Use `viewDidChangeEffectiveAppearance` to detect appearance changes --- crates/gpui/src/platform/mac/platform.rs | 2 -- crates/gpui/src/platform/mac/status_item.rs | 20 +++++--------------- crates/gpui/src/platform/mac/window.rs | 15 ++++----------- 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 29e6b58e59e37ad2e2b9f4c575ecc3fe29c0c579..9169e72d06a43b1c05efcf8735b2bb276211319f 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -54,8 +54,6 @@ const NSUTF8StringEncoding: NSUInteger = 4; #[allow(non_upper_case_globals)] pub const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; -#[allow(non_upper_case_globals)] -pub const NSKeyValueObservingOptionNew: NSInteger = 1; const MAC_PLATFORM_IVAR: &str = "platform"; static mut APP_CLASS: *const Class = ptr::null(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 1ed39894a105fa8d662be96323c315f6ae1270ad..404b5e2ef4649926b40a080c28aa498352920895 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,17 +5,14 @@ use crate::{ }, platform::{ self, - mac::{ - platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, - renderer::Renderer, - }, + mac::{platform::NSViewLayerContentsRedrawDuringViewResize, renderer::Renderer}, }, Event, FontSystem, Scene, }; use cocoa::{ appkit::{NSScreen, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSPoint, NSRect, NSSize, NSString}, + foundation::{NSPoint, NSRect, NSSize}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -91,8 +88,8 @@ unsafe fn build_classes() { make_backing_layer as extern "C" fn(&Object, Sel) -> id, ); decl.add_method( - sel!(observeValueForKeyPath:ofObject:change:context:), - appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + sel!(viewDidChangeEffectiveAppearance), + view_did_change_effective_appearance as extern "C" fn(&Object, Sel), ); decl.add_protocol(Protocol::get("CALayerDelegate").unwrap()); @@ -152,13 +149,6 @@ impl StatusItem { native_view, setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize ]; - let _: () = msg_send![ - button, - addObserver: native_view - forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") - options: NSKeyValueObservingOptionNew - context: nil - ]; parent_view.addSubview_(native_view); @@ -350,7 +340,7 @@ extern "C" fn display_layer(this: &Object, _: Sel, _: id) { } } -extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { +extern "C" fn view_did_change_effective_appearance(this: &Object, _: Sel) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 00688f1ea54f3a612fb0b816d8ec6424388d8f22..2bdd5a720ccd2c72f56af5969076c70672f347db 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -5,7 +5,7 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - mac::platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + mac::platform::NSViewLayerContentsRedrawDuringViewResize, platform::{ self, mac::{geometry::RectFExt, renderer::Renderer}, @@ -266,8 +266,8 @@ unsafe fn build_classes() { as extern "C" fn(&Object, Sel, NSRange, *mut c_void) -> id, ); decl.add_method( - sel!(observeValueForKeyPath:ofObject:change:context:), - appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + sel!(viewDidChangeEffectiveAppearance), + view_did_change_effective_appearance as extern "C" fn(&Object, Sel), ); // Suppress beep on keystrokes with modifier keys. @@ -452,13 +452,6 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); - let _: () = msg_send![ - native_window, - addObserver: native_view - forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") - options: NSKeyValueObservingOptionNew - context: nil - ]; window.0.borrow().move_traffic_light(); pool.drain(); @@ -1330,7 +1323,7 @@ extern "C" fn do_command_by_selector(this: &Object, _: Sel, _: Sel) { } } -extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { +extern "C" fn view_did_change_effective_appearance(this: &Object, _: Sel) { unsafe { let state = get_window_state(this); let mut state_borrow = state.as_ref().borrow_mut(); From 1c9c7ef7ae75e8051f76e124d9d9396d3d904e67 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 17:09:07 +0200 Subject: [PATCH 68/89] Add the ability to specify a level when creating windows This lets some windows stay on top of others, independently of whether the application is in the foreground. --- .../src/contacts_status_item.rs | 19 +++++++++---------- crates/gpui/src/platform.rs | 8 ++++++++ crates/gpui/src/platform/mac/window.rs | 13 ++++++++++++- crates/zed/src/zed.rs | 3 ++- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 70e8083a76b63a5e98ed6ca3fee559ac33f98c00..33ca61ce6c2a190ccb8750d7df25886b3a8d353d 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -62,17 +62,16 @@ impl ContactsStatusItem { let size = vec2f(360., 460.); let origin = window_bounds.lower_left() + vec2f(window_bounds.width() / 2. - size.x() / 2., 0.); - self.popover = Some( - cx.add_window( - gpui::WindowOptions { - bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), - titlebar: None, - center: false, - }, - |_| ContactsPopover::new(), - ) - .1, + let (_, popover) = cx.add_window( + gpui::WindowOptions { + bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), + titlebar: None, + center: false, + level: gpui::WindowLevel::PopUp, + }, + |_| ContactsPopover::new(), ); + self.popover = Some(popover); } } } diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index f2fd09d30b6c91977a2efcb163b9b180f550753a..87c44ae39015bdec5d0fea9909161290d2be0b71 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -142,6 +142,7 @@ pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, pub center: bool, + pub level: WindowLevel, } #[derive(Debug)] @@ -165,6 +166,12 @@ impl Default for Appearance { } } +#[derive(Copy, Clone, Debug)] +pub enum WindowLevel { + Normal, + PopUp, +} + #[derive(Debug)] pub enum WindowBounds { Maximized, @@ -276,6 +283,7 @@ impl<'a> Default for WindowOptions<'a> { traffic_light_position: Default::default(), }), center: false, + level: WindowLevel::Normal, } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 2bdd5a720ccd2c72f56af5969076c70672f347db..90dabfff4e49f34c9773c658b4d064cd20111202 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -12,7 +12,7 @@ use crate::{ Event, WindowBounds, }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, - MouseMovedEvent, Scene, + MouseMovedEvent, Scene, WindowLevel, }; use block::ConcreteBlock; use cocoa::{ @@ -56,6 +56,12 @@ const WINDOW_STATE_IVAR: &str = "windowState"; static mut WINDOW_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); +#[allow(non_upper_case_globals)] +const NSNormalWindowLevel: NSInteger = 0; + +#[allow(non_upper_case_globals)] +const NSPopUpWindowLevel: NSInteger = 101; + #[repr(C)] #[derive(Copy, Clone, Debug)] struct NSRange { @@ -452,6 +458,11 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); + let native_level = match options.level { + WindowLevel::Normal => NSNormalWindowLevel, + WindowLevel::PopUp => NSPopUpWindowLevel, + }; + native_window.setLevel_(native_level); window.0.borrow().move_traffic_light(); pool.drain(); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index a14e85e93b8c0cabfdc69e92a41d61c59ef039cb..077282f526c30db219e757f6ee15c6603330ca3f 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowLevel, }; use language::Rope; pub use lsp; @@ -336,6 +336,7 @@ pub fn build_window_options() -> WindowOptions<'static> { traffic_light_position: Some(vec2f(8., 8.)), }), center: false, + level: WindowLevel::Normal, } } From 44553875d0ec0e5445b428e5c0339054410b8800 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 11:44:51 +0200 Subject: [PATCH 69/89] Allow contacts popover to be activated even if app isn't foregrounded --- .../src/contacts_popover.rs | 18 ++- .../src/contacts_status_item.rs | 22 ++- crates/gpui/src/platform.rs | 8 +- crates/gpui/src/platform/mac/window.rs | 135 ++++++++++-------- crates/zed/src/zed.rs | 5 +- 5 files changed, 118 insertions(+), 70 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index 0365a2ee2a62babdc62f000cc5afd451ce384e20..7637b51cc9bc10f54661ea30e049350952675575 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,9 +1,13 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View}; +use gpui::{color::Color, elements::*, Entity, RenderContext, View, ViewContext}; + +pub enum Event { + Deactivated, +} pub struct ContactsPopover; impl Entity for ContactsPopover { - type Event = (); + type Event = Event; } impl View for ContactsPopover { @@ -20,7 +24,15 @@ impl View for ContactsPopover { } impl ContactsPopover { - pub fn new() -> Self { + pub fn new(cx: &mut ViewContext) -> Self { + cx.observe_window_activation(Self::window_activation_changed) + .detach(); Self } + + fn window_activation_changed(&mut self, is_active: bool, cx: &mut ViewContext) { + if !is_active { + cx.emit(Event::Deactivated); + } + } } diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 33ca61ce6c2a190ccb8750d7df25886b3a8d353d..83cdce7e8e84754f3214b7c8c41332aa65ac265e 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -7,7 +7,7 @@ use gpui::{ elements::*, geometry::{rect::RectF, vector::vec2f}, Appearance, Entity, MouseButton, MutableAppContext, RenderContext, View, ViewContext, - ViewHandle, + ViewHandle, WindowKind, }; actions!(contacts_status_item, [ToggleContactsPopover]); @@ -67,12 +67,28 @@ impl ContactsStatusItem { bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), titlebar: None, center: false, - level: gpui::WindowLevel::PopUp, + kind: WindowKind::PopUp, + is_movable: false, }, - |_| ContactsPopover::new(), + |cx| ContactsPopover::new(cx), ); + cx.subscribe(&popover, Self::on_popover_event).detach(); self.popover = Some(popover); } } } + + fn on_popover_event( + &mut self, + popover: ViewHandle, + event: &contacts_popover::Event, + cx: &mut ViewContext, + ) { + match event { + contacts_popover::Event::Deactivated => { + self.popover.take(); + cx.remove_window(popover.window_id()); + } + } + } } diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index 87c44ae39015bdec5d0fea9909161290d2be0b71..89e55c9f6f0ad815c1696db19fc1a46e60c9e01f 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -142,7 +142,8 @@ pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, pub center: bool, - pub level: WindowLevel, + pub kind: WindowKind, + pub is_movable: bool, } #[derive(Debug)] @@ -167,7 +168,7 @@ impl Default for Appearance { } #[derive(Copy, Clone, Debug)] -pub enum WindowLevel { +pub enum WindowKind { Normal, PopUp, } @@ -283,7 +284,8 @@ impl<'a> Default for WindowOptions<'a> { traffic_light_position: Default::default(), }), center: false, - level: WindowLevel::Normal, + kind: WindowKind::Normal, + is_movable: true, } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 90dabfff4e49f34c9773c658b4d064cd20111202..f21428a1a83ffd78b0cc9833d0d6c3bdcfb046c0 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -12,7 +12,7 @@ use crate::{ Event, WindowBounds, }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, - MouseMovedEvent, Scene, WindowLevel, + MouseMovedEvent, Scene, WindowKind, }; use block::ConcreteBlock; use cocoa::{ @@ -54,6 +54,7 @@ use std::{ const WINDOW_STATE_IVAR: &str = "windowState"; static mut WINDOW_CLASS: *const Class = ptr::null(); +static mut PANEL_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); #[allow(non_upper_case_globals)] @@ -114,50 +115,8 @@ unsafe impl objc::Encode for NSRange { #[ctor] unsafe fn build_classes() { - WINDOW_CLASS = { - let mut decl = ClassDecl::new("GPUIWindow", class!(NSWindow)).unwrap(); - decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); - decl.add_method(sel!(dealloc), dealloc_window as extern "C" fn(&Object, Sel)); - decl.add_method( - sel!(canBecomeMainWindow), - yes as extern "C" fn(&Object, Sel) -> BOOL, - ); - decl.add_method( - sel!(canBecomeKeyWindow), - yes as extern "C" fn(&Object, Sel) -> BOOL, - ); - decl.add_method( - sel!(sendEvent:), - send_event as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidResize:), - window_did_resize as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowWillEnterFullScreen:), - window_will_enter_fullscreen as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowWillExitFullScreen:), - window_will_exit_fullscreen as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidBecomeKey:), - window_did_change_key_status as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidResignKey:), - window_did_change_key_status as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowShouldClose:), - window_should_close as extern "C" fn(&Object, Sel, id) -> BOOL, - ); - decl.add_method(sel!(close), close_window as extern "C" fn(&Object, Sel)); - decl.register() - }; - + WINDOW_CLASS = build_window_class("GPUIWindow", class!(NSWindow)); + PANEL_CLASS = build_window_class("GPUIPanel", class!(NSPanel)); VIEW_CLASS = { let mut decl = ClassDecl::new("GPUIView", class!(NSView)).unwrap(); decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); @@ -286,6 +245,50 @@ unsafe fn build_classes() { }; } +unsafe fn build_window_class(name: &'static str, superclass: &Class) -> *const Class { + let mut decl = ClassDecl::new(name, superclass).unwrap(); + decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); + decl.add_method(sel!(dealloc), dealloc_window as extern "C" fn(&Object, Sel)); + decl.add_method( + sel!(canBecomeMainWindow), + yes as extern "C" fn(&Object, Sel) -> BOOL, + ); + decl.add_method( + sel!(canBecomeKeyWindow), + yes as extern "C" fn(&Object, Sel) -> BOOL, + ); + decl.add_method( + sel!(sendEvent:), + send_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidResize:), + window_did_resize as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowWillEnterFullScreen:), + window_will_enter_fullscreen as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowWillExitFullScreen:), + window_will_exit_fullscreen as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidBecomeKey:), + window_did_change_key_status as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidResignKey:), + window_did_change_key_status as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowShouldClose:), + window_should_close as extern "C" fn(&Object, Sel, id) -> BOOL, + ); + decl.add_method(sel!(close), close_window as extern "C" fn(&Object, Sel)); + decl.register() +} + pub struct Window(Rc>); ///Used to track what the IME does when we send it a keystroke. @@ -352,10 +355,21 @@ impl Window { style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; } } else { - style_mask = NSWindowStyleMask::empty(); + style_mask = NSWindowStyleMask::NSTitledWindowMask + | NSWindowStyleMask::NSFullSizeContentViewWindowMask; } - let native_window: id = msg_send![WINDOW_CLASS, alloc]; + let native_window: id = match options.kind { + WindowKind::Normal => msg_send![WINDOW_CLASS, alloc], + WindowKind::PopUp => { + #[allow(non_upper_case_globals)] + const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = + unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; + + style_mask |= NSWindowStyleMaskNonactivatingPanel; + msg_send![PANEL_CLASS, alloc] + } + }; let native_window = native_window.initWithContentRect_styleMask_backing_defer_( RectF::new(Default::default(), vec2f(1024., 768.)).to_ns_rect(), style_mask, @@ -425,13 +439,17 @@ impl Window { Rc::into_raw(window.0.clone()) as *const c_void, ); - if let Some(titlebar) = options.titlebar { - if let Some(title) = titlebar.title { - native_window.setTitle_(NSString::alloc(nil).init_str(title)); - } - if titlebar.appears_transparent { - native_window.setTitlebarAppearsTransparent_(YES); - } + if let Some(title) = options.titlebar.as_ref().and_then(|t| t.title) { + native_window.setTitle_(NSString::alloc(nil).init_str(title)); + } + + native_window.setMovable_(options.is_movable); + + if options + .titlebar + .map_or(true, |titlebar| titlebar.appears_transparent) + { + native_window.setTitlebarAppearsTransparent_(YES); } native_window.setAcceptsMouseMovedEvents_(YES); @@ -458,11 +476,10 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); - let native_level = match options.level { - WindowLevel::Normal => NSNormalWindowLevel, - WindowLevel::PopUp => NSPopUpWindowLevel, - }; - native_window.setLevel_(native_level); + match options.kind { + WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), + WindowKind::PopUp => native_window.setLevel_(NSPopUpWindowLevel), + } window.0.borrow().move_traffic_light(); pool.drain(); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index 077282f526c30db219e757f6ee15c6603330ca3f..bee77d2d034e62aa94a56fccc5c3a5ce52084500 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowLevel, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowKind, }; use language::Rope; pub use lsp; @@ -336,7 +336,8 @@ pub fn build_window_options() -> WindowOptions<'static> { traffic_light_position: Some(vec2f(8., 8.)), }), center: false, - level: WindowLevel::Normal, + kind: WindowKind::Normal, + is_movable: true, } } From 0c422fadb8315cab5d16c7a846c1b9925a7b60eb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 11:57:22 +0200 Subject: [PATCH 70/89] Style contacts popover background based on theme --- crates/contacts_status_item/src/contacts_popover.rs | 6 ++++-- crates/theme/src/theme.rs | 6 ++++++ styles/src/styleTree/app.ts | 2 ++ styles/src/styleTree/contactsPopover.ts | 8 ++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 styles/src/styleTree/contactsPopover.ts diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index 7637b51cc9bc10f54661ea30e049350952675575..cd0e361f5be3c5ec6460e51f60e76e83cc934dd5 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,4 +1,5 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View, ViewContext}; +use gpui::{elements::*, Entity, RenderContext, View, ViewContext}; +use settings::Settings; pub enum Event { Deactivated, @@ -16,9 +17,10 @@ impl View for ContactsPopover { } fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + let theme = &cx.global::().theme.contacts_popover; Empty::new() .contained() - .with_background_color(Color::red()) + .with_background_color(theme.background) .boxed() } } diff --git a/crates/theme/src/theme.rs b/crates/theme/src/theme.rs index 9d90b44402c41d0128af41669c4eb19ad9e827c9..244a2e6aecd5c635f75f4af681394bf02109b99c 100644 --- a/crates/theme/src/theme.rs +++ b/crates/theme/src/theme.rs @@ -19,6 +19,7 @@ pub struct Theme { pub workspace: Workspace, pub context_menu: ContextMenu, pub chat_panel: ChatPanel, + pub contacts_popover: ContactsPopover, pub contacts_panel: ContactsPanel, pub contact_finder: ContactFinder, pub project_panel: ProjectPanel, @@ -301,6 +302,11 @@ pub struct CommandPalette { pub keystroke_spacing: f32, } +#[derive(Deserialize, Default)] +pub struct ContactsPopover { + pub background: Color, +} + #[derive(Deserialize, Default)] pub struct ContactsPanel { #[serde(flatten)] diff --git a/styles/src/styleTree/app.ts b/styles/src/styleTree/app.ts index 1345d4e8554658ea9fe1686884f7b9e98b271db1..a3ab4b654c6f6d2835804c834dab6a7435cd75f3 100644 --- a/styles/src/styleTree/app.ts +++ b/styles/src/styleTree/app.ts @@ -3,6 +3,7 @@ import chatPanel from "./chatPanel"; import { text } from "./components"; import contactFinder from "./contactFinder"; import contactsPanel from "./contactsPanel"; +import contactsPopover from "./contactsPopover"; import commandPalette from "./commandPalette"; import editor from "./editor"; import projectPanel from "./projectPanel"; @@ -34,6 +35,7 @@ export default function app(theme: Theme): Object { commandPalette: commandPalette(theme), projectPanel: projectPanel(theme), chatPanel: chatPanel(theme), + contactsPopover: contactsPopover(theme), contactsPanel: contactsPanel(theme), contactFinder: contactFinder(theme), search: search(theme), diff --git a/styles/src/styleTree/contactsPopover.ts b/styles/src/styleTree/contactsPopover.ts new file mode 100644 index 0000000000000000000000000000000000000000..d9f7900cce5bb77e7e5e7aea5dc573aab560c5b6 --- /dev/null +++ b/styles/src/styleTree/contactsPopover.ts @@ -0,0 +1,8 @@ +import Theme from "../themes/common/theme"; +import { backgroundColor } from "./components"; + +export default function workspace(theme: Theme) { + return { + background: backgroundColor(theme, 300), + } +} \ No newline at end of file From 147268157cf703f91eb703e96c3d2c6887293714 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 12:15:40 +0200 Subject: [PATCH 71/89] Animate popup window as if it were an `NSMenu` --- crates/gpui/src/platform/mac/window.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index f21428a1a83ffd78b0cc9833d0d6c3bdcfb046c0..107870d9e60172ac1c78b3383de458edc345570c 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -475,11 +475,20 @@ impl Window { native_window.center(); } - native_window.makeKeyAndOrderFront_(nil); match options.kind { WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), - WindowKind::PopUp => native_window.setLevel_(NSPopUpWindowLevel), + WindowKind::PopUp => { + #[allow(non_upper_case_globals)] + const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; + + native_window.setLevel_(NSPopUpWindowLevel); + let _: () = msg_send![ + native_window, + setAnimationBehavior: NSWindowAnimationBehaviorUtilityWindow + ]; + } } + native_window.makeKeyAndOrderFront_(nil); window.0.borrow().move_traffic_light(); pool.drain(); From 462e5852c2d581651264356de75235832c48426f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:44:54 +0200 Subject: [PATCH 72/89] Use an "always active" tracking area to detect `mouseMoved` events This ensures that we can still receive mouse moved events (e.g. for setting the cursor style) for panels that float above other windows even if the application isn't active. --- crates/gpui/src/platform/mac/window.rs | 30 ++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 107870d9e60172ac1c78b3383de458edc345570c..a8e9853be67ba400cfd8aed99785b660f938c1f6 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -57,11 +57,21 @@ static mut WINDOW_CLASS: *const Class = ptr::null(); static mut PANEL_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); +#[allow(non_upper_case_globals)] +const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = + unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; #[allow(non_upper_case_globals)] const NSNormalWindowLevel: NSInteger = 0; - #[allow(non_upper_case_globals)] const NSPopUpWindowLevel: NSInteger = 101; +#[allow(non_upper_case_globals)] +const NSTrackingMouseMoved: NSUInteger = 0x02; +#[allow(non_upper_case_globals)] +const NSTrackingActiveAlways: NSUInteger = 0x80; +#[allow(non_upper_case_globals)] +const NSTrackingInVisibleRect: NSUInteger = 0x200; +#[allow(non_upper_case_globals)] +const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -362,10 +372,6 @@ impl Window { let native_window: id = match options.kind { WindowKind::Normal => msg_send![WINDOW_CLASS, alloc], WindowKind::PopUp => { - #[allow(non_upper_case_globals)] - const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = - unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; - style_mask |= NSWindowStyleMaskNonactivatingPanel; msg_send![PANEL_CLASS, alloc] } @@ -452,7 +458,16 @@ impl Window { native_window.setTitlebarAppearsTransparent_(YES); } - native_window.setAcceptsMouseMovedEvents_(YES); + let tracking_area: id = msg_send![class!(NSTrackingArea), alloc]; + let rect = NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)); + let _: () = msg_send![ + tracking_area, + initWithRect: rect + options: NSTrackingMouseMoved | NSTrackingActiveAlways | NSTrackingInVisibleRect + owner: native_view + userInfo: nil + ]; + let _: () = msg_send![native_view, addTrackingArea: tracking_area]; native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); native_view.setWantsBestResolutionOpenGLSurface_(YES); @@ -478,9 +493,6 @@ impl Window { match options.kind { WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), WindowKind::PopUp => { - #[allow(non_upper_case_globals)] - const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; - native_window.setLevel_(NSPopUpWindowLevel); let _: () = msg_send![ native_window, From ea00a000281387df8962ab45114a0fb01dac707b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:56:51 +0200 Subject: [PATCH 73/89] Start showing a filter query in contacts popover --- .../src/contacts_popover.rs | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index cd0e361f5be3c5ec6460e51f60e76e83cc934dd5..2998d74ed8c3b608210c9d482831c3407a973c7f 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,11 +1,14 @@ -use gpui::{elements::*, Entity, RenderContext, View, ViewContext}; +use editor::Editor; +use gpui::{elements::*, Entity, RenderContext, View, ViewContext, ViewHandle}; use settings::Settings; pub enum Event { Deactivated, } -pub struct ContactsPopover; +pub struct ContactsPopover { + filter_editor: ViewHandle, +} impl Entity for ContactsPopover { type Event = Event; @@ -18,9 +21,50 @@ impl View for ContactsPopover { fn render(&mut self, cx: &mut RenderContext) -> ElementBox { let theme = &cx.global::().theme.contacts_popover; - Empty::new() + + Flex::row() + .with_child( + ChildView::new(self.filter_editor.clone()) + .contained() + .with_style( + cx.global::() + .theme + .contacts_panel + .user_query_editor + .container, + ) + .flex(1., true) + .boxed(), + ) + // .with_child( + // MouseEventHandler::::new(0, cx, |_, _| { + // Svg::new("icons/user_plus_16.svg") + // .with_color(theme.add_contact_button.color) + // .constrained() + // .with_height(16.) + // .contained() + // .with_style(theme.add_contact_button.container) + // .aligned() + // .boxed() + // }) + // .with_cursor_style(CursorStyle::PointingHand) + // .on_click(MouseButton::Left, |_, cx| { + // cx.dispatch_action(contact_finder::Toggle) + // }) + // .boxed(), + // ) + .constrained() + .with_height( + cx.global::() + .theme + .contacts_panel + .user_query_editor_height, + ) + .aligned() + .top() .contained() .with_background_color(theme.background) + .with_uniform_padding(4.) .boxed() } } @@ -29,7 +73,17 @@ impl ContactsPopover { pub fn new(cx: &mut ViewContext) -> Self { cx.observe_window_activation(Self::window_activation_changed) .detach(); - Self + + let filter_editor = cx.add_view(|cx| { + let mut editor = Editor::single_line( + Some(|theme| theme.contacts_panel.user_query_editor.clone()), + cx, + ); + editor.set_placeholder_text("Filter contacts", cx); + editor + }); + + Self { filter_editor } } fn window_activation_changed(&mut self, is_active: bool, cx: &mut ViewContext) { From 66d13cf42c0178b30e2d0d28de4cf19b6e079c84 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:57:02 +0200 Subject: [PATCH 74/89] Query `isKeyWindow` on `windowDidBecomeKey` or `windowDidResignKey` Before we were assuming that receiving a callback meant that the window was in that "key" state accordingly, but with popups that's not always the case. In particular, there was a bug that caused an unrelated window to receive `windowDidBecomeKey` when making an `NSPanel` the key window. --- crates/gpui/src/platform/mac/window.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index a8e9853be67ba400cfd8aed99785b660f938c1f6..b0002709e0480f803a29e326b66db03bc70d599e 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1070,17 +1070,12 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { } } -extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { - let is_active = if selector == sel!(windowDidBecomeKey:) { - true - } else if selector == sel!(windowDidResignKey:) { - false - } else { - unreachable!(); - }; - +extern "C" fn window_did_change_key_status(this: &Object, _: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; - let executor = window_state.as_ref().borrow().executor.clone(); + let window_state_borrow = window_state.borrow(); + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + let executor = window_state_borrow.executor.clone(); + drop(window_state_borrow); executor .spawn(async move { let mut window_state_borrow = window_state.as_ref().borrow_mut(); From 5898fa61fb711571bb626798afce1923a937702b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:03:38 +0200 Subject: [PATCH 75/89] Temporarily disable status item so that we can merge to `main` --- crates/zed/src/main.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index f9e04782ab27d1320f7217c3bc6f633c679545c7..3bfd5e6e1a08791e262eab09ad6ede4771382278 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -14,7 +14,6 @@ use client::{ http::{self, HttpClient}, UserStore, ZED_SECRET_CLIENT_TOKEN, }; -use contacts_status_item::ContactsStatusItem; use fs::OpenOptions; use futures::{ channel::{mpsc, oneshot}, @@ -89,8 +88,6 @@ fn main() { }); app.run(move |cx| { - cx.add_status_bar_item(|_| ContactsStatusItem::new()); - let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); @@ -106,7 +103,6 @@ fn main() { watch_settings_file(default_settings, settings_file, themes.clone(), cx); watch_keymap_file(keymap_file, cx); - contacts_status_item::init(cx); context_menu::init(cx); project::Project::init(&client); client::Channel::init(&client); From cc316423ca3385aaba98578a88af67598076dccb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:07:13 +0200 Subject: [PATCH 76/89] :lipstick: --- styles/src/styleTree/contactsPopover.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/styles/src/styleTree/contactsPopover.ts b/styles/src/styleTree/contactsPopover.ts index d9f7900cce5bb77e7e5e7aea5dc573aab560c5b6..e9de5dddaf627120d12ab452137fbd47508771a5 100644 --- a/styles/src/styleTree/contactsPopover.ts +++ b/styles/src/styleTree/contactsPopover.ts @@ -5,4 +5,4 @@ export default function workspace(theme: Theme) { return { background: backgroundColor(theme, 300), } -} \ No newline at end of file +} From 3ec3f838dbf02e9d99dbda06160bc8c99239c147 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:24:02 +0200 Subject: [PATCH 77/89] Autorelease `NSTrackingArea` to avoid leaking it --- crates/gpui/src/platform/mac/window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index b0002709e0480f803a29e326b66db03bc70d599e..8a140a236130e1ff734b05b09f65b18cd8eaea21 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -467,7 +467,7 @@ impl Window { owner: native_view userInfo: nil ]; - let _: () = msg_send![native_view, addTrackingArea: tracking_area]; + let _: () = msg_send![native_view, addTrackingArea: tracking_area.autorelease()]; native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); native_view.setWantsBestResolutionOpenGLSurface_(YES); From 3163366a106120337455dc1ee17466403757dc20 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 09:54:48 +0200 Subject: [PATCH 78/89] Inline empty `NSTrackingArea` rect --- crates/gpui/src/platform/mac/window.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 8a140a236130e1ff734b05b09f65b18cd8eaea21..defaa612f43048c71d8eccbc06c3b1a573f32ab8 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -459,10 +459,9 @@ impl Window { } let tracking_area: id = msg_send![class!(NSTrackingArea), alloc]; - let rect = NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)); let _: () = msg_send![ tracking_area, - initWithRect: rect + initWithRect: NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)) options: NSTrackingMouseMoved | NSTrackingActiveAlways | NSTrackingInVisibleRect owner: native_view userInfo: nil From 7a16e9c04878672bd16151d3c724d075403415d7 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 10:48:20 +0200 Subject: [PATCH 79/89] Allow panels to appear on top of full-screen apps --- crates/gpui/src/platform/mac/window.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index defaa612f43048c71d8eccbc06c3b1a573f32ab8..5ecebc6f3907110b433665981dbd9a5fd184a991 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -18,7 +18,8 @@ use block::ConcreteBlock; use cocoa::{ appkit::{ CGPoint, NSApplication, NSBackingStoreBuffered, NSScreen, NSView, NSViewHeightSizable, - NSViewWidthSizable, NSWindow, NSWindowButton, NSWindowStyleMask, + NSViewWidthSizable, NSWindow, NSWindowButton, NSWindowCollectionBehavior, + NSWindowStyleMask, }, base::{id, nil}, foundation::{ @@ -497,6 +498,10 @@ impl Window { native_window, setAnimationBehavior: NSWindowAnimationBehaviorUtilityWindow ]; + native_window.setCollectionBehavior_( + NSWindowCollectionBehavior::NSWindowCollectionBehaviorCanJoinAllSpaces | + NSWindowCollectionBehavior::NSWindowCollectionBehaviorFullScreenAuxiliary + ); } } native_window.makeKeyAndOrderFront_(nil); From b6ff8e77499edf36f7ef378fa6ebeb2b9b47ad8b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 10:55:15 +0200 Subject: [PATCH 80/89] Introduce workaround for spurious `windowDidBecomeKey` event --- crates/gpui/src/platform/mac/window.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5ecebc6f3907110b433665981dbd9a5fd184a991..375a271eacdfd710f4daeb450a94a09178133cf7 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1074,10 +1074,29 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { } } -extern "C" fn window_did_change_key_status(this: &Object, _: Sel, _: id) { +extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + + // When opening a pop-up while the application isn't active, Cocoa sends a spurious + // `windowDidBecomeKey` message to the previous key window even though that window + // isn't actually key. This causes a bug if the application is later activated while + // the pop-up is still open, making it impossible to activate the previous key window + // even if the pop-up gets closed. The only way to activate it again is to de-activate + // the app and re-activate it, which is a pretty bad UX. + // The following code detects the spurious event and invokes `resignKeyWindow`: + // in theory, we're not supposed to invoke this method manually but it balances out + // the spurious `becomeKeyWindow` event and helps us work around that bug. + if selector == sel!(windowDidBecomeKey:) { + if !is_active { + unsafe { + let _: () = msg_send![window_state_borrow.native_window, resignKeyWindow]; + return; + } + } + } + let executor = window_state_borrow.executor.clone(); drop(window_state_borrow); executor From 80d7df7664fbafb0220cd0897480477073170c37 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:01:36 +0200 Subject: [PATCH 81/89] Use an older version of the swift toolchain --- crates/live_kit/LiveKitBridge/Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index fe715588045aa8c5f7ec23034128ec1177ae93a2..76e528bda98a2c9495256d2a2f57e612d4480f21 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.6 +// swift-tools-version: 5.5 import PackageDescription From 582ca666d013b5bf80f8dc12cf30c1f926c44a9e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:02:55 +0200 Subject: [PATCH 82/89] Don't assume `BOOL` is a boolean --- crates/gpui/src/platform/mac/status_item.rs | 2 +- crates/gpui/src/platform/mac/window.rs | 4 ++-- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 404b5e2ef4649926b40a080c28aa498352920895..33feb4808f17c15136df7daeee287a90af921f3a 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -144,7 +144,7 @@ impl StatusItem { Weak::into_raw(Rc::downgrade(&state)) as *const c_void, ); native_view.setWantsBestResolutionOpenGLSurface_(YES); - native_view.setWantsLayer(true); + native_view.setWantsLayer(YES); let _: () = msg_send![ native_view, setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 375a271eacdfd710f4daeb450a94a09178133cf7..be51b67560a31e99e7518f9cb0ec163a4833637d 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -450,7 +450,7 @@ impl Window { native_window.setTitle_(NSString::alloc(nil).init_str(title)); } - native_window.setMovable_(options.is_movable); + native_window.setMovable_(options.is_movable as BOOL); if options .titlebar @@ -1077,7 +1077,7 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); - let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() as bool }; // When opening a pop-up while the application isn't active, Cocoa sends a spurious // `windowDidBecomeKey` message to the previous key window even though that window diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index fb1eb9a79e1e349a56a69d5f7b076a297cab6512..f59b82920376e3ab9ffb11aa11ed50f151305079 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -10,7 +10,7 @@ class LKRoomDelegate: RoomDelegate { self.data = data self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack } - + func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) From a0e2b7a6e9ebab3f89769c801e6f64412702eb74 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:08:19 +0200 Subject: [PATCH 83/89] Update `Package.resolved` --- .../live_kit/LiveKitBridge/Package.resolved | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index e95cffe979d210cf039f213f5689fe03c9cadd71..b19e2980a493bd24b9f94f4811bdda08fa4f4a40 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -1,49 +1,52 @@ { - "pins" : [ - { - "identity" : "client-sdk-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/livekit/client-sdk-swift.git", - "state" : { - "revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4" + "object": { + "pins": [ + { + "package": "LiveKit", + "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", + "state": { + "branch": null, + "revision": "5cc3c001779ab147199ce3ea0dce465b846368b4", + "version": null + } + }, + { + "package": "Promises", + "repositoryURL": "https://github.com/google/promises.git", + "state": { + "branch": null, + "revision": "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version": "2.1.1" + } + }, + { + "package": "WebRTC", + "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", + "state": { + "branch": null, + "revision": "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version": "104.5112.2" + } + }, + { + "package": "swift-log", + "repositoryURL": "https://github.com/apple/swift-log.git", + "state": { + "branch": null, + "revision": "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version": "1.4.4" + } + }, + { + "package": "SwiftProtobuf", + "repositoryURL": "https://github.com/apple/swift-protobuf.git", + "state": { + "branch": null, + "revision": "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version": "1.20.1" + } } - }, - { - "identity" : "promises", - "kind" : "remoteSourceControl", - "location" : "https://github.com/google/promises.git", - "state" : { - "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", - "version" : "2.1.1" - } - }, - { - "identity" : "specs", - "kind" : "remoteSourceControl", - "location" : "https://github.com/webrtc-sdk/Specs.git", - "state" : { - "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", - "version" : "104.5112.2" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", - "version" : "1.4.4" - } - }, - { - "identity" : "swift-protobuf", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-protobuf.git", - "state" : { - "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", - "version" : "1.20.1" - } - } - ], - "version" : 2 + ] + }, + "version": 1 } From 458a6a73101d06798567c90e0c41cea38c81051e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:09:30 +0200 Subject: [PATCH 84/89] :lipstick: --- crates/gpui/src/platform/mac/window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index be51b67560a31e99e7518f9cb0ec163a4833637d..1ce42db9ed68d3332d3bb931e0f49c58748cc09e 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1077,7 +1077,7 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); - let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() as bool }; + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() == YES }; // When opening a pop-up while the application isn't active, Cocoa sends a spurious // `windowDidBecomeKey` message to the previous key window even though that window From b76f3372a34c3ffd777065b8c3dab777662e4d5e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:22:17 +0200 Subject: [PATCH 85/89] Target a triple explicitly when building Swift bridge --- crates/live_kit/build.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index 4f23cc4e926ea12d8ae12c76a1e08ab6d344af75..bd72460017512efd3ef3e497aca13cd18524266f 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -1,6 +1,8 @@ use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; +const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SwiftTargetInfo { @@ -42,9 +44,15 @@ fn build_bridge(swift_target: &SwiftTarget) { "cargo:rerun-if-changed={}/Package.swift", SWIFT_PACKAGE_NAME ); + println!( + "cargo:rerun-if-changed={}/Package.resolved", + SWIFT_PACKAGE_NAME + ); let swift_package_root = swift_package_root(); if !Command::new("swift") - .args(&["build", "-c", &env::var("PROFILE").unwrap()]) + .arg("build") + .args(&["--configuration", &env::var("PROFILE").unwrap()]) + .args(&["--triple", &swift_target.target.triple]) .current_dir(&swift_package_root) .status() .unwrap() @@ -116,8 +124,6 @@ fn get_swift_target() -> SwiftTarget { serde_json::from_slice(&swift_target_info_str).unwrap() } -const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; - fn swift_package_root() -> PathBuf { env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) } From 8bd059a293d111a755c563f0475e3cc818fa6d85 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:33:49 +0200 Subject: [PATCH 86/89] Try using the unversioned triple when compiling live_kit --- crates/live_kit/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index bd72460017512efd3ef3e497aca13cd18524266f..f11e4a9bac322563c3c4dc588f08364e3e852a33 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -52,7 +52,7 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&["--triple", &swift_target.target.triple]) + .args(&["--triple", &swift_target.target.unversioned_triple]) .current_dir(&swift_package_root) .status() .unwrap() From 32c65cfb2ade5343613a86fb7c69faedce076fef Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:35:33 +0200 Subject: [PATCH 87/89] Specify macOS target version manually --- crates/live_kit/build.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index f11e4a9bac322563c3c4dc588f08364e3e852a33..df2127cc3fdcbeece8991404c2ff3d42edb78d14 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -52,7 +52,13 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&["--triple", &swift_target.target.unversioned_triple]) + .args(&[ + "--triple", + &format!( + "{}{}", + swift_target.target.unversioned_triple, MACOS_TARGET_VERSION + ), + ]) .current_dir(&swift_package_root) .status() .unwrap() From 20778a0694e786a1e7b0852916fc229264327dbf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:47:17 +0200 Subject: [PATCH 88/89] Allow using `live_kit` with macOS >= 10.15 --- crates/live_kit/build.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index df2127cc3fdcbeece8991404c2ff3d42edb78d14..e34b4e2a3e1fee095ca5b08e3a4e33ef303ff244 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -28,7 +28,7 @@ pub struct SwiftTarget { pub paths: SwiftPaths, } -const MACOS_TARGET_VERSION: &str = "12"; +const MACOS_TARGET_VERSION: &str = "10.15"; fn main() { let swift_target = get_swift_target(); @@ -52,13 +52,7 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&[ - "--triple", - &format!( - "{}{}", - swift_target.target.unversioned_triple, MACOS_TARGET_VERSION - ), - ]) + .args(&["--triple", &swift_target.target.triple]) .current_dir(&swift_package_root) .status() .unwrap() @@ -78,10 +72,6 @@ fn build_bridge(swift_target: &SwiftTarget) { } fn link_swift_stdlib(swift_target: &SwiftTarget) { - if swift_target.target.libraries_require_rpath { - panic!("Libraries require RPath! Change minimum MacOS value to fix.") - } - swift_target .paths .runtime_library_paths From e07f4f3f53320a31a53813cb8f487e71cfb227d8 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 18:09:11 +0200 Subject: [PATCH 89/89] Copy WebRTC.framework in the `deps` directory Also, define the `rpath` on `live_kit` to avoid errors when running tests. --- crates/live_kit/build.rs | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index e34b4e2a3e1fee095ca5b08e3a4e33ef303ff244..79d7d84cdd96c63381e4ec9b29fb30e28b788e05 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -1,5 +1,9 @@ use serde::Deserialize; -use std::{env, path::PathBuf, process::Command}; +use std::{ + env, + path::{Path, PathBuf}, + process::Command, +}; const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; @@ -88,20 +92,16 @@ fn link_webrtc_framework(swift_target: &SwiftTarget) { "cargo:rustc-link-search=framework={}", swift_out_dir_path.display() ); + // Find WebRTC.framework as a sibling of the executable when running tests. + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); let source_path = swift_out_dir_path.join("WebRTC.framework"); - let target_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); - assert!( - Command::new("cp") - .arg("-r") - .args(&[&source_path, &target_path]) - .status() - .unwrap() - .success(), - "could not copy WebRTC.framework from {:?} to {:?}", - source_path, - target_path - ); + let deps_dir_path = + PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework"); + let target_dir_path = + PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); + copy_dir(&source_path, &deps_dir_path); + copy_dir(&source_path, &target_dir_path); } fn get_swift_target() -> SwiftTarget { @@ -124,6 +124,20 @@ fn swift_package_root() -> PathBuf { env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) } +fn copy_dir(source: &Path, destination: &Path) { + assert!( + Command::new("cp") + .arg("-r") + .args(&[source, destination]) + .status() + .unwrap() + .success(), + "could not copy {:?} to {:?}", + source, + destination + ); +} + impl SwiftTarget { fn out_dir_path(&self) -> PathBuf { swift_package_root()