From 5d02b490582f47ebc8641825755ac250d7aad992 Mon Sep 17 00:00:00 2001 From: Mikayla Maki Date: Tue, 27 Jun 2023 19:19:08 -0700 Subject: [PATCH] Added muted and currently speaking tracking --- crates/call/src/participant.rs | 1 + crates/call/src/room.rs | 39 ++++++++++- crates/collab_ui/src/collab_titlebar_item.rs | 31 +++++++-- .../Sources/LiveKitBridge/LiveKitBridge.swift | 21 ++++++ crates/live_kit_client/examples/test_app.rs | 39 ++++++++++- crates/live_kit_client/src/prod.rs | 64 ++++++++++++++++++- crates/live_kit_client/src/test.rs | 1 + script/start-local-collaboration | 2 +- 8 files changed, 185 insertions(+), 13 deletions(-) diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index 9773e837c3269aa42583fe374c8ab5cff5a689b6..e7858869ce63906b75f9cd0cb117cf7b54283efd 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -44,6 +44,7 @@ pub struct RemoteParticipant { pub projects: Vec, pub location: ParticipantLocation, pub muted: bool, + pub speaking: bool, pub video_tracks: HashMap>, pub audio_tracks: HashMap>, } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 7b7ec89a5ed176564ec4e656bee6c323fd26d3a5..e5f000c34b348844684a525f80dadb541aec959f 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -164,6 +164,7 @@ impl Room { microphone_track: LocalTrack::None, next_publish_id: 0, deafened: false, + speaking: false, _maintain_room, _maintain_tracks: [_maintain_video_tracks, _maintain_audio_tracks], }) @@ -648,6 +649,7 @@ impl Room { projects: participant.projects, location, muted: false, + speaking: false, video_tracks: Default::default(), audio_tracks: Default::default(), }, @@ -782,6 +784,30 @@ impl Room { cx: &mut ModelContext, ) -> Result<()> { match change { + RemoteAudioTrackUpdate::ActiveSpeakersChanged { speakers } => { + let mut speaker_ids = speakers + .into_iter() + .filter_map(|speaker_sid| speaker_sid.parse().ok()) + .collect::>(); + speaker_ids.sort_unstable(); + for (sid, participant) in &mut self.remote_participants { + if let Ok(_) = speaker_ids.binary_search(sid) { + participant.speaking = true; + } else { + participant.speaking = false; + } + } + if let Some(id) = self.client.user_id() { + if let Some(room) = &mut self.live_kit { + if let Ok(_) = speaker_ids.binary_search(&id) { + room.speaking = true; + } else { + room.speaking = false; + } + } + } + cx.notify(); + } RemoteAudioTrackUpdate::MuteChanged { track_id, muted } => { for participant in &mut self.remote_participants.values_mut() { let mut found = false; @@ -796,6 +822,7 @@ impl Room { break; } } + cx.notify(); } RemoteAudioTrackUpdate::Subscribed(track) => { let user_id = track.publisher_id().parse()?; @@ -1011,7 +1038,7 @@ impl Room { }) } - pub fn is_muted(&self) -> Option { + pub fn is_muted(&self) -> bool { self.live_kit .as_ref() .and_then(|live_kit| match &live_kit.microphone_track { @@ -1019,6 +1046,13 @@ impl Room { LocalTrack::Pending { muted, .. } => Some(*muted), LocalTrack::Published { muted, .. } => Some(*muted), }) + .unwrap_or(false) + } + + pub fn is_speaking(&self) -> bool { + self.live_kit + .as_ref() + .map_or(false, |live_kit| live_kit.speaking) } pub fn is_deafened(&self) -> Option { @@ -1215,7 +1249,7 @@ impl Room { } } pub fn toggle_mute(&mut self, cx: &mut ModelContext) -> Result>> { - let should_mute = self.is_muted().unwrap_or(false); + let should_mute = self.is_muted(); if let Some(live_kit) = self.live_kit.as_mut() { Self::set_mute(live_kit, !should_mute, cx) } else { @@ -1298,6 +1332,7 @@ struct LiveKitRoom { screen_track: LocalTrack, microphone_track: LocalTrack, deafened: bool, + speaking: bool, next_publish_id: usize, _maintain_room: Task<()>, _maintain_tracks: [Task<()>; 2], diff --git a/crates/collab_ui/src/collab_titlebar_item.rs b/crates/collab_ui/src/collab_titlebar_item.rs index 708fb89e86bded98d00c60c19a4691137d71ff0a..e23320f827722b8d95a3a6ee942ae2ea3141ed42 100644 --- a/crates/collab_ui/src/collab_titlebar_item.rs +++ b/crates/collab_ui/src/collab_titlebar_item.rs @@ -86,8 +86,10 @@ impl View for CollabTitlebarItem { right_container .add_children(self.render_in_call_share_unshare_button(&workspace, &theme, cx)); right_container.add_child(self.render_leave_call(&theme, cx)); + let muted = room.read(cx).is_muted(); + let speaking = room.read(cx).is_speaking(); left_container - .add_child(self.render_current_user(&workspace, &theme, &user, peer_id, cx)); + .add_child(self.render_current_user(&workspace, &theme, &user, peer_id, muted, speaking, cx)); left_container.add_children(self.render_collaborators(&workspace, &theme, &room, cx)); right_container.add_child(self.render_toggle_mute(&theme, &room, cx)); right_container.add_child(self.render_toggle_deafen(&theme, &room, cx)); @@ -449,7 +451,7 @@ impl CollabTitlebarItem { ) -> AnyElement { let icon; let tooltip; - let is_muted = room.read(cx).is_muted().unwrap_or(false); + let is_muted = room.read(cx).is_muted(); if is_muted { icon = "icons/radix/mic-mute.svg"; tooltip = "Unmute microphone\nRight click for options"; @@ -766,6 +768,8 @@ impl CollabTitlebarItem { replica_id, participant.peer_id, Some(participant.location), + participant.muted, + participant.speaking, workspace, theme, cx, @@ -782,14 +786,19 @@ impl CollabTitlebarItem { theme: &Theme, user: &Arc, peer_id: PeerId, + muted: bool, + speaking: bool, cx: &mut ViewContext, ) -> AnyElement { let replica_id = workspace.read(cx).project().read(cx).replica_id(); + Container::new(self.render_face_pile( user, Some(replica_id), peer_id, None, + muted, + speaking, workspace, theme, cx, @@ -804,6 +813,8 @@ impl CollabTitlebarItem { replica_id: Option, peer_id: PeerId, location: Option, + muted: bool, + speaking: bool, workspace: &ViewHandle, theme: &Theme, cx: &mut ViewContext, @@ -829,11 +840,17 @@ impl CollabTitlebarItem { let leader_style = theme.titlebar.leader_avatar; let follower_style = theme.titlebar.follower_avatar; - let mut background_color = theme - .titlebar - .container - .background_color - .unwrap_or_default(); + let mut background_color = if muted { + gpui::color::Color::red() + } else if speaking { + gpui::color::Color::green() + } else { + theme + .titlebar + .container + .background_color + .unwrap_or_default() + }; if let Some(replica_id) = replica_id { if followed_by_self { let selection = theme.editor.replica_selection_style(replica_id).selection; diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 74d43d78659d12cd1587914f2814185a82e01f68..40d3641db23afcbac801b7f5f2daa15411c15f72 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -8,6 +8,8 @@ class LKRoomDelegate: RoomDelegate { var onDidDisconnect: @convention(c) (UnsafeRawPointer) -> Void var onDidSubscribeToRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void var onDidUnsubscribeFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void + var onMuteChangedFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void + var onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void @@ -16,6 +18,8 @@ class LKRoomDelegate: RoomDelegate { onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, + onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, + onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) { @@ -25,6 +29,8 @@ class LKRoomDelegate: RoomDelegate { self.onDidUnsubscribeFromRemoteAudioTrack = onDidUnsubscribeFromRemoteAudioTrack self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack + self.onMuteChangedFromRemoteAudioTrack = onMuteChangedFromRemoteAudioTrack + self.onActiveSpeakersChanged = onActiveSpeakersChanged } func room(_ room: Room, didUpdate connectionState: ConnectionState, oldValue: ConnectionState) { @@ -41,6 +47,17 @@ class LKRoomDelegate: RoomDelegate { } } + func room(_ room: Room, participant: Participant, didUpdate publication: TrackPublication, muted: Bool) { + if publication.kind == .audio { + self.onMuteChangedFromRemoteAudioTrack(self.data, publication.sid as CFString, muted) + } + } + + func room(_ room: Room, didUpdate speakers: [Participant]) { + guard let speaker_ids = speakers.compactMap({ $0.identity as CFString }) as CFArray? else { return } + self.onActiveSpeakersChanged(self.data, speaker_ids) + } + func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString) @@ -89,6 +106,8 @@ public func LKRoomDelegateCreate( onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, + onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, + onActiveSpeakerChanged: @escaping @convention(c) (UnsafeRawPointer, CFArray) -> Void, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void ) -> UnsafeMutableRawPointer { @@ -97,6 +116,8 @@ public func LKRoomDelegateCreate( onDidDisconnect: onDidDisconnect, onDidSubscribeToRemoteAudioTrack: onDidSubscribeToRemoteAudioTrack, onDidUnsubscribeFromRemoteAudioTrack: onDidUnsubscribeFromRemoteAudioTrack, + onMuteChangedFromRemoteAudioTrack: onMuteChangedFromRemoteAudioTrack, + onActiveSpeakersChanged: onActiveSpeakerChanged, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack ) diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index faf1b54798f3722fe335fdf0e27748b94e9aba9e..4fc02d9a9da91c74c2b0101bc926aee20ddf1b63 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -74,19 +74,54 @@ fn main() { panic!("unexpected message"); } + audio_track_publication.set_mute(true).await.unwrap(); + + println!("waiting for mute changed!"); + if let RemoteAudioTrackUpdate::MuteChanged { track_id, muted } = + audio_track_updates.next().await.unwrap() + { + let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); + assert_eq!(remote_tracks[0].sid(), track_id); + assert_eq!(muted, true); + } else { + panic!("unexpected message"); + } + + audio_track_publication.set_mute(false).await.unwrap(); + + if let RemoteAudioTrackUpdate::MuteChanged { track_id, muted } = + audio_track_updates.next().await.unwrap() + { + let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); + assert_eq!(remote_tracks[0].sid(), track_id); + assert_eq!(muted, false); + } else { + panic!("unexpected message"); + } + println!("Pausing for 5 seconds to test audio, make some noise!"); let timer = cx.background().timer(Duration::from_secs(5)); timer.await; - let remote_audio_track = room_b .remote_audio_tracks("test-participant-1") .pop() .unwrap(); room_a.unpublish_track(audio_track_publication); + + // Clear out any active speakers changed messages + let mut next = audio_track_updates.next().await.unwrap(); + while let RemoteAudioTrackUpdate::ActiveSpeakersChanged { + speakers + } = next + { + println!("Speakers changed: {:?}", speakers); + next = audio_track_updates.next().await.unwrap(); + } + if let RemoteAudioTrackUpdate::Unsubscribed { publisher_id, track_id, - } = audio_track_updates.next().await.unwrap() + } = next { assert_eq!(publisher_id, "test-participant-1"); assert_eq!(remote_audio_track.sid(), track_id); diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index 96bf40ca4ede083a5ce9a94b12edca4321cddabb..bdbba87b51e9e1984fdd0c86276405c33ad3f38e 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -32,6 +32,15 @@ extern "C" { publisher_id: CFStringRef, track_id: CFStringRef, ), + on_mute_changed_from_remote_audio_track: extern "C" fn( + callback_data: *mut c_void, + track_id: CFStringRef, + muted: bool, + ), + on_active_speakers_changed: extern "C" fn( + callback_data: *mut c_void, + participants: CFArrayRef, + ), on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, publisher_id: CFStringRef, @@ -381,6 +390,24 @@ impl Room { }); } + fn mute_changed_from_remote_audio_track(&self, track_id: String, muted: bool) { + self.remote_audio_track_subscribers.lock().retain(|tx| { + tx.unbounded_send(RemoteAudioTrackUpdate::MuteChanged { + track_id: track_id.clone(), + muted, + }) + .is_ok() + }); + } + + // A vec of publisher IDs + fn active_speakers_changed(&self, speakers: Vec) { + self.remote_audio_track_subscribers.lock().retain(move |tx| { + tx.unbounded_send(RemoteAudioTrackUpdate::ActiveSpeakersChanged { speakers: speakers.clone() }) + .is_ok() + }); + } + fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { let track = Arc::new(track); self.remote_video_track_subscribers.lock().retain(|tx| { @@ -445,6 +472,8 @@ impl RoomDelegate { Self::on_did_disconnect, Self::on_did_subscribe_to_remote_audio_track, Self::on_did_unsubscribe_from_remote_audio_track, + Self::on_mute_change_from_remote_audio_track, + Self::on_active_speakers_changed, Self::on_did_subscribe_to_remote_video_track, Self::on_did_unsubscribe_from_remote_video_track, ) @@ -493,6 +522,38 @@ impl RoomDelegate { let _ = Weak::into_raw(room); } + extern "C" fn on_mute_change_from_remote_audio_track( + room: *mut c_void, + track_id: CFStringRef, + muted: bool, + ) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; + if let Some(room) = room.upgrade() { + room.mute_changed_from_remote_audio_track(track_id, muted); + } + let _ = Weak::into_raw(room); + } + + extern "C" fn on_active_speakers_changed(room: *mut c_void, participants: CFArrayRef) { + if participants.is_null() { + return; + } + + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let speakers = unsafe { + CFArray::wrap_under_get_rule(participants) + .into_iter() + .map(|speaker: core_foundation::base::ItemRef<'_, *const c_void>| CFString::wrap_under_get_rule(*speaker as CFStringRef).to_string()) + .collect() + }; + + if let Some(room) = room.upgrade() { + room.active_speakers_changed(speakers); + } + let _ = Weak::into_raw(room); + } + extern "C" fn on_did_subscribe_to_remote_video_track( room: *mut c_void, publisher_id: CFStringRef, @@ -761,7 +822,8 @@ pub enum RemoteVideoTrackUpdate { } pub enum RemoteAudioTrackUpdate { - MuteChanged { track_id: Sid, muted: bool}, + ActiveSpeakersChanged { speakers: Vec }, + MuteChanged { track_id: Sid, muted: bool }, Subscribed(Arc), Unsubscribed { publisher_id: Sid, track_id: Sid }, } diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 286c7215e9b6b722626c52b1923f13506896bf9b..809a1f5d29a7ad9d64fbd1f331fec655d4e566de 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -580,6 +580,7 @@ pub enum RemoteVideoTrackUpdate { #[derive(Clone)] pub enum RemoteAudioTrackUpdate { + ActiveSpeakersChanged { speakers: Vec }, MuteChanged { track_id: Sid, muted: bool}, Subscribed(Arc), Unsubscribed { publisher_id: Sid, track_id: Sid }, diff --git a/script/start-local-collaboration b/script/start-local-collaboration index b8632c4c229d754fec6ce8c19a4c892eea115c8a..b702fb4e02f9d0e3ae2a70ca99054b7bea2a711b 100755 --- a/script/start-local-collaboration +++ b/script/start-local-collaboration @@ -54,5 +54,5 @@ sleep 0.5 # Start the two Zed child processes. Open the given paths with the first instance. trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT ZED_IMPERSONATE=${username_1} ZED_WINDOW_POSITION=${position_1} target/debug/Zed $@ & -ZED_IMPERSONATE=${username_2} ZED_WINDOW_POSITION=${position_2} target/debug/Zed & +SECOND=true ZED_IMPERSONATE=${username_2} ZED_WINDOW_POSITION=${position_2} target/debug/Zed & wait