David Kleingeld created
Reverts zed-industries/zed#38493
Release Notes:
- N/ACargo.lock | 2
assets/settings/default.json | 32 -
crates/audio/Cargo.toml | 1
crates/audio/src/audio.rs | 65 --
crates/audio/src/audio_settings.rs | 95 +--
crates/audio/src/rodio_ext.rs | 167 ------
crates/denoise/src/engine.rs | 6
crates/denoise/src/lib.rs | 2
crates/livekit_client/src/livekit_client.rs | 20
crates/livekit_client/src/livekit_client/playback.rs | 89 +--
crates/livekit_client/src/livekit_client/playback/source.rs | 40 -
crates/settings/src/settings_content.rs | 44 -
tooling/workspace-hack/Cargo.toml | 14
13 files changed, 123 insertions(+), 454 deletions(-)
@@ -1405,7 +1405,6 @@ dependencies = [
"async-tar",
"collections",
"crossbeam",
- "denoise",
"gpui",
"libwebrtc",
"log",
@@ -20743,6 +20742,7 @@ dependencies = [
"nix 0.29.0",
"nix 0.30.1",
"nom 7.1.3",
+ "num",
"num-bigint",
"num-bigint-dig",
"num-complex",
@@ -413,33 +413,15 @@
"experimental.rodio_audio": false,
// Requires 'rodio_audio: true'
//
- // Automatically increase or decrease you microphone's volume. This affects how
- // loud you sound to others.
- //
- // Recommended: off (default)
- // Microphones are too quite in zed, until everyone is on experimental
- // audio and has auto speaker volume on this will make you very loud
- // compared to other speakers.
- "experimental.auto_microphone_volume": false,
- // Requires 'rodio_audio: true'
- //
- // Automatically increate or decrease the volume of other call members.
- // This only affects how things sound for you.
- "experimental.auto_speaker_volume": true,
+ // Use the new audio systems automatic gain control for your microphone.
+ // This affects how loud you sound to others.
+ "experimental.control_input_volume": false,
// Requires 'rodio_audio: true'
//
- // Remove background noises. Works great for typing, cars, dogs, AC. Does
- // not work well on music.
- "experimental.denoise": true,
- // Requires 'rodio_audio: true'
- //
- // Use audio parameters compatible with the previous versions of
- // experimental audio and non-experimental audio. When this is false you
- // will sound strange to anyone not on the latest experimental audio. In
- // the future we will migrate by setting this to false
- //
- // You need to rejoin a call for this setting to apply
- "experimental.legacy_audio_compatible": true
+ // Use the new audio systems automatic gain control on everyone in the
+ // call. This makes call members who are too quite louder and those who are
+ // too loud quieter. This only affects how things sound for you.
+ "experimental.control_output_volume": false
},
// Scrollbar related settings
"scrollbar": {
@@ -18,7 +18,6 @@ async-tar.workspace = true
collections.workspace = true
crossbeam.workspace = true
gpui.workspace = true
-denoise = { path = "../denoise" }
log.workspace = true
parking_lot.workspace = true
rodio = { workspace = true, features = [ "wav", "playback", "wav_output" ] }
@@ -9,7 +9,7 @@ mod non_windows_and_freebsd_deps {
pub(super) use log::info;
pub(super) use parking_lot::Mutex;
pub(super) use rodio::cpal::Sample;
- pub(super) use rodio::source::LimitSettings;
+ pub(super) use rodio::source::{LimitSettings, UniformSourceIterator};
pub(super) use std::sync::Arc;
}
@@ -31,20 +31,18 @@ pub use rodio_ext::RodioExt;
use crate::audio_settings::LIVE_SETTINGS;
-// We are migrating to 16kHz sample rate from 48kHz. In the future
-// once we are reasonably sure most users have upgraded we will
-// remove the LEGACY parameters.
+// NOTE: We used to use WebRTC's mixer which only supported
+// 16kHz, 32kHz and 48kHz. As 48 is the most common "next step up"
+// for audio output devices like speakers/bluetooth, we just hard-code
+// this; and downsample when we need to.
//
-// We migrate to 16kHz because it is sufficient for speech and required
-// by the denoiser and future Speech to Text layers.
-pub const SAMPLE_RATE: NonZero<u32> = nz!(16000);
-pub const CHANNEL_COUNT: NonZero<u16> = nz!(1);
+// Since most noise cancelling requires 16kHz we will move to
+// that in the future.
+pub const SAMPLE_RATE: NonZero<u32> = nz!(48000);
+pub const CHANNEL_COUNT: NonZero<u16> = nz!(2);
pub const BUFFER_SIZE: usize = // echo canceller and livekit want 10ms of audio
(SAMPLE_RATE.get() as usize / 100) * CHANNEL_COUNT.get() as usize;
-pub const LEGACY_SAMPLE_RATE: NonZero<u32> = nz!(48000);
-pub const LEGACY_CHANNEL_COUNT: NonZero<u16> = nz!(2);
-
pub const REPLAY_DURATION: Duration = Duration::from_secs(30);
pub fn init(cx: &mut App) {
@@ -108,11 +106,6 @@ impl Global for Audio {}
impl Audio {
fn ensure_output_exists(&mut self) -> Result<&Mixer> {
- #[cfg(debug_assertions)]
- log::warn!(
- "Audio does not sound correct without optimizations. Use a release build to debug audio issues"
- );
-
if self.output_handle.is_none() {
self.output_handle = Some(
OutputStreamBuilder::open_default_stream()
@@ -167,20 +160,13 @@ impl Audio {
let stream = rodio::microphone::MicrophoneBuilder::new()
.default_device()?
.default_config()?
- .prefer_sample_rates([
- SAMPLE_RATE, // sample rates trivially resamplable to `SAMPLE_RATE`
- SAMPLE_RATE.saturating_mul(nz!(2)),
- SAMPLE_RATE.saturating_mul(nz!(3)),
- SAMPLE_RATE.saturating_mul(nz!(4)),
- ])
- .prefer_channel_counts([nz!(1), nz!(2), nz!(3), nz!(4)])
+ .prefer_sample_rates([SAMPLE_RATE, SAMPLE_RATE.saturating_mul(nz!(2))])
+ // .prefer_channel_counts([nz!(1), nz!(2)])
.prefer_buffer_sizes(512..)
.open_stream()?;
info!("Opened microphone: {:?}", stream.config());
- let (replay, stream) = stream
- .possibly_disconnected_channels_to_mono()
- .constant_samplerate(SAMPLE_RATE)
+ let (replay, stream) = UniformSourceIterator::new(stream, CHANNEL_COUNT, SAMPLE_RATE)
.limit(LimitSettings::live_performance())
.process_buffer::<BUFFER_SIZE, _>(move |buffer| {
let mut int_buffer: [i16; _] = buffer.map(|s| s.to_sample());
@@ -201,28 +187,15 @@ impl Audio {
}
}
})
- .denoise()
- .context("Could not set up denoiser")?
- .periodic_access(Duration::from_millis(100), move |denoise| {
- denoise.set_enabled(LIVE_SETTINGS.denoise.load(Ordering::Relaxed));
- })
- .automatic_gain_control(1.0, 2.0, 0.0, 5.0)
+ .automatic_gain_control(1.0, 4.0, 0.0, 5.0)
.periodic_access(Duration::from_millis(100), move |agc_source| {
- agc_source
- .set_enabled(LIVE_SETTINGS.auto_microphone_volume.load(Ordering::Relaxed));
+ agc_source.set_enabled(LIVE_SETTINGS.control_input_volume.load(Ordering::Relaxed));
})
.replayable(REPLAY_DURATION)?;
voip_parts
.replays
.add_voip_stream("local microphone".to_string(), replay);
-
- let stream = if voip_parts.legacy_audio_compatible {
- stream.constant_params(LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE)
- } else {
- stream.constant_params(CHANNEL_COUNT, SAMPLE_RATE)
- };
-
Ok(stream)
}
@@ -233,10 +206,9 @@ impl Audio {
cx: &mut App,
) -> anyhow::Result<()> {
let (replay_source, source) = source
- .constant_params(CHANNEL_COUNT, SAMPLE_RATE)
- .automatic_gain_control(1.0, 2.0, 0.0, 5.0)
+ .automatic_gain_control(1.0, 4.0, 0.0, 5.0)
.periodic_access(Duration::from_millis(100), move |agc_source| {
- agc_source.set_enabled(LIVE_SETTINGS.auto_speaker_volume.load(Ordering::Relaxed));
+ agc_source.set_enabled(LIVE_SETTINGS.control_input_volume.load(Ordering::Relaxed));
})
.replayable(REPLAY_DURATION)
.expect("REPLAY_DURATION is longer than 100ms");
@@ -297,7 +269,6 @@ impl Audio {
pub struct VoipParts {
echo_canceller: Arc<Mutex<apm::AudioProcessingModule>>,
replays: replays::Replays,
- legacy_audio_compatible: bool,
}
#[cfg(not(any(all(target_os = "windows", target_env = "gnu"), target_os = "freebsd")))]
@@ -306,12 +277,8 @@ impl VoipParts {
let (apm, replays) = cx.try_read_default_global::<Audio, _>(|audio, _| {
(Arc::clone(&audio.echo_canceller), audio.replays.clone())
})?;
- let legacy_audio_compatible =
- AudioSettings::try_read_global(cx, |settings| settings.legacy_audio_compatible)
- .unwrap_or_default();
Ok(Self {
- legacy_audio_compatible,
echo_canceller: apm,
replays,
})
@@ -6,38 +6,18 @@ use settings::{Settings, SettingsStore};
#[derive(Clone, Debug)]
pub struct AudioSettings {
/// Opt into the new audio system.
- ///
- /// You need to rejoin a call for this setting to apply
pub rodio_audio: bool, // default is false
/// Requires 'rodio_audio: true'
///
- /// Automatically increase or decrease you microphone's volume. This affects how
- /// loud you sound to others.
- ///
- /// Recommended: off (default)
- /// Microphones are too quite in zed, until everyone is on experimental
- /// audio and has auto speaker volume on this will make you very loud
- /// compared to other speakers.
- pub auto_microphone_volume: bool,
- /// Requires 'rodio_audio: true'
- ///
- /// Automatically increate or decrease the volume of other call members.
- /// This only affects how things sound for you.
- pub auto_speaker_volume: bool,
- /// Requires 'rodio_audio: true'
- ///
- /// Remove background noises. Works great for typing, cars, dogs, AC. Does
- /// not work well on music.
- pub denoise: bool,
+ /// Use the new audio systems automatic gain control for your microphone.
+ /// This affects how loud you sound to others.
+ pub control_input_volume: bool,
/// Requires 'rodio_audio: true'
///
- /// Use audio parameters compatible with the previous versions of
- /// experimental audio and non-experimental audio. When this is false you
- /// will sound strange to anyone not on the latest experimental audio. In
- /// the future we will migrate by setting this to false
- ///
- /// You need to rejoin a call for this setting to apply
- pub legacy_audio_compatible: bool,
+ /// Use the new audio systems automatic gain control on everyone in the
+ /// call. This makes call members who are too quite louder and those who are
+ /// too loud quieter. This only affects how things sound for you.
+ pub control_output_volume: bool,
}
/// Configuration of audio in Zed
@@ -45,66 +25,46 @@ impl Settings for AudioSettings {
fn from_settings(content: &settings::SettingsContent, _cx: &mut App) -> Self {
let audio = &content.audio.as_ref().unwrap();
AudioSettings {
+ control_input_volume: audio.control_input_volume.unwrap(),
+ control_output_volume: audio.control_output_volume.unwrap(),
rodio_audio: audio.rodio_audio.unwrap(),
- auto_microphone_volume: audio.auto_microphone_volume.unwrap(),
- auto_speaker_volume: audio.auto_speaker_volume.unwrap(),
- denoise: audio.denoise.unwrap(),
- legacy_audio_compatible: audio.legacy_audio_compatible.unwrap(),
}
}
+
+ fn import_from_vscode(
+ _vscode: &settings::VsCodeSettings,
+ _current: &mut settings::SettingsContent,
+ ) {
+ }
}
/// See docs on [LIVE_SETTINGS]
pub(crate) struct LiveSettings {
- pub(crate) auto_microphone_volume: AtomicBool,
- pub(crate) auto_speaker_volume: AtomicBool,
- pub(crate) denoise: AtomicBool,
+ pub(crate) control_input_volume: AtomicBool,
+ pub(crate) control_output_volume: AtomicBool,
}
impl LiveSettings {
pub(crate) fn initialize(&self, cx: &mut App) {
cx.observe_global::<SettingsStore>(move |cx| {
- LIVE_SETTINGS.auto_microphone_volume.store(
- AudioSettings::get_global(cx).auto_microphone_volume,
+ LIVE_SETTINGS.control_input_volume.store(
+ AudioSettings::get_global(cx).control_input_volume,
Ordering::Relaxed,
);
- LIVE_SETTINGS.auto_speaker_volume.store(
- AudioSettings::get_global(cx).auto_speaker_volume,
+ LIVE_SETTINGS.control_output_volume.store(
+ AudioSettings::get_global(cx).control_output_volume,
Ordering::Relaxed,
);
-
- let denoise_enabled = AudioSettings::get_global(cx).denoise;
- #[cfg(debug_assertions)]
- {
- static DENOISE_WARNING_SEND: AtomicBool = AtomicBool::new(false);
- if denoise_enabled && !DENOISE_WARNING_SEND.load(Ordering::Relaxed) {
- DENOISE_WARNING_SEND.store(true, Ordering::Relaxed);
- log::warn!("Denoise does not work on debug builds, not enabling")
- }
- }
- #[cfg(not(debug_assertions))]
- LIVE_SETTINGS
- .denoise
- .store(denoise_enabled, Ordering::Relaxed);
})
.detach();
let init_settings = AudioSettings::get_global(cx);
LIVE_SETTINGS
- .auto_microphone_volume
- .store(init_settings.auto_microphone_volume, Ordering::Relaxed);
- LIVE_SETTINGS
- .auto_speaker_volume
- .store(init_settings.auto_speaker_volume, Ordering::Relaxed);
- let denoise_enabled = AudioSettings::get_global(cx).denoise;
- #[cfg(debug_assertions)]
- if denoise_enabled {
- log::warn!("Denoise does not work on debug builds, not enabling")
- }
- #[cfg(not(debug_assertions))]
+ .control_input_volume
+ .store(init_settings.control_input_volume, Ordering::Relaxed);
LIVE_SETTINGS
- .denoise
- .store(denoise_enabled, Ordering::Relaxed);
+ .control_output_volume
+ .store(init_settings.control_output_volume, Ordering::Relaxed);
}
}
@@ -113,7 +73,6 @@ impl LiveSettings {
/// real time and must each run in a dedicated OS thread, therefore we can not
/// use the background executor.
pub(crate) static LIVE_SETTINGS: LiveSettings = LiveSettings {
- auto_microphone_volume: AtomicBool::new(true),
- auto_speaker_volume: AtomicBool::new(true),
- denoise: AtomicBool::new(true),
+ control_input_volume: AtomicBool::new(true),
+ control_output_volume: AtomicBool::new(true),
};
@@ -1,5 +1,4 @@
use std::{
- num::NonZero,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -8,22 +7,12 @@ use std::{
};
use crossbeam::queue::ArrayQueue;
-use denoise::{Denoiser, DenoiserError};
-use log::warn;
-use rodio::{
- ChannelCount, Sample, SampleRate, Source, conversions::SampleRateConverter, nz,
- source::UniformSourceIterator,
-};
-
-const MAX_CHANNELS: usize = 8;
+use rodio::{ChannelCount, Sample, SampleRate, Source};
#[derive(Debug, thiserror::Error)]
#[error("Replay duration is too short must be >= 100ms")]
pub struct ReplayDurationTooShort;
-// These all require constant sources (so the span is infinitely long)
-// this is not guaranteed by rodio however we know it to be true in all our
-// applications. Rodio desperately needs a constant source concept.
pub trait RodioExt: Source + Sized {
fn process_buffer<const N: usize, F>(self, callback: F) -> ProcessBuffer<N, Self, F>
where
@@ -36,14 +25,6 @@ pub trait RodioExt: Source + Sized {
duration: Duration,
) -> Result<(Replay, Replayable<Self>), ReplayDurationTooShort>;
fn take_samples(self, n: usize) -> TakeSamples<Self>;
- fn denoise(self) -> Result<Denoiser<Self>, DenoiserError>;
- fn constant_params(
- self,
- channel_count: ChannelCount,
- sample_rate: SampleRate,
- ) -> UniformSourceIterator<Self>;
- fn constant_samplerate(self, sample_rate: SampleRate) -> ConstantSampleRate<Self>;
- fn possibly_disconnected_channels_to_mono(self) -> ToMono<Self>;
}
impl<S: Source> RodioExt for S {
@@ -120,149 +101,8 @@ impl<S: Source> RodioExt for S {
left_to_take: n,
}
}
- fn denoise(self) -> Result<Denoiser<Self>, DenoiserError> {
- let res = Denoiser::try_new(self);
- res
- }
- fn constant_params(
- self,
- channel_count: ChannelCount,
- sample_rate: SampleRate,
- ) -> UniformSourceIterator<Self> {
- UniformSourceIterator::new(self, channel_count, sample_rate)
- }
- fn constant_samplerate(self, sample_rate: SampleRate) -> ConstantSampleRate<Self> {
- ConstantSampleRate::new(self, sample_rate)
- }
- fn possibly_disconnected_channels_to_mono(self) -> ToMono<Self> {
- ToMono::new(self)
- }
-}
-
-pub struct ConstantSampleRate<S: Source> {
- inner: SampleRateConverter<S>,
- channels: ChannelCount,
- sample_rate: SampleRate,
-}
-
-impl<S: Source> ConstantSampleRate<S> {
- fn new(source: S, target_rate: SampleRate) -> Self {
- let input_sample_rate = source.sample_rate();
- let channels = source.channels();
- let inner = SampleRateConverter::new(source, input_sample_rate, target_rate, channels);
- Self {
- inner,
- channels,
- sample_rate: target_rate,
- }
- }
-}
-
-impl<S: Source> Iterator for ConstantSampleRate<S> {
- type Item = rodio::Sample;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.inner.next()
- }
-
- fn size_hint(&self) -> (usize, Option<usize>) {
- self.inner.size_hint()
- }
-}
-
-impl<S: Source> Source for ConstantSampleRate<S> {
- fn current_span_len(&self) -> Option<usize> {
- None
- }
-
- fn channels(&self) -> ChannelCount {
- self.channels
- }
-
- fn sample_rate(&self) -> SampleRate {
- self.sample_rate
- }
-
- fn total_duration(&self) -> Option<Duration> {
- None // not supported (not used by us)
- }
-}
-
-const TYPICAL_NOISE_FLOOR: Sample = 1e-3;
-
-/// constant source, only works on a single span
-pub struct ToMono<S> {
- inner: S,
- input_channel_count: ChannelCount,
- connected_channels: ChannelCount,
- /// running mean of second channel 'volume'
- means: [f32; MAX_CHANNELS],
-}
-impl<S: Source> ToMono<S> {
- fn new(input: S) -> Self {
- let channels = input
- .channels()
- .min(const { NonZero::<u16>::new(MAX_CHANNELS as u16).unwrap() });
- if channels < input.channels() {
- warn!("Ignoring input channels {}..", channels.get());
- }
-
- Self {
- connected_channels: channels,
- input_channel_count: channels,
- inner: input,
- means: [TYPICAL_NOISE_FLOOR; MAX_CHANNELS],
- }
- }
-}
-
-impl<S: Source> Source for ToMono<S> {
- fn current_span_len(&self) -> Option<usize> {
- None
- }
-
- fn channels(&self) -> ChannelCount {
- rodio::nz!(1)
- }
-
- fn sample_rate(&self) -> SampleRate {
- self.inner.sample_rate()
- }
-
- fn total_duration(&self) -> Option<Duration> {
- self.inner.total_duration()
- }
-}
-
-fn update_mean(mean: &mut f32, sample: Sample) {
- const HISTORY: f32 = 500.0;
- *mean *= (HISTORY - 1.0) / HISTORY;
- *mean += sample.abs() / HISTORY;
-}
-
-impl<S: Source> Iterator for ToMono<S> {
- type Item = Sample;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut mono_sample = 0f32;
- let mut active_channels = 0;
- for channel in 0..self.input_channel_count.get() as usize {
- let sample = self.inner.next()?;
- mono_sample += sample;
-
- update_mean(&mut self.means[channel], sample);
- if self.means[channel] > TYPICAL_NOISE_FLOOR / 10.0 {
- active_channels += 1;
- }
- }
- mono_sample /= self.connected_channels.get() as f32;
- self.connected_channels = NonZero::new(active_channels).unwrap_or(nz!(1));
-
- Some(mono_sample)
- }
}
-/// constant source, only works on a single span
pub struct TakeSamples<S> {
inner: S,
left_to_take: usize,
@@ -307,7 +147,6 @@ impl<S: Source> Source for TakeSamples<S> {
}
}
-/// constant source, only works on a single span
#[derive(Debug)]
struct ReplayQueue {
inner: ArrayQueue<Vec<Sample>>,
@@ -354,7 +193,6 @@ impl ReplayQueue {
}
}
-/// constant source, only works on a single span
pub struct ProcessBuffer<const N: usize, S, F>
where
S: Source + Sized,
@@ -422,7 +260,6 @@ where
}
}
-/// constant source, only works on a single span
pub struct InspectBuffer<const N: usize, S, F>
where
S: Source + Sized,
@@ -487,7 +324,6 @@ where
}
}
-/// constant source, only works on a single span
#[derive(Debug)]
pub struct Replayable<S: Source> {
inner: S,
@@ -539,7 +375,6 @@ impl<S: Source> Source for Replayable<S> {
}
}
-/// constant source, only works on a single span
#[derive(Debug)]
pub struct Replay {
rx: Arc<ReplayQueue>,
@@ -138,13 +138,13 @@ impl Engine {
const SPECTRUM_INPUT: &str = "input_2";
const MEMORY_INPUT: &str = "input_3";
- let spectrum =
+ let memory_input =
Tensor::from_slice::<_, f32>(&self.in_magnitude, (1, 1, FFT_OUT_SIZE), &Device::Cpu)
.expect("the in magnitude has enough elements to fill the Tensor");
let inputs = HashMap::from([
- (SPECTRUM_INPUT.to_string(), spectrum),
- (MEMORY_INPUT.to_string(), self.spectral_memory.clone()),
+ (MEMORY_INPUT.to_string(), memory_input),
+ (SPECTRUM_INPUT.to_string(), self.spectral_memory.clone()),
]);
inputs
}
@@ -84,7 +84,7 @@ impl<S: Source> Denoiser<S> {
.spawn(move || {
run_neural_denoiser(denoised_tx, input_rx);
})
- .expect("Should be ablet to spawn threads");
+ .unwrap();
Ok(Self {
inner: source,
@@ -1,6 +1,6 @@
use std::sync::Arc;
-use anyhow::{Context as _, Result, anyhow};
+use anyhow::{Context as _, Result};
use audio::AudioSettings;
use collections::HashMap;
use futures::{SinkExt, channel::mpsc};
@@ -12,10 +12,7 @@ use settings::Settings;
mod playback;
-use crate::{
- LocalTrack, Participant, RemoteTrack, RoomEvent, TrackPublication,
- livekit_client::playback::Speaker,
-};
+use crate::{LocalTrack, Participant, RemoteTrack, RoomEvent, TrackPublication};
pub use playback::AudioStream;
pub(crate) use playback::{RemoteVideoFrame, play_remote_video_track};
@@ -135,20 +132,11 @@ impl Room {
track: &RemoteAudioTrack,
cx: &mut App,
) -> Result<playback::AudioStream> {
- let speaker: Speaker =
- serde_urlencoded::from_str(&track.0.name()).unwrap_or_else(|_| Speaker {
- name: track.0.name(),
- is_staff: false,
- legacy_audio_compatible: true,
- });
-
if AudioSettings::get_global(cx).rodio_audio {
info!("Using experimental.rodio_audio audio pipeline for output");
- playback::play_remote_audio_track(&track.0, speaker, cx)
- } else if speaker.legacy_audio_compatible {
- Ok(self.playback.play_remote_audio_track(&track.0))
+ playback::play_remote_audio_track(&track.0, cx)
} else {
- Err(anyhow!("Client version too old to play audio in call"))
+ Ok(self.playback.play_remote_audio_track(&track.0))
}
}
}
@@ -1,6 +1,6 @@
use anyhow::{Context as _, Result};
-use audio::{AudioSettings, CHANNEL_COUNT, LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE, SAMPLE_RATE};
+use audio::{AudioSettings, CHANNEL_COUNT, SAMPLE_RATE};
use cpal::traits::{DeviceTrait, StreamTrait as _};
use futures::channel::mpsc::UnboundedSender;
use futures::{Stream, StreamExt as _};
@@ -43,17 +43,12 @@ pub(crate) struct AudioStack {
pub(crate) fn play_remote_audio_track(
track: &livekit::track::RemoteAudioTrack,
- speaker: Speaker,
cx: &mut gpui::App,
) -> Result<AudioStream> {
- let stream = source::LiveKitStream::new(
- cx.background_executor(),
- track,
- speaker.legacy_audio_compatible,
- );
-
let stop_handle = Arc::new(AtomicBool::new(false));
let stop_handle_clone = stop_handle.clone();
+ let stream = source::LiveKitStream::new(cx.background_executor(), track);
+
let stream = stream
.stoppable()
.periodic_access(Duration::from_millis(50), move |s| {
@@ -62,6 +57,10 @@ pub(crate) fn play_remote_audio_track(
}
});
+ let speaker: Speaker = serde_urlencoded::from_str(&track.name()).unwrap_or_else(|_| Speaker {
+ name: track.name(),
+ is_staff: false,
+ });
audio::Audio::play_voip_stream(stream, speaker.name, speaker.is_staff, cx)
.context("Could not play audio")?;
@@ -153,32 +152,17 @@ impl AudioStack {
is_staff: bool,
cx: &AsyncApp,
) -> Result<(crate::LocalAudioTrack, AudioStream)> {
- let legacy_audio_compatible =
- AudioSettings::try_read_global(cx, |setting| setting.legacy_audio_compatible)
- .unwrap_or_default();
-
- let source = if legacy_audio_compatible {
- NativeAudioSource::new(
- // n.b. this struct's options are always ignored, noise cancellation is provided by apm.
- AudioSourceOptions::default(),
- LEGACY_SAMPLE_RATE.get(),
- LEGACY_CHANNEL_COUNT.get().into(),
- 10,
- )
- } else {
- NativeAudioSource::new(
- // n.b. this struct's options are always ignored, noise cancellation is provided by apm.
- AudioSourceOptions::default(),
- SAMPLE_RATE.get(),
- CHANNEL_COUNT.get().into(),
- 10,
- )
- };
+ let source = NativeAudioSource::new(
+ // n.b. this struct's options are always ignored, noise cancellation is provided by apm.
+ AudioSourceOptions::default(),
+ SAMPLE_RATE.get(),
+ CHANNEL_COUNT.get().into(),
+ 10,
+ );
let track_name = serde_urlencoded::to_string(Speaker {
name: user_name,
is_staff,
- legacy_audio_compatible,
})
.context("Could not encode user information in track name")?;
@@ -202,32 +186,22 @@ impl AudioStack {
let capture_task = if rodio_pipeline {
info!("Using experimental.rodio_audio audio pipeline");
let voip_parts = audio::VoipParts::new(cx)?;
- // Audio needs to run real-time and should never be paused. That is
- // why we are using a normal std::thread and not a background task
+ // Audio needs to run real-time and should never be paused. That is why we are using a
+ // normal std::thread and not a background task
thread::Builder::new()
- .name("MicrophoneToLivekit".to_string())
+ .name("AudioCapture".to_string())
.spawn(move || {
// microphone is non send on mac
- let microphone = match audio::Audio::open_microphone(voip_parts) {
- Ok(m) => m,
- Err(e) => {
- log::error!("Could not open microphone: {e}");
- return;
- }
- };
+ let microphone = audio::Audio::open_microphone(voip_parts)?;
send_to_livekit(frame_tx, microphone);
+ Ok::<(), anyhow::Error>(())
})
- .expect("should be able to spawn threads");
+ .unwrap();
Task::ready(Ok(()))
} else {
self.executor.spawn(async move {
- Self::capture_input(
- apm,
- frame_tx,
- LEGACY_SAMPLE_RATE.get(),
- LEGACY_CHANNEL_COUNT.get().into(),
- )
- .await
+ Self::capture_input(apm, frame_tx, SAMPLE_RATE.get(), CHANNEL_COUNT.get().into())
+ .await
})
};
@@ -415,30 +389,25 @@ impl AudioStack {
}
#[derive(Serialize, Deserialize)]
-pub struct Speaker {
- pub name: String,
- pub is_staff: bool,
- pub legacy_audio_compatible: bool,
+struct Speaker {
+ name: String,
+ is_staff: bool,
}
fn send_to_livekit(frame_tx: UnboundedSender<AudioFrame<'static>>, mut microphone: impl Source) {
use cpal::Sample;
- let sample_rate = microphone.sample_rate().get();
- let num_channels = microphone.channels().get() as u32;
- let buffer_size = sample_rate / 100 * num_channels;
-
loop {
let sampled: Vec<_> = microphone
.by_ref()
- .take(buffer_size as usize)
+ .take(audio::BUFFER_SIZE)
.map(|s| s.to_sample())
.collect();
if frame_tx
.unbounded_send(AudioFrame {
- sample_rate,
- num_channels,
- samples_per_channel: sampled.len() as u32 / num_channels,
+ sample_rate: SAMPLE_RATE.get(),
+ num_channels: CHANNEL_COUNT.get() as u32,
+ samples_per_channel: sampled.len() as u32 / CHANNEL_COUNT.get() as u32,
data: Cow::Owned(sampled),
})
.is_err()
@@ -3,19 +3,17 @@ use std::num::NonZero;
use futures::StreamExt;
use libwebrtc::{audio_stream::native::NativeAudioStream, prelude::AudioFrame};
use livekit::track::RemoteAudioTrack;
-use rodio::{
- ChannelCount, SampleRate, Source, buffer::SamplesBuffer, conversions::SampleTypeConverter,
-};
+use rodio::{Source, buffer::SamplesBuffer, conversions::SampleTypeConverter, nz};
-use audio::{CHANNEL_COUNT, LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE, SAMPLE_RATE};
+use audio::{CHANNEL_COUNT, SAMPLE_RATE};
fn frame_to_samplesbuffer(frame: AudioFrame) -> SamplesBuffer {
let samples = frame.data.iter().copied();
let samples = SampleTypeConverter::<_, _>::new(samples);
let samples: Vec<f32> = samples.collect();
SamplesBuffer::new(
- NonZero::new(frame.num_channels as u16).expect("zero channels is nonsense"),
- NonZero::new(frame.sample_rate).expect("samplerate zero is nonsense"),
+ nz!(2), // frame always has two channels
+ NonZero::new(frame.sample_rate).expect("audio frame sample rate is nonzero"),
samples,
)
}
@@ -24,26 +22,14 @@ pub struct LiveKitStream {
// shared_buffer: SharedBuffer,
inner: rodio::queue::SourcesQueueOutput,
_receiver_task: gpui::Task<()>,
- channel_count: ChannelCount,
- sample_rate: SampleRate,
}
impl LiveKitStream {
- pub fn new(
- executor: &gpui::BackgroundExecutor,
- track: &RemoteAudioTrack,
- legacy: bool,
- ) -> Self {
- let (channel_count, sample_rate) = if legacy {
- (LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE)
- } else {
- (CHANNEL_COUNT, SAMPLE_RATE)
- };
-
+ pub fn new(executor: &gpui::BackgroundExecutor, track: &RemoteAudioTrack) -> Self {
let mut stream = NativeAudioStream::new(
track.rtc_track(),
- sample_rate.get() as i32,
- channel_count.get().into(),
+ SAMPLE_RATE.get() as i32,
+ CHANNEL_COUNT.get().into(),
);
let (queue_input, queue_output) = rodio::queue::queue(true);
// spawn rtc stream
@@ -59,8 +45,6 @@ impl LiveKitStream {
LiveKitStream {
_receiver_task: receiver_task,
inner: queue_output,
- sample_rate,
- channel_count,
}
}
}
@@ -79,11 +63,17 @@ impl Source for LiveKitStream {
}
fn channels(&self) -> rodio::ChannelCount {
- self.channel_count
+ // This must be hardcoded because the playback source assumes constant
+ // sample rate and channel count. The queue upon which this is build
+ // will however report different counts and rates. Even though we put in
+ // only items with our (constant) CHANNEL_COUNT & SAMPLE_RATE this will
+ // play silence on one channel and at 44100 which is not what our
+ // constants are.
+ CHANNEL_COUNT
}
fn sample_rate(&self) -> rodio::SampleRate {
- self.sample_rate
+ SAMPLE_RATE // see comment on channels
}
fn total_duration(&self) -> Option<std::time::Duration> {
@@ -291,43 +291,21 @@ pub enum TitleBarVisibility {
#[derive(Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema, MergeFrom, Debug)]
pub struct AudioSettingsContent {
/// Opt into the new audio system.
- ///
- /// You need to rejoin a call for this setting to apply
- #[serde(rename = "experimental.rodio_audio")]
- pub rodio_audio: Option<bool>, // default is false
- /// Requires 'rodio_audio: true'
- ///
- /// Automatically increase or decrease you microphone's volume. This affects how
- /// loud you sound to others.
- ///
- /// Recommended: off (default)
- /// Microphones are too quite in zed, until everyone is on experimental
- /// audio and has auto speaker volume on this will make you very loud
- /// compared to other speakers.
- #[serde(rename = "experimental.auto_microphone_volume")]
- pub auto_microphone_volume: Option<bool>,
+ #[serde(rename = "experimental.rodio_audio", default)]
+ pub rodio_audio: Option<bool>,
/// Requires 'rodio_audio: true'
///
- /// Automatically increate or decrease the volume of other call members.
- /// This only affects how things sound for you.
- #[serde(rename = "experimental.auto_speaker_volume")]
- pub auto_speaker_volume: Option<bool>,
+ /// Use the new audio systems automatic gain control for your microphone.
+ /// This affects how loud you sound to others.
+ #[serde(rename = "experimental.control_input_volume", default)]
+ pub control_input_volume: Option<bool>,
/// Requires 'rodio_audio: true'
///
- /// Remove background noises. Works great for typing, cars, dogs, AC. Does
- /// not work well on music.
- #[serde(rename = "experimental.denoise")]
- pub denoise: Option<bool>,
- /// Requires 'rodio_audio: true'
- ///
- /// Use audio parameters compatible with the previous versions of
- /// experimental audio and non-experimental audio. When this is false you
- /// will sound strange to anyone not on the latest experimental audio. In
- /// the future we will migrate by setting this to false
- ///
- /// You need to rejoin a call for this setting to apply
- #[serde(rename = "experimental.legacy_audio_compatible")]
- pub legacy_audio_compatible: Option<bool>,
+ /// Use the new audio systems automatic gain control on everyone in the
+ /// call. This makes call members who are too quite louder and those who are
+ /// too loud quieter. This only affects how things sound for you.
+ #[serde(rename = "experimental.control_output_volume", default)]
+ pub control_output_volume: Option<bool>,
}
/// Control what info is collected by Zed.
@@ -90,6 +90,7 @@ mime_guess = { version = "2" }
miniz_oxide = { version = "0.8", features = ["simd"] }
nom = { version = "7" }
num-bigint = { version = "0.4" }
+num-complex = { version = "0.4", features = ["bytemuck"] }
num-integer = { version = "0.1", features = ["i128"] }
num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
num-rational = { version = "0.4", features = ["num-bigint-std"] }
@@ -228,6 +229,7 @@ mime_guess = { version = "2" }
miniz_oxide = { version = "0.8", features = ["simd"] }
nom = { version = "7" }
num-bigint = { version = "0.4" }
+num-complex = { version = "0.4", features = ["bytemuck"] }
num-integer = { version = "0.1", features = ["i128"] }
num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
num-rational = { version = "0.4", features = ["num-bigint-std"] }
@@ -306,6 +308,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http1"
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+num = { version = "0.4" }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -335,6 +338,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http1"
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+num = { version = "0.4" }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -365,6 +369,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http1"
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+num = { version = "0.4" }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -394,6 +399,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http1"
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+num = { version = "0.4" }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -436,7 +442,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
@@ -478,7 +483,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
@@ -518,7 +522,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
@@ -560,7 +563,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
@@ -585,6 +587,7 @@ getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-f
getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2", default-features = false, features = ["js", "rdrand"] }
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
+num = { version = "0.4" }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
ring = { version = "0.17", features = ["std"] }
rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38", features = ["event", "fs", "net"] }
@@ -610,6 +613,7 @@ getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-f
getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2", default-features = false, features = ["js", "rdrand"] }
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
livekit-runtime = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d" }
+num = { version = "0.4" }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
ring = { version = "0.17", features = ["std"] }
@@ -651,7 +655,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }
@@ -693,7 +696,6 @@ nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "m
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["fs", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
-num-complex = { version = "0.4", features = ["bytemuck"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
prost-5ef9efb8ec2df382 = { package = "prost", version = "0.12", features = ["prost-derive"] }