From 3e43840f88f790e0bfff519ef62d74bb4e7e9e6c Mon Sep 17 00:00:00 2001 From: David Kleingeld Date: Fri, 12 Sep 2025 15:11:31 +0200 Subject: [PATCH] integrates denoising --- Cargo.lock | 699 ++++++++++++++++-- crates/audio/Cargo.toml | 1 + crates/audio/src/audio.rs | 12 +- crates/audio/src/audio_settings.rs | 28 + crates/audio/src/rodio_ext.rs | 21 +- crates/livekit_client/src/lib.rs | 66 +- .../src/livekit_client/playback.rs | 11 +- 7 files changed, 718 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82e1b202cb5ac18870bc0cf825847d711835c9f6..c756d66a6108b57fc3b25e033d213aa7960a0ebd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,7 +184,7 @@ dependencies = [ "telemetry", "text", "theme", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "util", "uuid", @@ -269,7 +269,7 @@ dependencies = [ "terminal", "text", "theme", - "thiserror 2.0.12", + "thiserror 2.0.16", "tree-sitter-rust", "ui", "unindent", @@ -316,7 +316,7 @@ dependencies = [ "smol", "task", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "ui", "util", "watch", @@ -659,7 +659,7 @@ dependencies = [ "serde", "serde_json", "strum 0.27.1", - "thiserror 2.0.12", + "thiserror 2.0.16", "workspace-hack", ] @@ -689,6 +689,9 @@ name = "arbitrary" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arc-swap" @@ -1386,6 +1389,7 @@ dependencies = [ "async-tar", "collections", "crossbeam", + "denoise", "gpui", "libwebrtc", "log", @@ -1395,7 +1399,7 @@ dependencies = [ "serde", "settings", "smol", - "thiserror 2.0.12", + "thiserror 2.0.16", "util", "workspace-hack", ] @@ -2130,7 +2134,7 @@ dependencies = [ "serde", "serde_json", "strum 0.27.1", - "thiserror 2.0.12", + "thiserror 2.0.16", "workspace-hack", ] @@ -2671,6 +2675,52 @@ dependencies = [ "serde", ] +[[package]] +name = "candle-core" +version = "0.9.1" +source = "git+https://github.com/dvdsk/candle?branch=9.1-patched#591d35bb503cb93a63bb49d5fe2d9531ee6ad651" +dependencies = [ + "byteorder", + "gemm 0.17.1", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.9.1", + "rand_distr", + "rayon", + "safetensors", + "thiserror 1.0.69", + "ug", + "yoke", + "zip 1.1.4", +] + +[[package]] +name = "candle-nn" +version = "0.9.1" +source = "git+https://github.com/dvdsk/candle?branch=9.1-patched#591d35bb503cb93a63bb49d5fe2d9531ee6ad651" +dependencies = [ + "candle-core", + "half", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "candle-onnx" +version = "0.9.1" +source = "git+https://github.com/dvdsk/candle?branch=9.1-patched#591d35bb503cb93a63bb49d5fe2d9531ee6ad651" +dependencies = [ + "candle-core", + "candle-nn", + "prost 0.12.6", + "prost-build 0.12.6", +] + [[package]] name = "cap-fs-ext" version = "3.4.4" @@ -2769,7 +2819,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -3092,7 +3142,7 @@ dependencies = [ "telemetry", "telemetry_events", "text", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tiny_http", "tokio", @@ -3451,7 +3501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ec1052629a80c28594777d1252efc8a6b005d13f9edfd8c3fc0f44d5b32489a" dependencies = [ "nix 0.30.1", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -4622,6 +4672,21 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" +[[package]] +name = "denoise" +version = "0.1.0" +dependencies = [ + "candle-core", + "candle-nn", + "candle-onnx", + "prost 0.12.6", + "realfft", + "rodio", + "rtrb", + "rustfft", + "thiserror 2.0.16", +] + [[package]] name = "der" version = "0.6.1" @@ -4653,6 +4718,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "derive_more" version = "0.99.19" @@ -4966,6 +5042,25 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + +[[package]] +name = "dyn-stack" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490bd48eb68fffcfed519b4edbfd82c69cbe741d175b84f0e0cbe8c57cbe0bdd" +dependencies = [ + "bytemuck", +] + [[package]] name = "ec4rs" version = "1.2.0" @@ -5210,6 +5305,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3d8a32ae18130a3c84dd492d4215c3d913c3b07c6b63c2eb3eb7ff1101ab7bf" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "enumflags2" version = "0.7.11" @@ -6295,6 +6402,243 @@ dependencies = [ "thread_local", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-c32 0.17.1", + "gemm-c64 0.17.1", + "gemm-common 0.17.1", + "gemm-f16 0.17.1", + "gemm-f32 0.17.1", + "gemm-f64 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-c32 0.18.2", + "gemm-c64 0.18.2", + "gemm-common 0.18.2", + "gemm-f16 0.18.2", + "gemm-f32 0.18.2", + "gemm-f64 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack 0.10.0", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.18.22", + "raw-cpuid 10.7.0", + "rayon", + "seq-macro", + "sysctl 0.5.5", +] + +[[package]] +name = "gemm-common" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" +dependencies = [ + "bytemuck", + "dyn-stack 0.13.0", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.21.5", + "raw-cpuid 11.6.0", + "rayon", + "seq-macro", + "sysctl 0.6.0", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "gemm-f32 0.17.1", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f16" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "gemm-f32 0.18.2", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + [[package]] name = "generator" version = "0.8.5" @@ -6403,7 +6747,7 @@ dependencies = [ "sum_tree", "tempfile", "text", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "unindent", "url", @@ -6549,7 +6893,7 @@ dependencies = [ "gix-worktree", "once_cell", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6562,7 +6906,7 @@ dependencies = [ "gix-date", "gix-utils 0.2.0", "itoa", - "thiserror 2.0.12", + "thiserror 2.0.16", "winnow", ] @@ -6579,7 +6923,7 @@ dependencies = [ "gix-trace", "kstring", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-bom", ] @@ -6589,7 +6933,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1db9765c69502650da68f0804e3dc2b5f8ccc6a2d104ca6c85bc40700d37540" dependencies = [ - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6598,7 +6942,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b1f1d8764958699dc764e3f727cef280ff4d1bd92c107bbf8acd85b30c1bd6f" dependencies = [ - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6624,7 +6968,7 @@ dependencies = [ "gix-chunk", "gix-hash 0.17.0", "memmap2", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6643,7 +6987,7 @@ dependencies = [ "memchr", "once_cell", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-bom", "winnow", ] @@ -6658,7 +7002,7 @@ dependencies = [ "bstr", "gix-path", "libc", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6670,7 +7014,7 @@ dependencies = [ "bstr", "itoa", "jiff", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6691,7 +7035,7 @@ dependencies = [ "gix-traverse", "gix-worktree", "imara-diff", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6707,7 +7051,7 @@ dependencies = [ "gix-path", "gix-ref", "gix-sec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6726,7 +7070,7 @@ dependencies = [ "once_cell", "parking_lot", "prodash", - "thiserror 2.0.12", + "thiserror 2.0.16", "walkdir", ] @@ -6760,7 +7104,7 @@ dependencies = [ "gix-trace", "gix-utils 0.2.0", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6774,7 +7118,7 @@ dependencies = [ "gix-features 0.41.1", "gix-path", "gix-utils 0.2.0", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6788,7 +7132,7 @@ dependencies = [ "gix-features 0.42.1", "gix-path", "gix-utils 0.3.0", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6812,7 +7156,7 @@ dependencies = [ "faster-hex 0.9.0", "gix-features 0.41.1", "sha1-checked", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6824,7 +7168,7 @@ dependencies = [ "faster-hex 0.10.0", "gix-features 0.42.1", "sha1-checked", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6876,7 +7220,7 @@ dependencies = [ "memmap2", "rustix 0.38.44", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6887,7 +7231,7 @@ checksum = "570f8b034659f256366dc90f1a24924902f20acccd6a15be96d44d1269e7a796" dependencies = [ "gix-tempfile", "gix-utils 0.3.0", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6907,7 +7251,7 @@ dependencies = [ "gix-validate 0.9.4", "itoa", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "winnow", ] @@ -6929,7 +7273,7 @@ dependencies = [ "gix-quote", "parking_lot", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6947,7 +7291,7 @@ dependencies = [ "gix-path", "memmap2", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "uluru", ] @@ -6960,7 +7304,7 @@ dependencies = [ "bstr", "faster-hex 0.9.0", "gix-trace", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6972,7 +7316,7 @@ dependencies = [ "bstr", "faster-hex 0.9.0", "gix-trace", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -6986,7 +7330,7 @@ dependencies = [ "gix-validate 0.10.0", "home", "once_cell", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7001,7 +7345,7 @@ dependencies = [ "gix-config-value", "gix-glob", "gix-path", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7019,7 +7363,7 @@ dependencies = [ "gix-transport", "gix-utils 0.2.0", "maybe-async", - "thiserror 2.0.12", + "thiserror 2.0.16", "winnow", ] @@ -7031,7 +7375,7 @@ checksum = "1b005c550bf84de3b24aa5e540a23e6146a1c01c7d30470e35d75a12f827f969" dependencies = [ "bstr", "gix-utils 0.2.0", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7051,7 +7395,7 @@ dependencies = [ "gix-utils 0.2.0", "gix-validate 0.9.4", "memmap2", - "thiserror 2.0.12", + "thiserror 2.0.16", "winnow", ] @@ -7066,7 +7410,7 @@ dependencies = [ "gix-revision", "gix-validate 0.9.4", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7081,7 +7425,7 @@ dependencies = [ "gix-hash 0.17.0", "gix-object", "gix-revwalk", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7096,7 +7440,7 @@ dependencies = [ "gix-hashtable", "gix-object", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7120,7 +7464,7 @@ dependencies = [ "bstr", "gix-hash 0.17.0", "gix-lock", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7135,7 +7479,7 @@ dependencies = [ "gix-pathspec", "gix-refspec", "gix-url", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7171,7 +7515,7 @@ dependencies = [ "gix-quote", "gix-sec", "gix-url", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7188,7 +7532,7 @@ dependencies = [ "gix-object", "gix-revwalk", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7201,7 +7545,7 @@ dependencies = [ "gix-features 0.41.1", "gix-path", "percent-encoding", - "thiserror 2.0.12", + "thiserror 2.0.16", "url", ] @@ -7232,7 +7576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34b5f1253109da6c79ed7cf6e1e38437080bb6d704c76af14c93e2f255234084" dependencies = [ "bstr", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7242,7 +7586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77b9e00cacde5b51388d28ed746c493b18a6add1f19b5e01d686b3b9ece66d4d" dependencies = [ "bstr", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -7465,7 +7809,7 @@ dependencies = [ "strum 0.27.1", "sum_tree", "taffy", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-segmentation", "usvg", "util", @@ -7571,9 +7915,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ + "bytemuck", "cfg-if", "crunchy", "num-traits", + "rand 0.9.1", + "rand_distr", ] [[package]] @@ -8753,7 +9100,7 @@ dependencies = [ "smallvec", "strsim", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "toml_edit", "tracing", "version_check", @@ -9140,7 +9487,7 @@ dependencies = [ "serde_json", "smol", "telemetry_events", - "thiserror 2.0.12", + "thiserror 2.0.16", "util", "workspace-hack", ] @@ -9190,7 +9537,7 @@ dependencies = [ "smol", "strum 0.27.1", "theme", - "thiserror 2.0.12", + "thiserror 2.0.16", "tiktoken-rs", "tokio", "ui", @@ -10158,6 +10505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", + "stable_deref_trait", ] [[package]] @@ -10451,7 +10799,7 @@ dependencies = [ "rustc-hash 1.1.0", "spirv", "strum 0.26.3", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-ident", ] @@ -10803,6 +11151,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -11239,7 +11588,7 @@ dependencies = [ "serde", "serde_json", "strum 0.27.1", - "thiserror 2.0.12", + "thiserror 2.0.16", "workspace-hack", ] @@ -11695,7 +12044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.16", "ucd-trie", ] @@ -12495,6 +12844,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -12733,7 +13091,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -12995,6 +13353,32 @@ dependencies = [ "wasmtime-math", ] +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + +[[package]] +name = "pulp" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "reborrow", + "version_check", +] + [[package]] name = "qoi" version = "0.4.1" @@ -13051,7 +13435,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.26", "socket2", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", "web-time", @@ -13071,7 +13455,7 @@ dependencies = [ "rustls 0.23.26", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyvec", "tracing", "web-time", @@ -13171,6 +13555,16 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.1", +] + [[package]] name = "range-map" version = "0.2.0" @@ -13236,6 +13630,24 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -13284,6 +13696,21 @@ dependencies = [ "font-types", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "recent_projects" version = "0.1.0" @@ -13359,7 +13786,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -13486,7 +13913,7 @@ dependencies = [ "shlex", "smol", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "urlencoding", "util", "which 6.0.3", @@ -13550,7 +13977,7 @@ dependencies = [ "smol", "sysinfo", "telemetry_events", - "thiserror 2.0.12", + "thiserror 2.0.16", "toml 0.8.20", "unindent", "util", @@ -13893,7 +14320,7 @@ dependencies = [ "num-rational", "rtrb", "symphonia", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -14103,6 +14530,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f140db74548f7c9d7cce60912c9ac414e74df5e718dc947d514b051b42f3f4" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "0.38.44" @@ -14327,6 +14768,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safetensors" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "salsa20" version = "0.10.2" @@ -14546,7 +14997,7 @@ dependencies = [ "serde_json", "sqlx", "strum 0.26.3", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tracing", "url", @@ -14708,6 +15159,12 @@ dependencies = [ "serde", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.219" @@ -15080,7 +15537,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", ] @@ -15382,7 +15839,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tokio", "tokio-stream", @@ -15471,7 +15928,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tracing", "uuid", @@ -15514,7 +15971,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tracing", "uuid", @@ -15541,7 +15998,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tracing", "url", @@ -15652,6 +16109,12 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -16143,6 +16606,34 @@ dependencies = [ "libc", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.9.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.9.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysinfo" version = "0.31.4" @@ -16451,7 +16942,7 @@ dependencies = [ "sysinfo", "task", "theme", - "thiserror 2.0.12", + "thiserror 2.0.16", "url", "urlencoding", "util", @@ -16554,7 +17045,7 @@ dependencies = [ "serde_repr", "settings", "strum 0.27.1", - "thiserror 2.0.12", + "thiserror 2.0.16", "util", "uuid", "workspace-hack", @@ -16623,11 +17114,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.16", ] [[package]] @@ -16643,9 +17134,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -17239,6 +17730,16 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tree-sitter" version = "0.25.6" @@ -17552,7 +18053,7 @@ dependencies = [ "rustls 0.23.26", "rustls-pki-types", "sha1", - "thiserror 2.0.12", + "thiserror 2.0.16", "utf-8", ] @@ -17600,6 +18101,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "ug" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b70b37e9074642bc5f60bb23247fd072a84314ca9e71cdf8527593406a0dd3" +dependencies = [ + "gemm 0.18.2", + "half", + "libloading", + "memmap2", + "num", + "num-traits", + "num_cpus", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.69", + "tracing", + "yoke", +] + [[package]] name = "ui" version = "0.1.0" @@ -18866,7 +19388,7 @@ dependencies = [ "reqwest 0.11.27", "scratch", "semver", - "zip", + "zip 0.6.6", ] [[package]] @@ -19052,7 +19574,7 @@ dependencies = [ "ctrlc", "parking_lot", "rayon", - "thiserror 2.0.12", + "thiserror 2.0.16", "windows 0.61.1", "windows-future", ] @@ -20081,7 +20603,7 @@ dependencies = [ "syn 1.0.109", "syn 2.0.101", "sync_wrapper 1.0.2", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "time-macros", "tokio", @@ -20906,7 +21428,7 @@ dependencies = [ "telemetry", "telemetry_events", "theme", - "thiserror 2.0.12", + "thiserror 2.0.16", "tree-sitter-go", "tree-sitter-rust", "ui", @@ -20975,6 +21497,21 @@ dependencies = [ "zstd", ] +[[package]] +name = "zip" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "indexmap", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "zlib-rs" version = "0.5.0" diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index 85274f651417f8df91e2f785056e5ee8da0220de..c52230e670ca1ec8e4b9ee15355787e6fab7619a 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -17,6 +17,7 @@ anyhow.workspace = true async-tar.workspace = true collections.workspace = true crossbeam.workspace = true +denoise = { path = "/home/davidsk/Documents/denoise"} gpui.workspace = true log.workspace = true parking_lot.workspace = true diff --git a/crates/audio/src/audio.rs b/crates/audio/src/audio.rs index 44f2d5e07d5e7ddb9bbb18925bb8bdbe7581e903..c6566306b83766a3a76b652b6455166356a79790 100644 --- a/crates/audio/src/audio.rs +++ b/crates/audio/src/audio.rs @@ -9,7 +9,7 @@ mod non_windows_and_freebsd_deps { pub(super) use log::info; pub(super) use parking_lot::Mutex; pub(super) use rodio::cpal::Sample; - pub(super) use rodio::source::{LimitSettings, UniformSourceIterator}; + pub(super) use rodio::source::LimitSettings; pub(super) use std::sync::Arc; } @@ -166,7 +166,8 @@ impl Audio { .open_stream()?; info!("Opened microphone: {:?}", stream.config()); - let (replay, stream) = UniformSourceIterator::new(stream, CHANNEL_COUNT, SAMPLE_RATE) + let (replay, stream) = stream + .constant_params(CHANNEL_COUNT, SAMPLE_RATE) .limit(LimitSettings::live_performance()) .process_buffer::(move |buffer| { let mut int_buffer: [i16; _] = buffer.map(|s| s.to_sample()); @@ -187,6 +188,13 @@ impl Audio { } } }) + .constant_params(nz!(1), nz!(16_000)) + .denoise() + .context("Could not set up denoiser")? + .periodic_access(Duration::from_millis(100), move |denoise| { + denoise.set_enabled(LIVE_SETTINGS.denoise.load(Ordering::Relaxed)); + }) + .constant_params(CHANNEL_COUNT, SAMPLE_RATE) .automatic_gain_control(1.0, 4.0, 0.0, 5.0) .periodic_access(Duration::from_millis(100), move |agc_source| { agc_source.set_enabled(LIVE_SETTINGS.control_input_volume.load(Ordering::Relaxed)); diff --git a/crates/audio/src/audio_settings.rs b/crates/audio/src/audio_settings.rs index ea0ea5f3558e015f5579cca43eeb8c529273cb52..b2a3940881b9f6d8b2a7eeb0b7457eccde99c8cd 100644 --- a/crates/audio/src/audio_settings.rs +++ b/crates/audio/src/audio_settings.rs @@ -24,6 +24,16 @@ pub struct AudioSettings { /// too loud quieter. This only affects how things sound for you. #[serde(rename = "experimental.control_output_volume", default)] pub control_output_volume: bool, + /// Requires 'rodio_audio: true' + /// + /// Use the new denoising neural network to remove background noises like + /// typing, airconditioning and passing cars from your microphone input. + /// When your in a noisy environment this will make your speech easier to + /// understand for the other call members. + /// + /// Note: does not work well on music. + #[serde(rename = "experimental.denoise", default)] + pub denoise: bool, } /// Configuration of audio in Zed. @@ -47,6 +57,16 @@ pub struct AudioSettingsContent { /// too loud quieter. This only affects how things sound for you. #[serde(rename = "experimental.control_output_volume", default)] pub control_output_volume: bool, + /// Requires 'rodio_audio: true' + /// + /// Use the new denoising neural network to remove background noises like + /// typing, airconditioning and passing cars from your microphone input. + /// When your in a noisy environment this will make your speech easier to + /// understand for the other call members. + /// + /// Note: does not work well on music. + #[serde(rename = "experimental.denoise", default)] + pub denoise: bool, } impl Settings for AudioSettings { @@ -63,6 +83,7 @@ impl Settings for AudioSettings { pub(crate) struct LiveSettings { pub(crate) control_input_volume: AtomicBool, pub(crate) control_output_volume: AtomicBool, + pub(crate) denoise: AtomicBool, } impl LiveSettings { @@ -76,6 +97,9 @@ impl LiveSettings { AudioSettings::get_global(cx).control_output_volume, Ordering::Relaxed, ); + LIVE_SETTINGS + .control_output_volume + .store(AudioSettings::get_global(cx).denoise, Ordering::Relaxed); }) .detach(); @@ -86,6 +110,9 @@ impl LiveSettings { LIVE_SETTINGS .control_output_volume .store(init_settings.control_output_volume, Ordering::Relaxed); + LIVE_SETTINGS + .control_output_volume + .store(init_settings.denoise, Ordering::Relaxed); } } @@ -96,4 +123,5 @@ impl LiveSettings { pub(crate) static LIVE_SETTINGS: LiveSettings = LiveSettings { control_input_volume: AtomicBool::new(true), control_output_volume: AtomicBool::new(true), + denoise: AtomicBool::new(true), }; diff --git a/crates/audio/src/rodio_ext.rs b/crates/audio/src/rodio_ext.rs index ba4e4ff0554dd3c9bc2a7e2691de270c0d00908b..edcb49fea8ad2c10b63301803c9fa46bbd86d66f 100644 --- a/crates/audio/src/rodio_ext.rs +++ b/crates/audio/src/rodio_ext.rs @@ -7,7 +7,8 @@ use std::{ }; use crossbeam::queue::ArrayQueue; -use rodio::{ChannelCount, Sample, SampleRate, Source}; +use denoise::{Denoiser, DenoiserError}; +use rodio::{ChannelCount, Sample, SampleRate, Source, source::UniformSourceIterator}; #[derive(Debug, thiserror::Error)] #[error("Replay duration is too short must be >= 100ms")] @@ -25,6 +26,12 @@ pub trait RodioExt: Source + Sized { duration: Duration, ) -> Result<(Replay, Replayable), ReplayDurationTooShort>; fn take_samples(self, n: usize) -> TakeSamples; + fn denoise(self) -> Result, DenoiserError>; + fn constant_params( + self, + channel_count: ChannelCount, + sample_rate: SampleRate, + ) -> UniformSourceIterator; } impl RodioExt for S { @@ -101,6 +108,18 @@ impl RodioExt for S { left_to_take: n, } } + fn denoise(self) -> Result, DenoiserError> { + let res = Denoiser::try_new(self); + log::info!("result of new: {res:?}"); + res + } + fn constant_params( + self, + channel_count: ChannelCount, + sample_rate: SampleRate, + ) -> UniformSourceIterator { + UniformSourceIterator::new(self, channel_count, sample_rate) + } } pub struct TakeSamples { diff --git a/crates/livekit_client/src/lib.rs b/crates/livekit_client/src/lib.rs index 055aa3704e06f25a21c69294343539289d8acb49..71638df2ec0390ffaa2b30dd25aec04369e716d9 100644 --- a/crates/livekit_client/src/lib.rs +++ b/crates/livekit_client/src/lib.rs @@ -9,19 +9,19 @@ use rodio::DeviceTrait as _; mod record; pub use record::CaptureInput; -#[cfg(not(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu"), - target_os = "freebsd" -)))] +// #[cfg(not(any( +// test, +// feature = "test-support", +// all(target_os = "windows", target_env = "gnu"), +// target_os = "freebsd" +// )))] mod livekit_client; -#[cfg(not(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu"), - target_os = "freebsd" -)))] +// #[cfg(not(any( +// test, +// feature = "test-support", +// all(target_os = "windows", target_env = "gnu"), +// target_os = "freebsd" +// )))] pub use livekit_client::*; // If you need proper LSP in livekit_client you've got to comment @@ -29,27 +29,27 @@ pub use livekit_client::*; // - the mods: mock_client & test and their conditional blocks // - the pub use mock_client::* and their conditional blocks -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu"), - target_os = "freebsd" -))] -mod mock_client; -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu"), - target_os = "freebsd" -))] -pub mod test; -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu"), - target_os = "freebsd" -))] -pub use mock_client::*; +// #[cfg(any( +// test, +// feature = "test-support", +// all(target_os = "windows", target_env = "gnu"), +// target_os = "freebsd" +// ))] +// mod mock_client; +// #[cfg(any( +// test, +// feature = "test-support", +// all(target_os = "windows", target_env = "gnu"), +// target_os = "freebsd" +// ))] +// pub mod test; +// #[cfg(any( +// test, +// feature = "test-support", +// all(target_os = "windows", target_env = "gnu"), +// target_os = "freebsd" +// ))] +// pub use mock_client::*; #[derive(Debug, Clone)] pub enum Participant { diff --git a/crates/livekit_client/src/livekit_client/playback.rs b/crates/livekit_client/src/livekit_client/playback.rs index 7c866113103a883e7e7a2d9d3f5651d833d7e637..1689a3cc8a1559d647eda866c44532d1a400c128 100644 --- a/crates/livekit_client/src/livekit_client/playback.rs +++ b/crates/livekit_client/src/livekit_client/playback.rs @@ -184,15 +184,20 @@ impl AudioStack { let rodio_pipeline = AudioSettings::try_read_global(cx, |setting| setting.rodio_audio).unwrap_or_default(); let capture_task = if rodio_pipeline { - info!("Using experimental.rodio_audio audio pipeline"); + info!("Using experimental.rodio_audio audio pipeline for input"); let voip_parts = audio::VoipParts::new(cx)?; // Audio needs to run real-time and should never be paused. That is why we are using a // normal std::thread and not a background task thread::spawn(move || { // microphone is non send on mac - let microphone = audio::Audio::open_microphone(voip_parts)?; + let microphone = match audio::Audio::open_microphone(voip_parts) { + Err(e) => { + log::error!("Could not open microphone: {e}"); + return; + } + Ok(m) => m, + }; send_to_livekit(frame_tx, microphone); - Ok::<(), anyhow::Error>(()) }); Task::ready(Ok(())) } else {