fs: Replace a bunch of uses of smol::fs with manual impls (#40172)

Piotr Osiewicz and Smit Barmase created 2 weeks ago

smol::fs uses a separate threadpool, which is a bit yuck.

This PR also added a benchmark you can use to run a full worktree scan
(initial one, that is) for arbitrary worktree.. and refactored worktree
scanner to use async locks, as otherwise tests were deadlocking. :)
I've benchmarked it against Zed, Linux and Chromium and saw a ~60% drop
in initial worktree scan times across the board.
Release Notes:

- Significantly (3.3x speedup over the old implementation) improved
speed of Zed's worktree scanner, that's responsible for synchronizing
the state of your project with the state of files on hard drive.

---------

Co-authored-by: Smit Barmase <heysmitbarmase@gmail.com>

Change summary

Cargo.lock                             |  48 +++
Cargo.toml                             |   3 
crates/editor/src/editor_tests.rs      |   1 
crates/fs/src/fs.rs                    |  93 +++++--
crates/fs_benchmarks/Cargo.toml        |  13 +
crates/fs_benchmarks/LICENSE-GPL       |   1 
crates/fs_benchmarks/src/main.rs       |  32 ++
crates/worktree/Cargo.toml             |   1 
crates/worktree/src/worktree.rs        | 355 ++++++++++++++++-----------
crates/worktree/src/worktree_tests.rs  |   3 
crates/worktree_benchmarks/Cargo.toml  |  15 +
crates/worktree_benchmarks/LICENSE-GPL |   1 
crates/worktree_benchmarks/src/main.rs |  54 ++++
13 files changed, 432 insertions(+), 188 deletions(-)

Detailed changes

Cargo.lock 🔗

@@ -1166,7 +1166,7 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09f7e37c0ed80b2a977691c47dae8625cfb21e205827106c64f7c588766b2e50"
 dependencies = [
- "async-lock",
+ "async-lock 3.4.1",
  "blocking",
  "futures-lite 2.6.0",
 ]
@@ -1180,7 +1180,7 @@ dependencies = [
  "async-channel 2.3.1",
  "async-executor",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "blocking",
  "futures-lite 2.6.0",
  "once_cell",
@@ -1192,7 +1192,7 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19634d6336019ef220f09fd31168ce5c184b295cbf80345437cc36094ef223ca"
 dependencies = [
- "async-lock",
+ "async-lock 3.4.1",
  "cfg-if",
  "concurrent-queue",
  "futures-io",
@@ -1204,6 +1204,15 @@ dependencies = [
  "windows-sys 0.60.2",
 ]
 
+[[package]]
+name = "async-lock"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
+dependencies = [
+ "event-listener 2.5.3",
+]
+
 [[package]]
 name = "async-lock"
 version = "3.4.1"
@@ -1243,7 +1252,7 @@ checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb"
 dependencies = [
  "async-channel 2.3.1",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "async-signal",
  "async-task",
  "blocking",
@@ -1272,7 +1281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3"
 dependencies = [
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "atomic-waker",
  "cfg-if",
  "futures-core",
@@ -1293,7 +1302,7 @@ dependencies = [
  "async-channel 1.9.0",
  "async-global-executor",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "async-process",
  "crossbeam-utils",
  "futures-channel",
@@ -6528,6 +6537,15 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "fs_benchmarks"
+version = "0.1.0"
+dependencies = [
+ "fs",
+ "gpui",
+ "workspace-hack",
+]
+
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -11034,7 +11052,7 @@ dependencies = [
  "ashpd 0.12.0",
  "async-fs",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "blocking",
  "cbc",
  "cipher",
@@ -15735,7 +15753,7 @@ dependencies = [
  "async-executor",
  "async-fs",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "async-net",
  "async-process",
  "blocking",
@@ -20849,6 +20867,7 @@ name = "worktree"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "async-lock 2.8.0",
  "clock",
  "collections",
  "fs",
@@ -20879,6 +20898,17 @@ dependencies = [
  "zlog",
 ]
 
+[[package]]
+name = "worktree_benchmarks"
+version = "0.1.0"
+dependencies = [
+ "fs",
+ "gpui",
+ "settings",
+ "workspace-hack",
+ "worktree",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"
@@ -21150,7 +21180,7 @@ dependencies = [
  "async-broadcast",
  "async-executor",
  "async-io",
- "async-lock",
+ "async-lock 3.4.1",
  "async-process",
  "async-recursion",
  "async-task",

Cargo.toml 🔗

@@ -222,7 +222,7 @@ members = [
 
     "tooling/perf",
     "tooling/workspace-hack",
-    "tooling/xtask",
+    "tooling/xtask", "crates/fs_benchmarks", "crates/worktree_benchmarks",
 ]
 default-members = ["crates/zed"]
 
@@ -455,6 +455,7 @@ async-compat = "0.2.1"
 async-compression = { version = "0.4", features = ["gzip", "futures-io"] }
 async-dispatcher = "0.1"
 async-fs = "2.1"
+async-lock = "2.1"
 async-pipe = { git = "https://github.com/zed-industries/async-pipe-rs", rev = "82d00a04211cf4e1236029aa03e6b6ce2a74c553" }
 async-recursion = "1.0.0"
 async-tar = "0.5.0"

crates/editor/src/editor_tests.rs 🔗

@@ -12509,6 +12509,7 @@ async fn test_strip_whitespace_and_format_via_lsp(cx: &mut TestAppContext) {
     )
     .await;
 
+    cx.run_until_parked();
     // Set up a buffer white some trailing whitespace and no trailing newline.
     cx.set_state(
         &[

crates/fs/src/fs.rs 🔗

@@ -7,6 +7,7 @@ pub mod fs_watcher;
 use anyhow::{Context as _, Result, anyhow};
 #[cfg(any(target_os = "linux", target_os = "freebsd"))]
 use ashpd::desktop::trash;
+use futures::stream::iter;
 use gpui::App;
 use gpui::BackgroundExecutor;
 use gpui::Global;
@@ -562,12 +563,17 @@ impl Fs for RealFs {
 
     async fn load(&self, path: &Path) -> Result<String> {
         let path = path.to_path_buf();
-        let text = smol::unblock(|| std::fs::read_to_string(path)).await?;
-        Ok(text)
+        self.executor
+            .spawn(async move { Ok(std::fs::read_to_string(path)?) })
+            .await
     }
+
     async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>> {
         let path = path.to_path_buf();
-        let bytes = smol::unblock(|| std::fs::read(path)).await?;
+        let bytes = self
+            .executor
+            .spawn(async move { std::fs::read(path) })
+            .await?;
         Ok(bytes)
     }
 
@@ -635,30 +641,46 @@ impl Fs for RealFs {
         if let Some(path) = path.parent() {
             self.create_dir(path).await?;
         }
-        smol::fs::write(path, content).await?;
-        Ok(())
+        let path = path.to_owned();
+        let contents = content.to_owned();
+        self.executor
+            .spawn(async move {
+                std::fs::write(path, contents)?;
+                Ok(())
+            })
+            .await
     }
 
     async fn canonicalize(&self, path: &Path) -> Result<PathBuf> {
-        Ok(smol::fs::canonicalize(path)
+        let path = path.to_owned();
+        self.executor
+            .spawn(async move {
+                std::fs::canonicalize(&path).with_context(|| format!("canonicalizing {path:?}"))
+            })
             .await
-            .with_context(|| format!("canonicalizing {path:?}"))?)
     }
 
     async fn is_file(&self, path: &Path) -> bool {
-        smol::fs::metadata(path)
+        let path = path.to_owned();
+        self.executor
+            .spawn(async move { std::fs::metadata(path).is_ok_and(|metadata| metadata.is_file()) })
             .await
-            .is_ok_and(|metadata| metadata.is_file())
     }
 
     async fn is_dir(&self, path: &Path) -> bool {
-        smol::fs::metadata(path)
+        let path = path.to_owned();
+        self.executor
+            .spawn(async move { std::fs::metadata(path).is_ok_and(|metadata| metadata.is_dir()) })
             .await
-            .is_ok_and(|metadata| metadata.is_dir())
     }
 
     async fn metadata(&self, path: &Path) -> Result<Option<Metadata>> {
-        let symlink_metadata = match smol::fs::symlink_metadata(path).await {
+        let path_buf = path.to_owned();
+        let symlink_metadata = match self
+            .executor
+            .spawn(async move { std::fs::symlink_metadata(&path_buf) })
+            .await
+        {
             Ok(metadata) => metadata,
             Err(err) => {
                 return match (err.kind(), err.raw_os_error()) {
@@ -669,19 +691,28 @@ impl Fs for RealFs {
             }
         };
 
-        let path_buf = path.to_path_buf();
-        let path_exists = smol::unblock(move || {
-            path_buf
-                .try_exists()
-                .with_context(|| format!("checking existence for path {path_buf:?}"))
-        })
-        .await?;
         let is_symlink = symlink_metadata.file_type().is_symlink();
-        let metadata = match (is_symlink, path_exists) {
-            (true, true) => smol::fs::metadata(path)
-                .await
-                .with_context(|| "accessing symlink for path {path}")?,
-            _ => symlink_metadata,
+        let metadata = if is_symlink {
+            let path_buf = path.to_path_buf();
+            let path_exists = self
+                .executor
+                .spawn(async move {
+                    path_buf
+                        .try_exists()
+                        .with_context(|| format!("checking existence for path {path_buf:?}"))
+                })
+                .await?;
+            if path_exists {
+                let path_buf = path.to_path_buf();
+                self.executor
+                    .spawn(async move { std::fs::metadata(path_buf) })
+                    .await
+                    .with_context(|| "accessing symlink for path {path}")?
+            } else {
+                symlink_metadata
+            }
+        } else {
+            symlink_metadata
         };
 
         #[cfg(unix)]
@@ -707,7 +738,11 @@ impl Fs for RealFs {
     }
 
     async fn read_link(&self, path: &Path) -> Result<PathBuf> {
-        let path = smol::fs::read_link(path).await?;
+        let path = path.to_owned();
+        let path = self
+            .executor
+            .spawn(async move { std::fs::read_link(&path) })
+            .await?;
         Ok(path)
     }
 
@@ -715,7 +750,13 @@ impl Fs for RealFs {
         &self,
         path: &Path,
     ) -> Result<Pin<Box<dyn Send + Stream<Item = Result<PathBuf>>>>> {
-        let result = smol::fs::read_dir(path).await?.map(|entry| match entry {
+        let path = path.to_owned();
+        let result = iter(
+            self.executor
+                .spawn(async move { std::fs::read_dir(path) })
+                .await?,
+        )
+        .map(|entry| match entry {
             Ok(entry) => Ok(entry.path()),
             Err(error) => Err(anyhow!("failed to read dir entry {error:?}")),
         });

crates/fs_benchmarks/Cargo.toml 🔗

@@ -0,0 +1,13 @@
+[package]
+name = "fs_benchmarks"
+version = "0.1.0"
+publish.workspace = true
+edition.workspace = true
+
+[dependencies]
+fs.workspace = true
+gpui = {workspace = true, features = ["windows-manifest"]}
+workspace-hack.workspace = true
+
+[lints]
+workspace = true

crates/fs_benchmarks/LICENSE-GPL 🔗

@@ -0,0 +1 @@
+../../LICENSE-GPL

crates/fs_benchmarks/src/main.rs 🔗

@@ -0,0 +1,32 @@
+use fs::Fs;
+use gpui::{AppContext, Application};
+fn main() {
+    let Some(path_to_read) = std::env::args().nth(1) else {
+        println!("Expected path to read as 1st argument.");
+        return;
+    };
+
+    let _ = Application::headless().run(|cx| {
+        let fs = fs::RealFs::new(None, cx.background_executor().clone());
+        cx.background_spawn(async move {
+            let timer = std::time::Instant::now();
+            let result = fs.load_bytes(path_to_read.as_ref()).await;
+            let elapsed = timer.elapsed();
+            if let Err(e) = result {
+                println!("Failed `load_bytes` after {elapsed:?} with error `{e}`");
+            } else {
+                println!("Took {elapsed:?} to read {} bytes", result.unwrap().len());
+            };
+            let timer = std::time::Instant::now();
+            let result = fs.metadata(path_to_read.as_ref()).await;
+            let elapsed = timer.elapsed();
+            if let Err(e) = result {
+                println!("Failed `metadata` after {elapsed:?} with error `{e}`");
+            } else {
+                println!("Took {elapsed:?} to query metadata");
+            };
+            std::process::exit(0);
+        })
+        .detach();
+    });
+}

crates/worktree/Cargo.toml 🔗

@@ -24,6 +24,7 @@ test-support = [
 
 [dependencies]
 anyhow.workspace = true
+async-lock.workspace = true
 clock.workspace = true
 collections.workspace = true
 fs.workspace = true

crates/worktree/src/worktree.rs 🔗

@@ -64,7 +64,7 @@ use std::{
 use sum_tree::{Bias, Dimensions, Edit, KeyedItem, SeekTarget, SumTree, Summary, TreeMap, TreeSet};
 use text::{LineEnding, Rope};
 use util::{
-    ResultExt, debug_panic,
+    ResultExt, debug_panic, maybe,
     paths::{PathMatcher, PathStyle, SanitizedPath, home_dir},
     rel_path::RelPath,
 };
@@ -226,7 +226,7 @@ impl Default for WorkDirectory {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct LocalSnapshot {
     snapshot: Snapshot,
     global_gitignore: Option<Arc<Gitignore>>,
@@ -239,6 +239,7 @@ pub struct LocalSnapshot {
     /// The file handle of the worktree root. `None` if the worktree is a directory.
     /// (so we can find it after it's been moved)
     root_file_handle: Option<Arc<dyn fs::FileHandle>>,
+    executor: BackgroundExecutor,
 }
 
 struct BackgroundScannerState {
@@ -321,7 +322,6 @@ impl DerefMut for LocalSnapshot {
     }
 }
 
-#[derive(Debug)]
 enum ScanState {
     Started,
     Updated {
@@ -402,6 +402,7 @@ impl Worktree {
                     PathStyle::local(),
                 ),
                 root_file_handle,
+                executor: cx.background_executor().clone(),
             };
 
             let worktree_id = snapshot.id();
@@ -1069,7 +1070,7 @@ impl LocalWorktree {
                     scan_requests_rx,
                     path_prefixes_to_scan_rx,
                     next_entry_id,
-                    state: Mutex::new(BackgroundScannerState {
+                    state: async_lock::Mutex::new(BackgroundScannerState {
                         prev_snapshot: snapshot.snapshot.clone(),
                         snapshot,
                         scanned_dirs: Default::default(),
@@ -2442,7 +2443,7 @@ impl LocalSnapshot {
         log::trace!("insert entry {:?}", entry.path);
         if entry.is_file() && entry.path.file_name() == Some(&GITIGNORE) {
             let abs_path = self.absolutize(&entry.path);
-            match smol::block_on(build_gitignore(&abs_path, fs)) {
+            match self.executor.block(build_gitignore(&abs_path, fs)) {
                 Ok(ignore) => {
                     self.ignores_by_parent_abs_path
                         .insert(abs_path.parent().unwrap().into(), (Arc::new(ignore), true));
@@ -2493,7 +2494,12 @@ impl LocalSnapshot {
         inodes
     }
 
-    fn ignore_stack_for_abs_path(&self, abs_path: &Path, is_dir: bool, fs: &dyn Fs) -> IgnoreStack {
+    async fn ignore_stack_for_abs_path(
+        &self,
+        abs_path: &Path,
+        is_dir: bool,
+        fs: &dyn Fs,
+    ) -> IgnoreStack {
         let mut new_ignores = Vec::new();
         let mut repo_root = None;
         for (index, ancestor) in abs_path.ancestors().enumerate() {
@@ -2504,9 +2510,8 @@ impl LocalSnapshot {
                     new_ignores.push((ancestor, None));
                 }
             }
-            let metadata = smol::block_on(fs.metadata(&ancestor.join(DOT_GIT)))
-                .ok()
-                .flatten();
+
+            let metadata = fs.metadata(&ancestor.join(DOT_GIT)).await.ok().flatten();
             if metadata.is_some() {
                 repo_root = Some(Arc::from(ancestor));
                 break;
@@ -2652,7 +2657,7 @@ impl BackgroundScannerState {
                 .any(|p| entry.path.starts_with(p))
     }
 
-    fn enqueue_scan_dir(
+    async fn enqueue_scan_dir(
         &self,
         abs_path: Arc<Path>,
         entry: &Entry,
@@ -2660,7 +2665,10 @@ impl BackgroundScannerState {
         fs: &dyn Fs,
     ) {
         let path = entry.path.clone();
-        let ignore_stack = self.snapshot.ignore_stack_for_abs_path(&abs_path, true, fs);
+        let ignore_stack = self
+            .snapshot
+            .ignore_stack_for_abs_path(&abs_path, true, fs)
+            .await;
         let mut ancestor_inodes = self.snapshot.ancestor_inodes_for_path(&path);
 
         if !ancestor_inodes.contains(&entry.inode) {
@@ -2698,11 +2706,17 @@ impl BackgroundScannerState {
         }
     }
 
-    fn insert_entry(&mut self, mut entry: Entry, fs: &dyn Fs, watcher: &dyn Watcher) -> Entry {
+    async fn insert_entry(
+        &mut self,
+        mut entry: Entry,
+        fs: &dyn Fs,
+        watcher: &dyn Watcher,
+    ) -> Entry {
         self.reuse_entry_id(&mut entry);
         let entry = self.snapshot.insert_entry(entry, fs);
         if entry.path.file_name() == Some(&DOT_GIT) {
-            self.insert_git_repository(entry.path.clone(), fs, watcher);
+            self.insert_git_repository(entry.path.clone(), fs, watcher)
+                .await;
         }
 
         #[cfg(test)]
@@ -2833,7 +2847,7 @@ impl BackgroundScannerState {
         self.snapshot.check_invariants(false);
     }
 
-    fn insert_git_repository(
+    async fn insert_git_repository(
         &mut self,
         dot_git_path: Arc<RelPath>,
         fs: &dyn Fs,
@@ -2874,10 +2888,11 @@ impl BackgroundScannerState {
             fs,
             watcher,
         )
+        .await
         .log_err();
     }
 
-    fn insert_git_repository_for_path(
+    async fn insert_git_repository_for_path(
         &mut self,
         work_directory: WorkDirectory,
         dot_git_abs_path: Arc<Path>,
@@ -2899,7 +2914,7 @@ impl BackgroundScannerState {
         let work_directory_abs_path = self.snapshot.work_directory_abs_path(&work_directory);
 
         let (repository_dir_abs_path, common_dir_abs_path) =
-            discover_git_paths(&dot_git_abs_path, fs);
+            discover_git_paths(&dot_git_abs_path, fs).await;
         watcher
             .add(&common_dir_abs_path)
             .context("failed to add common directory to watcher")
@@ -3543,7 +3558,7 @@ impl<'a> sum_tree::Dimension<'a, EntrySummary> for PathKey {
 }
 
 struct BackgroundScanner {
-    state: Mutex<BackgroundScannerState>,
+    state: async_lock::Mutex<BackgroundScannerState>,
     fs: Arc<dyn Fs>,
     fs_case_sensitive: bool,
     status_updates_tx: UnboundedSender<ScanState>,
@@ -3569,31 +3584,39 @@ impl BackgroundScanner {
         // If the worktree root does not contain a git repository, then find
         // the git repository in an ancestor directory. Find any gitignore files
         // in ancestor directories.
-        let root_abs_path = self.state.lock().snapshot.abs_path.clone();
+        let root_abs_path = self.state.lock().await.snapshot.abs_path.clone();
         let (ignores, repo) = discover_ancestor_git_repo(self.fs.clone(), &root_abs_path).await;
         self.state
             .lock()
+            .await
             .snapshot
             .ignores_by_parent_abs_path
             .extend(ignores);
-        let containing_git_repository = repo.and_then(|(ancestor_dot_git, work_directory)| {
-            self.state
-                .lock()
-                .insert_git_repository_for_path(
-                    work_directory,
-                    ancestor_dot_git.clone().into(),
-                    self.fs.as_ref(),
-                    self.watcher.as_ref(),
-                )
-                .log_err()?;
-            Some(ancestor_dot_git)
-        });
+        let containing_git_repository = if let Some((ancestor_dot_git, work_directory)) = repo {
+            maybe!(async {
+                self.state
+                    .lock()
+                    .await
+                    .insert_git_repository_for_path(
+                        work_directory,
+                        ancestor_dot_git.clone().into(),
+                        self.fs.as_ref(),
+                        self.watcher.as_ref(),
+                    )
+                    .await
+                    .log_err()?;
+                Some(ancestor_dot_git)
+            })
+            .await
+        } else {
+            None
+        };
 
         log::trace!("containing git repository: {containing_git_repository:?}");
 
         let mut global_gitignore_events =
             if let Some(global_gitignore_path) = &paths::global_gitignore_path() {
-                self.state.lock().snapshot.global_gitignore =
+                self.state.lock().await.snapshot.global_gitignore =
                     if self.fs.is_file(&global_gitignore_path).await {
                         build_gitignore(global_gitignore_path, self.fs.as_ref())
                             .await
@@ -3607,31 +3630,34 @@ impl BackgroundScanner {
                     .await
                     .0
             } else {
-                self.state.lock().snapshot.global_gitignore = None;
+                self.state.lock().await.snapshot.global_gitignore = None;
                 Box::pin(futures::stream::empty())
             };
 
         let (scan_job_tx, scan_job_rx) = channel::unbounded();
         {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             state.snapshot.scan_id += 1;
             if let Some(mut root_entry) = state.snapshot.root_entry().cloned() {
-                let ignore_stack = state.snapshot.ignore_stack_for_abs_path(
-                    root_abs_path.as_path(),
-                    true,
-                    self.fs.as_ref(),
-                );
+                let ignore_stack = state
+                    .snapshot
+                    .ignore_stack_for_abs_path(root_abs_path.as_path(), true, self.fs.as_ref())
+                    .await;
                 if ignore_stack.is_abs_path_ignored(root_abs_path.as_path(), true) {
                     root_entry.is_ignored = true;
-                    state.insert_entry(root_entry.clone(), self.fs.as_ref(), self.watcher.as_ref());
+                    state
+                        .insert_entry(root_entry.clone(), self.fs.as_ref(), self.watcher.as_ref())
+                        .await;
                 }
                 if root_entry.is_dir() {
-                    state.enqueue_scan_dir(
-                        root_abs_path.as_path().into(),
-                        &root_entry,
-                        &scan_job_tx,
-                        self.fs.as_ref(),
-                    );
+                    state
+                        .enqueue_scan_dir(
+                            root_abs_path.as_path().into(),
+                            &root_entry,
+                            &scan_job_tx,
+                            self.fs.as_ref(),
+                        )
+                        .await;
                 }
             }
         };
@@ -3640,11 +3666,11 @@ impl BackgroundScanner {
         drop(scan_job_tx);
         self.scan_dirs(true, scan_job_rx).await;
         {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             state.snapshot.completed_scan_id = state.snapshot.scan_id;
         }
 
-        self.send_status_update(false, SmallVec::new());
+        self.send_status_update(false, SmallVec::new()).await;
 
         // Process any any FS events that occurred while performing the initial scan.
         // For these events, update events cannot be as precise, because we didn't
@@ -3689,7 +3715,7 @@ impl BackgroundScanner {
                     if did_scan {
                         let abs_path =
                         {
-                            let mut state = self.state.lock();
+                            let mut state = self.state.lock().await;
                             state.path_prefixes_to_scan.insert(request.path.clone());
                             state.snapshot.absolutize(&request.path)
                         };
@@ -3698,7 +3724,7 @@ impl BackgroundScanner {
                             self.process_events(vec![abs_path]).await;
                         }
                     }
-                    self.send_status_update(false, request.done);
+                    self.send_status_update(false, request.done).await;
                 }
 
                 paths = fs_events_rx.next().fuse() => {
@@ -3727,7 +3753,7 @@ impl BackgroundScanner {
         request.relative_paths.sort_unstable();
         self.forcibly_load_paths(&request.relative_paths).await;
 
-        let root_path = self.state.lock().snapshot.abs_path.clone();
+        let root_path = self.state.lock().await.snapshot.abs_path.clone();
         let root_canonical_path = self.fs.canonicalize(root_path.as_path()).await;
         let root_canonical_path = match &root_canonical_path {
             Ok(path) => SanitizedPath::new(path),
@@ -3749,7 +3775,7 @@ impl BackgroundScanner {
             .collect::<Vec<_>>();
 
         {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             let is_idle = state.snapshot.completed_scan_id == state.snapshot.scan_id;
             state.snapshot.scan_id += 1;
             if is_idle {
@@ -3766,12 +3792,12 @@ impl BackgroundScanner {
         )
         .await;
 
-        self.send_status_update(scanning, request.done)
+        self.send_status_update(scanning, request.done).await
     }
 
     async fn process_events(&self, mut abs_paths: Vec<PathBuf>) {
         log::trace!("process events: {abs_paths:?}");
-        let root_path = self.state.lock().snapshot.abs_path.clone();
+        let root_path = self.state.lock().await.snapshot.abs_path.clone();
         let root_canonical_path = self.fs.canonicalize(root_path.as_path()).await;
         let root_canonical_path = match &root_canonical_path {
             Ok(path) => SanitizedPath::new(path),
@@ -3779,6 +3805,7 @@ impl BackgroundScanner {
                 let new_path = self
                     .state
                     .lock()
+                    .await
                     .snapshot
                     .root_file_handle
                     .clone()
@@ -3811,24 +3838,31 @@ impl BackgroundScanner {
         let mut dot_git_abs_paths = Vec::new();
         abs_paths.sort_unstable();
         abs_paths.dedup_by(|a, b| a.starts_with(b));
-        abs_paths.retain(|abs_path| {
+        {
+            let snapshot = &self.state.lock().await.snapshot;
+            abs_paths.retain(|abs_path| {
             let abs_path = &SanitizedPath::new(abs_path);
 
-            let snapshot = &self.state.lock().snapshot;
+
             {
                 let mut is_git_related = false;
 
-                let dot_git_paths = abs_path.as_path().ancestors().find_map(|ancestor| {
-                    if smol::block_on(is_git_dir(ancestor, self.fs.as_ref())) {
+                let dot_git_paths = self.executor.block(maybe!(async  {
+                    let mut path = None;
+                    for ancestor in abs_path.as_path().ancestors() {
+
+                    if is_git_dir(ancestor, self.fs.as_ref()).await {
                         let path_in_git_dir = abs_path
                             .as_path()
                             .strip_prefix(ancestor)
                             .expect("stripping off the ancestor");
-                        Some((ancestor.to_owned(), path_in_git_dir.to_owned()))
-                    } else {
-                        None
+                       path = Some((ancestor.to_owned(), path_in_git_dir.to_owned()));
+                       break;
                     }
-                });
+                    }
+                    path
+
+                }));
 
                 if let Some((dot_git_abs_path, path_in_git_dir)) = dot_git_paths {
                     if skipped_files_in_dot_git
@@ -3901,12 +3935,12 @@ impl BackgroundScanner {
                 true
             }
         });
-
+        }
         if relative_paths.is_empty() && dot_git_abs_paths.is_empty() {
             return;
         }
 
-        self.state.lock().snapshot.scan_id += 1;
+        self.state.lock().await.snapshot.scan_id += 1;
 
         let (scan_job_tx, scan_job_rx) = channel::unbounded();
         log::debug!("received fs events {:?}", relative_paths);
@@ -3920,29 +3954,29 @@ impl BackgroundScanner {
         .await;
 
         let affected_repo_roots = if !dot_git_abs_paths.is_empty() {
-            self.update_git_repositories(dot_git_abs_paths)
+            self.update_git_repositories(dot_git_abs_paths).await
         } else {
             Vec::new()
         };
 
         {
-            let mut ignores_to_update = self.ignores_needing_update();
+            let mut ignores_to_update = self.ignores_needing_update().await;
             ignores_to_update.extend(affected_repo_roots);
-            let ignores_to_update = self.order_ignores(ignores_to_update);
-            let snapshot = self.state.lock().snapshot.clone();
+            let ignores_to_update = self.order_ignores(ignores_to_update).await;
+            let snapshot = self.state.lock().await.snapshot.clone();
             self.update_ignore_statuses_for_paths(scan_job_tx, snapshot, ignores_to_update)
                 .await;
             self.scan_dirs(false, scan_job_rx).await;
         }
 
         {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             state.snapshot.completed_scan_id = state.snapshot.scan_id;
             for (_, entry) in mem::take(&mut state.removed_entries) {
                 state.scanned_dirs.remove(&entry.id);
             }
         }
-        self.send_status_update(false, SmallVec::new());
+        self.send_status_update(false, SmallVec::new()).await;
     }
 
     async fn update_global_gitignore(&self, abs_path: &Path) {
@@ -3951,30 +3985,30 @@ impl BackgroundScanner {
             .log_err()
             .map(Arc::new);
         let (prev_snapshot, ignore_stack, abs_path) = {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             state.snapshot.global_gitignore = ignore;
             let abs_path = state.snapshot.abs_path().clone();
-            let ignore_stack =
-                state
-                    .snapshot
-                    .ignore_stack_for_abs_path(&abs_path, true, self.fs.as_ref());
+            let ignore_stack = state
+                .snapshot
+                .ignore_stack_for_abs_path(&abs_path, true, self.fs.as_ref())
+                .await;
             (state.snapshot.clone(), ignore_stack, abs_path)
         };
         let (scan_job_tx, scan_job_rx) = channel::unbounded();
         self.update_ignore_statuses_for_paths(
             scan_job_tx,
             prev_snapshot,
-            vec![(abs_path, ignore_stack)].into_iter(),
+            vec![(abs_path, ignore_stack)],
         )
         .await;
         self.scan_dirs(false, scan_job_rx).await;
-        self.send_status_update(false, SmallVec::new());
+        self.send_status_update(false, SmallVec::new()).await;
     }
 
     async fn forcibly_load_paths(&self, paths: &[Arc<RelPath>]) -> bool {
         let (scan_job_tx, scan_job_rx) = channel::unbounded();
         {
-            let mut state = self.state.lock();
+            let mut state = self.state.lock().await;
             let root_path = state.snapshot.abs_path.clone();
             for path in paths {
                 for ancestor in path.ancestors() {
@@ -3982,12 +4016,14 @@ impl BackgroundScanner {
                         && entry.kind == EntryKind::UnloadedDir
                     {
                         let abs_path = root_path.join(ancestor.as_std_path());
-                        state.enqueue_scan_dir(
-                            abs_path.into(),
-                            entry,
-                            &scan_job_tx,
-                            self.fs.as_ref(),
-                        );
+                        state
+                            .enqueue_scan_dir(
+                                abs_path.into(),
+                                entry,
+                                &scan_job_tx,
+                                self.fs.as_ref(),
+                            )
+                            .await;
                         state.paths_to_scan.insert(path.clone());
                         break;
                     }
@@ -3999,7 +4035,7 @@ impl BackgroundScanner {
             self.scan_dir(&job).await.log_err();
         }
 
-        !mem::take(&mut self.state.lock().paths_to_scan).is_empty()
+        !mem::take(&mut self.state.lock().await.paths_to_scan).is_empty()
     }
 
     async fn scan_dirs(
@@ -4047,7 +4083,7 @@ impl BackgroundScanner {
                                     ) {
                                         Ok(_) => {
                                             last_progress_update_count += 1;
-                                            self.send_status_update(true, SmallVec::new());
+                                            self.send_status_update(true, SmallVec::new()).await;
                                         }
                                         Err(count) => {
                                             last_progress_update_count = count;
@@ -4072,8 +4108,12 @@ impl BackgroundScanner {
             .await;
     }
 
-    fn send_status_update(&self, scanning: bool, barrier: SmallVec<[barrier::Sender; 1]>) -> bool {
-        let mut state = self.state.lock();
+    async fn send_status_update(
+        &self,
+        scanning: bool,
+        barrier: SmallVec<[barrier::Sender; 1]>,
+    ) -> bool {
+        let mut state = self.state.lock().await;
         if state.changed_paths.is_empty() && scanning {
             return true;
         }
@@ -4102,7 +4142,7 @@ impl BackgroundScanner {
         let root_abs_path;
         let root_char_bag;
         {
-            let snapshot = &self.state.lock().snapshot;
+            let snapshot = &self.state.lock().await.snapshot;
             if self.settings.is_path_excluded(&job.path) {
                 log::error!("skipping excluded directory {:?}", job.path);
                 return Ok(());
@@ -4155,12 +4195,14 @@ impl BackgroundScanner {
             };
 
             if child_name == DOT_GIT {
-                let mut state = self.state.lock();
-                state.insert_git_repository(
-                    child_path.clone(),
-                    self.fs.as_ref(),
-                    self.watcher.as_ref(),
-                );
+                let mut state = self.state.lock().await;
+                state
+                    .insert_git_repository(
+                        child_path.clone(),
+                        self.fs.as_ref(),
+                        self.watcher.as_ref(),
+                    )
+                    .await;
             } else if child_name == GITIGNORE {
                 match build_gitignore(&child_abs_path, self.fs.as_ref()).await {
                     Ok(ignore) => {
@@ -4180,7 +4222,7 @@ impl BackgroundScanner {
 
             if self.settings.is_path_excluded(&child_path) {
                 log::debug!("skipping excluded child entry {child_path:?}");
-                self.state.lock().remove_path(&child_path);
+                self.state.lock().await.remove_path(&child_path);
                 continue;
             }
 
@@ -4280,7 +4322,7 @@ impl BackgroundScanner {
             new_entries.push(child_entry);
         }
 
-        let mut state = self.state.lock();
+        let mut state = self.state.lock().await;
 
         // Identify any subdirectories that should not be scanned.
         let mut job_ix = 0;
@@ -4362,7 +4404,7 @@ impl BackgroundScanner {
             None
         };
 
-        let mut state = self.state.lock();
+        let mut state = self.state.lock().await;
         let doing_recursive_update = scan_queue_tx.is_some();
 
         // Remove any entries for paths that no longer exist or are being recursively
@@ -4378,11 +4420,10 @@ impl BackgroundScanner {
             let abs_path: Arc<Path> = root_abs_path.join(path.as_std_path()).into();
             match metadata {
                 Ok(Some((metadata, canonical_path))) => {
-                    let ignore_stack = state.snapshot.ignore_stack_for_abs_path(
-                        &abs_path,
-                        metadata.is_dir,
-                        self.fs.as_ref(),
-                    );
+                    let ignore_stack = state
+                        .snapshot
+                        .ignore_stack_for_abs_path(&abs_path, metadata.is_dir, self.fs.as_ref())
+                        .await;
                     let is_external = !canonical_path.starts_with(&root_canonical_path);
                     let mut fs_entry = Entry::new(
                         path.clone(),
@@ -4414,18 +4455,22 @@ impl BackgroundScanner {
                             || (fs_entry.path.is_empty()
                                 && abs_path.file_name() == Some(OsStr::new(DOT_GIT)))
                         {
-                            state.enqueue_scan_dir(
-                                abs_path,
-                                &fs_entry,
-                                scan_queue_tx,
-                                self.fs.as_ref(),
-                            );
+                            state
+                                .enqueue_scan_dir(
+                                    abs_path,
+                                    &fs_entry,
+                                    scan_queue_tx,
+                                    self.fs.as_ref(),
+                                )
+                                .await;
                         } else {
                             fs_entry.kind = EntryKind::UnloadedDir;
                         }
                     }
 
-                    state.insert_entry(fs_entry.clone(), self.fs.as_ref(), self.watcher.as_ref());
+                    state
+                        .insert_entry(fs_entry.clone(), self.fs.as_ref(), self.watcher.as_ref())
+                        .await;
 
                     if path.is_empty()
                         && let Some((ignores, repo)) = new_ancestor_repo.take()
@@ -4440,6 +4485,7 @@ impl BackgroundScanner {
                                     self.fs.as_ref(),
                                     self.watcher.as_ref(),
                                 )
+                                .await
                                 .log_err();
                         }
                     }
@@ -4478,11 +4524,11 @@ impl BackgroundScanner {
         &self,
         scan_job_tx: Sender<ScanJob>,
         prev_snapshot: LocalSnapshot,
-        mut ignores_to_update: impl Iterator<Item = (Arc<Path>, IgnoreStack)>,
+        ignores_to_update: Vec<(Arc<Path>, IgnoreStack)>,
     ) {
         let (ignore_queue_tx, ignore_queue_rx) = channel::unbounded();
         {
-            while let Some((parent_abs_path, ignore_stack)) = ignores_to_update.next() {
+            for (parent_abs_path, ignore_stack) in ignores_to_update {
                 ignore_queue_tx
                     .send_blocking(UpdateIgnoreStatusJob {
                         abs_path: parent_abs_path,
@@ -4523,11 +4569,11 @@ impl BackgroundScanner {
             .await;
     }
 
-    fn ignores_needing_update(&self) -> Vec<Arc<Path>> {
+    async fn ignores_needing_update(&self) -> Vec<Arc<Path>> {
         let mut ignores_to_update = Vec::new();
 
         {
-            let snapshot = &mut self.state.lock().snapshot;
+            let snapshot = &mut self.state.lock().await.snapshot;
             let abs_path = snapshot.abs_path.clone();
             snapshot
                 .ignores_by_parent_abs_path
@@ -4555,26 +4601,27 @@ impl BackgroundScanner {
         ignores_to_update
     }
 
-    fn order_ignores(
-        &self,
-        mut ignores: Vec<Arc<Path>>,
-    ) -> impl use<> + Iterator<Item = (Arc<Path>, IgnoreStack)> {
+    async fn order_ignores(&self, mut ignores: Vec<Arc<Path>>) -> Vec<(Arc<Path>, IgnoreStack)> {
         let fs = self.fs.clone();
-        let snapshot = self.state.lock().snapshot.clone();
+        let snapshot = self.state.lock().await.snapshot.clone();
         ignores.sort_unstable();
         let mut ignores_to_update = ignores.into_iter().peekable();
-        std::iter::from_fn(move || {
-            let parent_abs_path = ignores_to_update.next()?;
+
+        let mut result = vec![];
+        while let Some(parent_abs_path) = ignores_to_update.next() {
             while ignores_to_update
                 .peek()
                 .map_or(false, |p| p.starts_with(&parent_abs_path))
             {
                 ignores_to_update.next().unwrap();
             }
-            let ignore_stack =
-                snapshot.ignore_stack_for_abs_path(&parent_abs_path, true, fs.as_ref());
-            Some((parent_abs_path, ignore_stack))
-        })
+            let ignore_stack = snapshot
+                .ignore_stack_for_abs_path(&parent_abs_path, true, fs.as_ref())
+                .await;
+            result.push((parent_abs_path, ignore_stack));
+        }
+
+        result
     }
 
     async fn update_ignore_status(&self, job: UpdateIgnoreStatusJob, snapshot: &LocalSnapshot) {
@@ -4606,7 +4653,7 @@ impl BackgroundScanner {
             return;
         };
 
-        if let Ok(Some(metadata)) = smol::block_on(self.fs.metadata(&job.abs_path.join(DOT_GIT)))
+        if let Ok(Some(metadata)) = self.fs.metadata(&job.abs_path.join(DOT_GIT)).await
             && metadata.is_dir
         {
             ignore_stack.repo_root = Some(job.abs_path.clone());
@@ -4626,14 +4673,16 @@ impl BackgroundScanner {
 
                 // Scan any directories that were previously ignored and weren't previously scanned.
                 if was_ignored && !entry.is_ignored && entry.kind.is_unloaded() {
-                    let state = self.state.lock();
+                    let state = self.state.lock().await;
                     if state.should_scan_directory(&entry) {
-                        state.enqueue_scan_dir(
-                            abs_path.clone(),
-                            &entry,
-                            &job.scan_queue,
-                            self.fs.as_ref(),
-                        );
+                        state
+                            .enqueue_scan_dir(
+                                abs_path.clone(),
+                                &entry,
+                                &job.scan_queue,
+                                self.fs.as_ref(),
+                            )
+                            .await;
                     }
                 }
 
@@ -4657,7 +4706,7 @@ impl BackgroundScanner {
             }
         }
 
-        let state = &mut self.state.lock();
+        let state = &mut self.state.lock().await;
         for edit in &entries_by_path_edits {
             if let Edit::Insert(entry) = edit
                 && let Err(ix) = state.changed_paths.binary_search(&entry.path)
@@ -4673,9 +4722,9 @@ impl BackgroundScanner {
         state.snapshot.entries_by_id.edit(entries_by_id_edits, ());
     }
 
-    fn update_git_repositories(&self, dot_git_paths: Vec<PathBuf>) -> Vec<Arc<Path>> {
+    async fn update_git_repositories(&self, dot_git_paths: Vec<PathBuf>) -> Vec<Arc<Path>> {
         log::trace!("reloading repositories: {dot_git_paths:?}");
-        let mut state = self.state.lock();
+        let mut state = self.state.lock().await;
         let scan_id = state.snapshot.scan_id;
         let mut affected_repo_roots = Vec::new();
         for dot_git_dir in dot_git_paths {
@@ -4705,13 +4754,15 @@ impl BackgroundScanner {
                         return Vec::new();
                     };
                     affected_repo_roots.push(dot_git_dir.parent().unwrap().into());
-                    state.insert_git_repository(
-                        RelPath::new(relative, PathStyle::local())
-                            .unwrap()
-                            .into_arc(),
-                        self.fs.as_ref(),
-                        self.watcher.as_ref(),
-                    );
+                    state
+                        .insert_git_repository(
+                            RelPath::new(relative, PathStyle::local())
+                                .unwrap()
+                                .into_arc(),
+                            self.fs.as_ref(),
+                            self.watcher.as_ref(),
+                        )
+                        .await;
                 }
                 Some(local_repository) => {
                     state.snapshot.git_repositories.update(
@@ -4739,7 +4790,7 @@ impl BackgroundScanner {
 
             if exists_in_snapshot
                 || matches!(
-                    smol::block_on(self.fs.metadata(&entry.common_dir_abs_path)),
+                    self.fs.metadata(&entry.common_dir_abs_path).await,
                     Ok(Some(_))
                 )
             {
@@ -5498,11 +5549,13 @@ fn parse_gitfile(content: &str) -> anyhow::Result<&Path> {
     Ok(Path::new(path.trim()))
 }
 
-fn discover_git_paths(dot_git_abs_path: &Arc<Path>, fs: &dyn Fs) -> (Arc<Path>, Arc<Path>) {
+async fn discover_git_paths(dot_git_abs_path: &Arc<Path>, fs: &dyn Fs) -> (Arc<Path>, Arc<Path>) {
     let mut repository_dir_abs_path = dot_git_abs_path.clone();
     let mut common_dir_abs_path = dot_git_abs_path.clone();
 
-    if let Some(path) = smol::block_on(fs.load(dot_git_abs_path))
+    if let Some(path) = fs
+        .load(dot_git_abs_path)
+        .await
         .ok()
         .as_ref()
         .and_then(|contents| parse_gitfile(contents).log_err())
@@ -5511,17 +5564,19 @@ fn discover_git_paths(dot_git_abs_path: &Arc<Path>, fs: &dyn Fs) -> (Arc<Path>,
             .parent()
             .unwrap_or(Path::new(""))
             .join(path);
-        if let Some(path) = smol::block_on(fs.canonicalize(&path)).log_err() {
+        if let Some(path) = fs.canonicalize(&path).await.log_err() {
             repository_dir_abs_path = Path::new(&path).into();
             common_dir_abs_path = repository_dir_abs_path.clone();
-            if let Some(commondir_contents) = smol::block_on(fs.load(&path.join("commondir"))).ok()
-                && let Some(commondir_path) =
-                    smol::block_on(fs.canonicalize(&path.join(commondir_contents.trim()))).log_err()
+
+            if let Some(commondir_contents) = fs.load(&path.join("commondir")).await.ok()
+                && let Some(commondir_path) = fs
+                    .canonicalize(&path.join(commondir_contents.trim()))
+                    .await
+                    .log_err()
             {
                 common_dir_abs_path = commondir_path.as_path().into();
             }
         }
     };
-
     (repository_dir_abs_path, common_dir_abs_path)
 }

crates/worktree/src/worktree_tests.rs 🔗

@@ -734,7 +734,6 @@ async fn test_write_file(cx: &mut TestAppContext) {
         })
         .await
         .unwrap();
-
     worktree.read_with(cx, |tree, _| {
         let tracked = tree
             .entry_for_path(rel_path("tracked-dir/file.txt"))
@@ -1537,7 +1536,7 @@ async fn test_random_worktree_operations_during_initial_scan(
         assert_eq!(
             updated_snapshot.entries(true, 0).collect::<Vec<_>>(),
             final_snapshot.entries(true, 0).collect::<Vec<_>>(),
-            "wrong updates after snapshot {i}: {snapshot:#?} {updates:#?}",
+            "wrong updates after snapshot {i}: {updates:#?}",
         );
     }
 }

crates/worktree_benchmarks/Cargo.toml 🔗

@@ -0,0 +1,15 @@
+[package]
+name = "worktree_benchmarks"
+version = "0.1.0"
+publish.workspace = true
+edition.workspace = true
+
+[dependencies]
+fs.workspace = true
+gpui = { workspace = true, features = ["windows-manifest"] }
+settings.workspace = true
+worktree.workspace = true
+workspace-hack = { version = "0.1", path = "../../tooling/workspace-hack" }
+
+[lints]
+workspace = true

crates/worktree_benchmarks/LICENSE-GPL 🔗

@@ -0,0 +1 @@
+../../LICENSE-GPL

crates/worktree_benchmarks/src/main.rs 🔗

@@ -0,0 +1,54 @@
+use std::{
+    path::Path,
+    sync::{Arc, atomic::AtomicUsize},
+};
+
+use fs::RealFs;
+use gpui::Application;
+use settings::Settings;
+use worktree::{Worktree, WorktreeSettings};
+
+fn main() {
+    let Some(worktree_root_path) = std::env::args().nth(1) else {
+        println!(
+            "Missing path to worktree root\nUsage: bench_background_scan PATH_TO_WORKTREE_ROOT"
+        );
+        return;
+    };
+    let app = Application::headless();
+
+    app.run(|cx| {
+        settings::init(cx);
+        WorktreeSettings::register(cx);
+        let fs = Arc::new(RealFs::new(None, cx.background_executor().clone()));
+
+        cx.spawn(async move |cx| {
+            let worktree = Worktree::local(
+                Path::new(&worktree_root_path),
+                true,
+                fs,
+                Arc::new(AtomicUsize::new(0)),
+                cx,
+            )
+            .await
+            .expect("Worktree initialization to succeed");
+            let did_finish_scan = worktree
+                .update(cx, |this, _| this.as_local().unwrap().scan_complete())
+                .unwrap();
+            let start = std::time::Instant::now();
+            did_finish_scan.await;
+            let elapsed = start.elapsed();
+            let (files, directories) = worktree
+                .read_with(cx, |this, _| (this.file_count(), this.dir_count()))
+                .unwrap();
+            println!(
+                "{:?} for {directories} directories and {files} files",
+                elapsed
+            );
+            cx.update(|cx| {
+                cx.quit();
+            })
+        })
+        .detach();
+    })
+}