1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use editor::Bias;
  11use fs::{FakeFs, Fs as _};
  12use futures::StreamExt as _;
  13use gpui::{executor::Deterministic, ModelHandle, Task, TestAppContext};
  14use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  15use lsp::FakeLanguageServer;
  16use parking_lot::Mutex;
  17use project::{search::SearchQuery, Project, ProjectPath};
  18use rand::{
  19    distributions::{Alphanumeric, DistString},
  20    prelude::*,
  21};
  22use serde::{Deserialize, Serialize};
  23use std::{
  24    env,
  25    ops::Range,
  26    path::{Path, PathBuf},
  27    rc::Rc,
  28    sync::{
  29        atomic::{AtomicBool, Ordering::SeqCst},
  30        Arc,
  31    },
  32};
  33use util::ResultExt;
  34
  35#[gpui::test(iterations = 100)]
  36async fn test_random_collaboration(
  37    cx: &mut TestAppContext,
  38    deterministic: Arc<Deterministic>,
  39    rng: StdRng,
  40) {
  41    deterministic.forbid_parking();
  42
  43    let max_peers = env::var("MAX_PEERS")
  44        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  45        .unwrap_or(3);
  46    let max_operations = env::var("OPERATIONS")
  47        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  48        .unwrap_or(10);
  49
  50    let plan_load_path = path_env_var("LOAD_PLAN");
  51    let plan_save_path = path_env_var("SAVE_PLAN");
  52
  53    let mut server = TestServer::start(&deterministic).await;
  54    let db = server.app_state.db.clone();
  55
  56    let mut users = Vec::new();
  57    for ix in 0..max_peers {
  58        let username = format!("user-{}", ix + 1);
  59        let user_id = db
  60            .create_user(
  61                &format!("{username}@example.com"),
  62                false,
  63                NewUserParams {
  64                    github_login: username.clone(),
  65                    github_user_id: (ix + 1) as i32,
  66                    invite_count: 0,
  67                },
  68            )
  69            .await
  70            .unwrap()
  71            .user_id;
  72        users.push(UserTestPlan {
  73            user_id,
  74            username,
  75            online: false,
  76            next_root_id: 0,
  77            operation_ix: 0,
  78        });
  79    }
  80
  81    for (ix, user_a) in users.iter().enumerate() {
  82        for user_b in &users[ix + 1..] {
  83            server
  84                .app_state
  85                .db
  86                .send_contact_request(user_a.user_id, user_b.user_id)
  87                .await
  88                .unwrap();
  89            server
  90                .app_state
  91                .db
  92                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  93                .await
  94                .unwrap();
  95        }
  96    }
  97
  98    let plan = Arc::new(Mutex::new(TestPlan::new(rng, users, max_operations)));
  99
 100    if let Some(path) = &plan_load_path {
 101        eprintln!("loaded plan from path {:?}", path);
 102        plan.lock().load(path);
 103    }
 104
 105    let mut clients = Vec::new();
 106    let mut client_tasks = Vec::new();
 107    let mut operation_channels = Vec::new();
 108
 109    loop {
 110        let Some((next_operation, skipped)) = plan.lock().next_server_operation(&clients) else { break };
 111        let applied = apply_server_operation(
 112            deterministic.clone(),
 113            &mut server,
 114            &mut clients,
 115            &mut client_tasks,
 116            &mut operation_channels,
 117            plan.clone(),
 118            next_operation,
 119            cx,
 120        )
 121        .await;
 122        if !applied {
 123            skipped.store(false, SeqCst);
 124        }
 125    }
 126
 127    drop(operation_channels);
 128    deterministic.start_waiting();
 129    futures::future::join_all(client_tasks).await;
 130    deterministic.finish_waiting();
 131    deterministic.run_until_parked();
 132
 133    if let Some(path) = &plan_save_path {
 134        eprintln!("saved test plan to path {:?}", path);
 135        plan.lock().save(path);
 136    }
 137
 138    for (client, client_cx) in &clients {
 139        for guest_project in client.remote_projects().iter() {
 140            guest_project.read_with(client_cx, |guest_project, cx| {
 141                let host_project = clients.iter().find_map(|(client, cx)| {
 142                    let project = client
 143                        .local_projects()
 144                        .iter()
 145                        .find(|host_project| {
 146                            host_project.read_with(cx, |host_project, _| {
 147                                host_project.remote_id() == guest_project.remote_id()
 148                            })
 149                        })?
 150                        .clone();
 151                    Some((project, cx))
 152                });
 153
 154                if !guest_project.is_read_only() {
 155                    if let Some((host_project, host_cx)) = host_project {
 156                        let host_worktree_snapshots =
 157                            host_project.read_with(host_cx, |host_project, cx| {
 158                                host_project
 159                                    .worktrees(cx)
 160                                    .map(|worktree| {
 161                                        let worktree = worktree.read(cx);
 162                                        (worktree.id(), worktree.snapshot())
 163                                    })
 164                                    .collect::<BTreeMap<_, _>>()
 165                            });
 166                        let guest_worktree_snapshots = guest_project
 167                            .worktrees(cx)
 168                            .map(|worktree| {
 169                                let worktree = worktree.read(cx);
 170                                (worktree.id(), worktree.snapshot())
 171                            })
 172                            .collect::<BTreeMap<_, _>>();
 173
 174                        assert_eq!(
 175                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 176                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 177                            "{} has different worktrees than the host",
 178                            client.username
 179                        );
 180
 181                        for (id, host_snapshot) in &host_worktree_snapshots {
 182                            let guest_snapshot = &guest_worktree_snapshots[id];
 183                            assert_eq!(
 184                                guest_snapshot.root_name(),
 185                                host_snapshot.root_name(),
 186                                "{} has different root name than the host for worktree {}",
 187                                client.username,
 188                                id
 189                            );
 190                            assert_eq!(
 191                                guest_snapshot.abs_path(),
 192                                host_snapshot.abs_path(),
 193                                "{} has different abs path than the host for worktree {}",
 194                                client.username,
 195                                id
 196                            );
 197                            assert_eq!(
 198                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 199                                host_snapshot.entries(false).collect::<Vec<_>>(),
 200                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 201                                client.username,
 202                                id,
 203                                host_snapshot.abs_path(),
 204                                host_project.read_with(host_cx, |project, _| project.remote_id())
 205                            );
 206                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 207                        }
 208                    }
 209                }
 210
 211                guest_project.check_invariants(cx);
 212            });
 213        }
 214
 215        let buffers = client.buffers().clone();
 216        for (guest_project, guest_buffers) in &buffers {
 217            let project_id = if guest_project.read_with(client_cx, |project, _| {
 218                project.is_local() || project.is_read_only()
 219            }) {
 220                continue;
 221            } else {
 222                guest_project
 223                    .read_with(client_cx, |project, _| project.remote_id())
 224                    .unwrap()
 225            };
 226            let guest_user_id = client.user_id().unwrap();
 227
 228            let host_project = clients.iter().find_map(|(client, cx)| {
 229                let project = client
 230                    .local_projects()
 231                    .iter()
 232                    .find(|host_project| {
 233                        host_project.read_with(cx, |host_project, _| {
 234                            host_project.remote_id() == Some(project_id)
 235                        })
 236                    })?
 237                    .clone();
 238                Some((client.user_id().unwrap(), project, cx))
 239            });
 240
 241            let (host_user_id, host_project, host_cx) =
 242                if let Some((host_user_id, host_project, host_cx)) = host_project {
 243                    (host_user_id, host_project, host_cx)
 244                } else {
 245                    continue;
 246                };
 247
 248            for guest_buffer in guest_buffers {
 249                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 250                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 251                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 252                        panic!(
 253                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 254                            client.username,
 255                            client.peer_id(),
 256                            buffer_id
 257                        )
 258                    })
 259                });
 260                let path = host_buffer
 261                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 262
 263                assert_eq!(
 264                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 265                    0,
 266                    "{}, buffer {}, path {:?} has deferred operations",
 267                    client.username,
 268                    buffer_id,
 269                    path,
 270                );
 271                assert_eq!(
 272                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 273                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 274                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 275                    client.username,
 276                    buffer_id,
 277                    path
 278                );
 279
 280                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 281                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 282                match (host_file, guest_file) {
 283                    (Some(host_file), Some(guest_file)) => {
 284                        assert_eq!(guest_file.path(), host_file.path());
 285                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 286                        assert_eq!(
 287                            guest_file.mtime(),
 288                            host_file.mtime(),
 289                            "guest {} mtime does not match host {} for path {:?} in project {}",
 290                            guest_user_id,
 291                            host_user_id,
 292                            guest_file.path(),
 293                            project_id,
 294                        );
 295                    }
 296                    (None, None) => {}
 297                    (None, _) => panic!("host's file is None, guest's isn't"),
 298                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 299                }
 300
 301                let host_diff_base =
 302                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 303                let guest_diff_base = guest_buffer
 304                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 305                assert_eq!(guest_diff_base, host_diff_base);
 306            }
 307        }
 308    }
 309
 310    for (client, mut cx) in clients {
 311        cx.update(|cx| {
 312            cx.clear_globals();
 313            drop(client);
 314        });
 315    }
 316}
 317
 318async fn apply_server_operation(
 319    deterministic: Arc<Deterministic>,
 320    server: &mut TestServer,
 321    clients: &mut Vec<(Rc<TestClient>, TestAppContext)>,
 322    client_tasks: &mut Vec<Task<()>>,
 323    operation_channels: &mut Vec<futures::channel::mpsc::UnboundedSender<usize>>,
 324    plan: Arc<Mutex<TestPlan>>,
 325    operation: Operation,
 326    cx: &mut TestAppContext,
 327) -> bool {
 328    match operation {
 329        Operation::AddConnection { user_id } => {
 330            let username;
 331            {
 332                let mut plan = plan.lock();
 333                let mut user = plan.user(user_id);
 334                if user.online {
 335                    return false;
 336                }
 337                user.online = true;
 338                username = user.username.clone();
 339            };
 340            log::info!("Adding new connection for {}", username);
 341            let next_entity_id = (user_id.0 * 10_000) as usize;
 342            let mut client_cx = TestAppContext::new(
 343                cx.foreground_platform(),
 344                cx.platform(),
 345                deterministic.build_foreground(user_id.0 as usize),
 346                deterministic.build_background(),
 347                cx.font_cache(),
 348                cx.leak_detector(),
 349                next_entity_id,
 350                cx.function_name.clone(),
 351            );
 352
 353            let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 354            let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 355            operation_channels.push(operation_tx);
 356            clients.push((client.clone(), client_cx.clone()));
 357            client_tasks.push(client_cx.foreground().spawn(simulate_client(
 358                client,
 359                operation_rx,
 360                plan.clone(),
 361                client_cx,
 362            )));
 363
 364            log::info!("Added connection for {}", username);
 365        }
 366
 367        Operation::RemoveConnection { user_id } => {
 368            log::info!("Simulating full disconnection of user {}", user_id);
 369            let client_ix = clients
 370                .iter()
 371                .position(|(client, cx)| client.current_user_id(cx) == user_id);
 372            let Some(client_ix) = client_ix else { return false };
 373            let user_connection_ids = server
 374                .connection_pool
 375                .lock()
 376                .user_connection_ids(user_id)
 377                .collect::<Vec<_>>();
 378            assert_eq!(user_connection_ids.len(), 1);
 379            let removed_peer_id = user_connection_ids[0].into();
 380            let (client, mut client_cx) = clients.remove(client_ix);
 381            let client_task = client_tasks.remove(client_ix);
 382            operation_channels.remove(client_ix);
 383            server.forbid_connections();
 384            server.disconnect_client(removed_peer_id);
 385            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 386            deterministic.start_waiting();
 387            log::info!("Waiting for user {} to exit...", user_id);
 388            client_task.await;
 389            deterministic.finish_waiting();
 390            server.allow_connections();
 391
 392            for project in client.remote_projects().iter() {
 393                project.read_with(&client_cx, |project, _| {
 394                    assert!(
 395                        project.is_read_only(),
 396                        "project {:?} should be read only",
 397                        project.remote_id()
 398                    )
 399                });
 400            }
 401
 402            for (client, cx) in clients {
 403                let contacts = server
 404                    .app_state
 405                    .db
 406                    .get_contacts(client.current_user_id(cx))
 407                    .await
 408                    .unwrap();
 409                let pool = server.connection_pool.lock();
 410                for contact in contacts {
 411                    if let db::Contact::Accepted { user_id: id, .. } = contact {
 412                        if pool.is_user_online(id) {
 413                            assert_ne!(
 414                                id, user_id,
 415                                "removed client is still a contact of another peer"
 416                            );
 417                        }
 418                    }
 419                }
 420            }
 421
 422            log::info!("{} removed", client.username);
 423            plan.lock().user(user_id).online = false;
 424            client_cx.update(|cx| {
 425                cx.clear_globals();
 426                drop(client);
 427            });
 428        }
 429
 430        Operation::BounceConnection { user_id } => {
 431            log::info!("Simulating temporary disconnection of user {}", user_id);
 432            let user_connection_ids = server
 433                .connection_pool
 434                .lock()
 435                .user_connection_ids(user_id)
 436                .collect::<Vec<_>>();
 437            if user_connection_ids.is_empty() {
 438                return false;
 439            }
 440            assert_eq!(user_connection_ids.len(), 1);
 441            let peer_id = user_connection_ids[0].into();
 442            server.disconnect_client(peer_id);
 443            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 444        }
 445
 446        Operation::RestartServer => {
 447            log::info!("Simulating server restart");
 448            server.reset().await;
 449            deterministic.advance_clock(RECEIVE_TIMEOUT);
 450            server.start().await.unwrap();
 451            deterministic.advance_clock(CLEANUP_TIMEOUT);
 452            let environment = &server.app_state.config.zed_environment;
 453            let stale_room_ids = server
 454                .app_state
 455                .db
 456                .stale_room_ids(environment, server.id())
 457                .await
 458                .unwrap();
 459            assert_eq!(stale_room_ids, vec![]);
 460        }
 461
 462        Operation::MutateClients {
 463            user_ids,
 464            batch_id,
 465            quiesce,
 466        } => {
 467            let mut applied = false;
 468            for user_id in user_ids {
 469                let client_ix = clients
 470                    .iter()
 471                    .position(|(client, cx)| client.current_user_id(cx) == user_id);
 472                let Some(client_ix) = client_ix else { continue };
 473                applied = true;
 474                if let Err(err) = operation_channels[client_ix].unbounded_send(batch_id) {
 475                    // panic!("error signaling user {}, client {}", user_id, client_ix);
 476                }
 477            }
 478
 479            if quiesce && applied {
 480                deterministic.run_until_parked();
 481            }
 482
 483            return applied;
 484        }
 485    }
 486    true
 487}
 488
 489async fn apply_client_operation(
 490    client: &TestClient,
 491    operation: ClientOperation,
 492    cx: &mut TestAppContext,
 493) -> Result<(), TestError> {
 494    match operation {
 495        ClientOperation::AcceptIncomingCall => {
 496            let active_call = cx.read(ActiveCall::global);
 497            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 498                Err(TestError::Inapplicable)?;
 499            }
 500
 501            log::info!("{}: accepting incoming call", client.username);
 502            active_call
 503                .update(cx, |call, cx| call.accept_incoming(cx))
 504                .await?;
 505        }
 506
 507        ClientOperation::RejectIncomingCall => {
 508            let active_call = cx.read(ActiveCall::global);
 509            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 510                Err(TestError::Inapplicable)?;
 511            }
 512
 513            log::info!("{}: declining incoming call", client.username);
 514            active_call.update(cx, |call, _| call.decline_incoming())?;
 515        }
 516
 517        ClientOperation::LeaveCall => {
 518            let active_call = cx.read(ActiveCall::global);
 519            if active_call.read_with(cx, |call, _| call.room().is_none()) {
 520                Err(TestError::Inapplicable)?;
 521            }
 522
 523            log::info!("{}: hanging up", client.username);
 524            active_call.update(cx, |call, cx| call.hang_up(cx))?;
 525        }
 526
 527        ClientOperation::InviteContactToCall { user_id } => {
 528            let active_call = cx.read(ActiveCall::global);
 529
 530            log::info!("{}: inviting {}", client.username, user_id,);
 531            active_call
 532                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 533                .await
 534                .log_err();
 535        }
 536
 537        ClientOperation::OpenLocalProject { first_root_name } => {
 538            log::info!(
 539                "{}: opening local project at {:?}",
 540                client.username,
 541                first_root_name
 542            );
 543
 544            let root_path = Path::new("/").join(&first_root_name);
 545            client.fs.create_dir(&root_path).await.unwrap();
 546            client
 547                .fs
 548                .create_file(&root_path.join("main.rs"), Default::default())
 549                .await
 550                .unwrap();
 551            let project = client.build_local_project(root_path, cx).await.0;
 552            ensure_project_shared(&project, client, cx).await;
 553            client.local_projects_mut().push(project.clone());
 554        }
 555
 556        ClientOperation::AddWorktreeToProject {
 557            project_root_name,
 558            new_root_path,
 559        } => {
 560            let project = project_for_root_name(client, &project_root_name, cx)
 561                .ok_or(TestError::Inapplicable)?;
 562
 563            log::info!(
 564                "{}: finding/creating local worktree at {:?} to project with root path {}",
 565                client.username,
 566                new_root_path,
 567                project_root_name
 568            );
 569
 570            ensure_project_shared(&project, client, cx).await;
 571            if !client.fs.paths().await.contains(&new_root_path) {
 572                client.fs.create_dir(&new_root_path).await.unwrap();
 573            }
 574            project
 575                .update(cx, |project, cx| {
 576                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 577                })
 578                .await
 579                .unwrap();
 580        }
 581
 582        ClientOperation::CloseRemoteProject { project_root_name } => {
 583            let project = project_for_root_name(client, &project_root_name, cx)
 584                .ok_or(TestError::Inapplicable)?;
 585
 586            log::info!(
 587                "{}: closing remote project with root path {}",
 588                client.username,
 589                project_root_name,
 590            );
 591
 592            let ix = client
 593                .remote_projects()
 594                .iter()
 595                .position(|p| p == &project)
 596                .unwrap();
 597            cx.update(|_| {
 598                client.remote_projects_mut().remove(ix);
 599                client.buffers().retain(|project, _| project != project);
 600                drop(project);
 601            });
 602        }
 603
 604        ClientOperation::OpenRemoteProject {
 605            host_id,
 606            first_root_name,
 607        } => {
 608            let active_call = cx.read(ActiveCall::global);
 609            let project = active_call
 610                .update(cx, |call, cx| {
 611                    let room = call.room().cloned()?;
 612                    let participant = room
 613                        .read(cx)
 614                        .remote_participants()
 615                        .get(&host_id.to_proto())?;
 616                    let project_id = participant
 617                        .projects
 618                        .iter()
 619                        .find(|project| project.worktree_root_names[0] == first_root_name)?
 620                        .id;
 621                    Some(room.update(cx, |room, cx| {
 622                        room.join_project(
 623                            project_id,
 624                            client.language_registry.clone(),
 625                            FakeFs::new(cx.background().clone()),
 626                            cx,
 627                        )
 628                    }))
 629                })
 630                .ok_or(TestError::Inapplicable)?;
 631
 632            log::info!(
 633                "{}: joining remote project of user {}, root name {}",
 634                client.username,
 635                host_id,
 636                first_root_name,
 637            );
 638
 639            let project = project.await?;
 640            client.remote_projects_mut().push(project.clone());
 641        }
 642
 643        ClientOperation::CreateWorktreeEntry {
 644            project_root_name,
 645            is_local,
 646            full_path,
 647            is_dir,
 648        } => {
 649            let project = project_for_root_name(client, &project_root_name, cx)
 650                .ok_or(TestError::Inapplicable)?;
 651            let project_path = project_path_for_full_path(&project, &full_path, cx)
 652                .ok_or(TestError::Inapplicable)?;
 653
 654            log::info!(
 655                "{}: creating {} at path {:?} in {} project {}",
 656                client.username,
 657                if is_dir { "dir" } else { "file" },
 658                full_path,
 659                if is_local { "local" } else { "remote" },
 660                project_root_name,
 661            );
 662
 663            ensure_project_shared(&project, client, cx).await;
 664            project
 665                .update(cx, |p, cx| p.create_entry(project_path, is_dir, cx))
 666                .unwrap()
 667                .await?;
 668        }
 669
 670        ClientOperation::OpenBuffer {
 671            project_root_name,
 672            is_local,
 673            full_path,
 674        } => {
 675            let project = project_for_root_name(client, &project_root_name, cx)
 676                .ok_or(TestError::Inapplicable)?;
 677            let project_path = project_path_for_full_path(&project, &full_path, cx)
 678                .ok_or(TestError::Inapplicable)?;
 679
 680            log::info!(
 681                "{}: opening buffer {:?} in {} project {}",
 682                client.username,
 683                full_path,
 684                if is_local { "local" } else { "remote" },
 685                project_root_name,
 686            );
 687
 688            ensure_project_shared(&project, client, cx).await;
 689            let buffer = project
 690                .update(cx, |project, cx| project.open_buffer(project_path, cx))
 691                .await?;
 692            client.buffers_for_project(&project).insert(buffer);
 693        }
 694
 695        ClientOperation::EditBuffer {
 696            project_root_name,
 697            is_local,
 698            full_path,
 699            edits,
 700        } => {
 701            let project = project_for_root_name(client, &project_root_name, cx)
 702                .ok_or(TestError::Inapplicable)?;
 703            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 704                .ok_or(TestError::Inapplicable)?;
 705
 706            log::info!(
 707                "{}: editing buffer {:?} in {} project {} with {:?}",
 708                client.username,
 709                full_path,
 710                if is_local { "local" } else { "remote" },
 711                project_root_name,
 712                edits
 713            );
 714
 715            ensure_project_shared(&project, client, cx).await;
 716            buffer.update(cx, |buffer, cx| {
 717                let snapshot = buffer.snapshot();
 718                buffer.edit(
 719                    edits.into_iter().map(|(range, text)| {
 720                        let start = snapshot.clip_offset(range.start, Bias::Left);
 721                        let end = snapshot.clip_offset(range.end, Bias::Right);
 722                        (start..end, text)
 723                    }),
 724                    None,
 725                    cx,
 726                );
 727            });
 728        }
 729
 730        ClientOperation::CloseBuffer {
 731            project_root_name,
 732            is_local,
 733            full_path,
 734        } => {
 735            let project = project_for_root_name(client, &project_root_name, cx)
 736                .ok_or(TestError::Inapplicable)?;
 737            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 738                .ok_or(TestError::Inapplicable)?;
 739
 740            log::info!(
 741                "{}: closing buffer {:?} in {} project {}",
 742                client.username,
 743                full_path,
 744                if is_local { "local" } else { "remote" },
 745                project_root_name
 746            );
 747
 748            ensure_project_shared(&project, client, cx).await;
 749            cx.update(|_| {
 750                client.buffers_for_project(&project).remove(&buffer);
 751                drop(buffer);
 752            });
 753        }
 754
 755        ClientOperation::SaveBuffer {
 756            project_root_name,
 757            is_local,
 758            full_path,
 759            detach,
 760        } => {
 761            let project = project_for_root_name(client, &project_root_name, cx)
 762                .ok_or(TestError::Inapplicable)?;
 763            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 764                .ok_or(TestError::Inapplicable)?;
 765
 766            log::info!(
 767                "{}: saving buffer {:?} in {} project {}{}",
 768                client.username,
 769                full_path,
 770                if is_local { "local" } else { "remote" },
 771                project_root_name,
 772                if detach { ", detaching" } else { ", awaiting" }
 773            );
 774
 775            ensure_project_shared(&project, client, cx).await;
 776            let (requested_version, save) =
 777                buffer.update(cx, |buffer, cx| (buffer.version(), buffer.save(cx)));
 778            let save = cx.background().spawn(async move {
 779                let (saved_version, _, _) = save
 780                    .await
 781                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 782                assert!(saved_version.observed_all(&requested_version));
 783                anyhow::Ok(())
 784            });
 785            if detach {
 786                cx.update(|cx| save.detach_and_log_err(cx));
 787            } else {
 788                save.await?;
 789            }
 790        }
 791
 792        ClientOperation::RequestLspDataInBuffer {
 793            project_root_name,
 794            is_local,
 795            full_path,
 796            offset,
 797            kind,
 798            detach,
 799        } => {
 800            let project = project_for_root_name(client, &project_root_name, cx)
 801                .ok_or(TestError::Inapplicable)?;
 802            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 803                .ok_or(TestError::Inapplicable)?;
 804
 805            log::info!(
 806                "{}: request LSP {:?} for buffer {:?} in {} project {}{}",
 807                client.username,
 808                kind,
 809                full_path,
 810                if is_local { "local" } else { "remote" },
 811                project_root_name,
 812                if detach { ", detaching" } else { ", awaiting" }
 813            );
 814
 815            let offset = buffer.read_with(cx, |b, _| b.clip_offset(offset, Bias::Left));
 816            let request = match kind {
 817                LspRequestKind::Rename => cx.spawn(|mut cx| async move {
 818                    project
 819                        .update(&mut cx, |p, cx| p.prepare_rename(buffer, offset, cx))
 820                        .await?;
 821                    anyhow::Ok(())
 822                }),
 823                LspRequestKind::Completion => cx.spawn(|mut cx| async move {
 824                    project
 825                        .update(&mut cx, |p, cx| p.completions(&buffer, offset, cx))
 826                        .await?;
 827                    Ok(())
 828                }),
 829                LspRequestKind::CodeAction => cx.spawn(|mut cx| async move {
 830                    project
 831                        .update(&mut cx, |p, cx| p.code_actions(&buffer, offset..offset, cx))
 832                        .await?;
 833                    Ok(())
 834                }),
 835                LspRequestKind::Definition => cx.spawn(|mut cx| async move {
 836                    project
 837                        .update(&mut cx, |p, cx| p.definition(&buffer, offset, cx))
 838                        .await?;
 839                    Ok(())
 840                }),
 841                LspRequestKind::Highlights => cx.spawn(|mut cx| async move {
 842                    project
 843                        .update(&mut cx, |p, cx| p.document_highlights(&buffer, offset, cx))
 844                        .await?;
 845                    Ok(())
 846                }),
 847            };
 848            if detach {
 849                request.detach();
 850            } else {
 851                request.await?;
 852            }
 853        }
 854
 855        ClientOperation::SearchProject {
 856            project_root_name,
 857            is_local,
 858            query,
 859            detach,
 860        } => {
 861            let project = project_for_root_name(client, &project_root_name, cx)
 862                .ok_or(TestError::Inapplicable)?;
 863
 864            log::info!(
 865                "{}: search {} project {} for {:?}{}",
 866                client.username,
 867                if is_local { "local" } else { "remote" },
 868                project_root_name,
 869                query,
 870                if detach { ", detaching" } else { ", awaiting" }
 871            );
 872
 873            let search = project.update(cx, |project, cx| {
 874                project.search(SearchQuery::text(query, false, false), cx)
 875            });
 876            let search = cx.background().spawn(async move {
 877                search
 878                    .await
 879                    .map_err(|err| anyhow!("search request failed: {:?}", err))
 880            });
 881            if detach {
 882                cx.update(|cx| search.detach_and_log_err(cx));
 883            } else {
 884                search.await?;
 885            }
 886        }
 887
 888        ClientOperation::CreateFsEntry { path, is_dir } => {
 889            client
 890                .fs
 891                .metadata(&path.parent().unwrap())
 892                .await?
 893                .ok_or(TestError::Inapplicable)?;
 894
 895            log::info!(
 896                "{}: creating {} at {:?}",
 897                client.username,
 898                if is_dir { "dir" } else { "file" },
 899                path
 900            );
 901
 902            if is_dir {
 903                client.fs.create_dir(&path).await.unwrap();
 904            } else {
 905                client
 906                    .fs
 907                    .create_file(&path, Default::default())
 908                    .await
 909                    .unwrap();
 910            }
 911        }
 912
 913        ClientOperation::WriteGitIndex {
 914            repo_path,
 915            contents,
 916        } => {
 917            if !client
 918                .fs
 919                .metadata(&repo_path)
 920                .await?
 921                .map_or(false, |m| m.is_dir)
 922            {
 923                Err(TestError::Inapplicable)?;
 924            }
 925
 926            log::info!(
 927                "{}: writing git index for repo {:?}: {:?}",
 928                client.username,
 929                repo_path,
 930                contents
 931            );
 932
 933            let dot_git_dir = repo_path.join(".git");
 934            let contents = contents
 935                .iter()
 936                .map(|(path, contents)| (path.as_path(), contents.clone()))
 937                .collect::<Vec<_>>();
 938            if client.fs.metadata(&dot_git_dir).await?.is_none() {
 939                client.fs.create_dir(&dot_git_dir).await?;
 940            }
 941            client.fs.set_index_for_repo(&dot_git_dir, &contents).await;
 942        }
 943    }
 944    Ok(())
 945}
 946
 947struct TestPlan {
 948    rng: StdRng,
 949    replay: bool,
 950    stored_operations: Vec<(StoredOperation, Arc<AtomicBool>)>,
 951    max_operations: usize,
 952    operation_ix: usize,
 953    users: Vec<UserTestPlan>,
 954    next_batch_id: usize,
 955    allow_server_restarts: bool,
 956    allow_client_reconnection: bool,
 957    allow_client_disconnection: bool,
 958}
 959
 960struct UserTestPlan {
 961    user_id: UserId,
 962    username: String,
 963    next_root_id: usize,
 964    operation_ix: usize,
 965    online: bool,
 966}
 967
 968#[derive(Clone, Debug, Serialize, Deserialize)]
 969#[serde(untagged)]
 970enum StoredOperation {
 971    Server(Operation),
 972    Client {
 973        user_id: UserId,
 974        batch_id: usize,
 975        operation: ClientOperation,
 976    },
 977}
 978
 979#[derive(Clone, Debug, Serialize, Deserialize)]
 980enum Operation {
 981    AddConnection {
 982        user_id: UserId,
 983    },
 984    RemoveConnection {
 985        user_id: UserId,
 986    },
 987    BounceConnection {
 988        user_id: UserId,
 989    },
 990    RestartServer,
 991    MutateClients {
 992        batch_id: usize,
 993        #[serde(skip_serializing)]
 994        #[serde(skip_deserializing)]
 995        user_ids: Vec<UserId>,
 996        quiesce: bool,
 997    },
 998}
 999
1000#[derive(Clone, Debug, Serialize, Deserialize)]
1001enum ClientOperation {
1002    AcceptIncomingCall,
1003    RejectIncomingCall,
1004    LeaveCall,
1005    InviteContactToCall {
1006        user_id: UserId,
1007    },
1008    OpenLocalProject {
1009        first_root_name: String,
1010    },
1011    OpenRemoteProject {
1012        host_id: UserId,
1013        first_root_name: String,
1014    },
1015    AddWorktreeToProject {
1016        project_root_name: String,
1017        new_root_path: PathBuf,
1018    },
1019    CloseRemoteProject {
1020        project_root_name: String,
1021    },
1022    OpenBuffer {
1023        project_root_name: String,
1024        is_local: bool,
1025        full_path: PathBuf,
1026    },
1027    SearchProject {
1028        project_root_name: String,
1029        is_local: bool,
1030        query: String,
1031        detach: bool,
1032    },
1033    EditBuffer {
1034        project_root_name: String,
1035        is_local: bool,
1036        full_path: PathBuf,
1037        edits: Vec<(Range<usize>, Arc<str>)>,
1038    },
1039    CloseBuffer {
1040        project_root_name: String,
1041        is_local: bool,
1042        full_path: PathBuf,
1043    },
1044    SaveBuffer {
1045        project_root_name: String,
1046        is_local: bool,
1047        full_path: PathBuf,
1048        detach: bool,
1049    },
1050    RequestLspDataInBuffer {
1051        project_root_name: String,
1052        is_local: bool,
1053        full_path: PathBuf,
1054        offset: usize,
1055        kind: LspRequestKind,
1056        detach: bool,
1057    },
1058    CreateWorktreeEntry {
1059        project_root_name: String,
1060        is_local: bool,
1061        full_path: PathBuf,
1062        is_dir: bool,
1063    },
1064    CreateFsEntry {
1065        path: PathBuf,
1066        is_dir: bool,
1067    },
1068    WriteGitIndex {
1069        repo_path: PathBuf,
1070        contents: Vec<(PathBuf, String)>,
1071    },
1072}
1073
1074#[derive(Clone, Debug, Serialize, Deserialize)]
1075enum LspRequestKind {
1076    Rename,
1077    Completion,
1078    CodeAction,
1079    Definition,
1080    Highlights,
1081}
1082
1083enum TestError {
1084    Inapplicable,
1085    Other(anyhow::Error),
1086}
1087
1088impl From<anyhow::Error> for TestError {
1089    fn from(value: anyhow::Error) -> Self {
1090        Self::Other(value)
1091    }
1092}
1093
1094impl TestPlan {
1095    fn new(mut rng: StdRng, users: Vec<UserTestPlan>, max_operations: usize) -> Self {
1096        Self {
1097            replay: false,
1098            allow_server_restarts: rng.gen_bool(0.7),
1099            allow_client_reconnection: rng.gen_bool(0.7),
1100            allow_client_disconnection: rng.gen_bool(0.1),
1101            stored_operations: Vec::new(),
1102            operation_ix: 0,
1103            next_batch_id: 0,
1104            max_operations,
1105            users,
1106            rng,
1107        }
1108    }
1109
1110    fn load(&mut self, path: &Path) {
1111        let json = std::fs::read_to_string(path).unwrap();
1112        self.replay = true;
1113        let stored_operations: Vec<StoredOperation> = serde_json::from_str(&json).unwrap();
1114        self.stored_operations = stored_operations
1115            .iter()
1116            .cloned()
1117            .enumerate()
1118            .map(|(i, mut operation)| {
1119                if let StoredOperation::Server(Operation::MutateClients {
1120                    batch_id: current_batch_id,
1121                    user_ids,
1122                    ..
1123                }) = &mut operation
1124                {
1125                    assert!(user_ids.is_empty());
1126                    user_ids.extend(stored_operations[i + 1..].iter().filter_map(|operation| {
1127                        if let StoredOperation::Client {
1128                            user_id, batch_id, ..
1129                        } = operation
1130                        {
1131                            if batch_id == current_batch_id {
1132                                return Some(user_id);
1133                            }
1134                        }
1135                        None
1136                    }));
1137                    user_ids.sort_unstable();
1138                }
1139                (operation, Arc::new(AtomicBool::new(false)))
1140            })
1141            .collect()
1142    }
1143
1144    fn save(&mut self, path: &Path) {
1145        // Format each operation as one line
1146        let mut json = Vec::new();
1147        json.push(b'[');
1148        for (operation, skipped) in &self.stored_operations {
1149            if skipped.load(SeqCst) {
1150                continue;
1151            }
1152            if json.len() > 1 {
1153                json.push(b',');
1154            }
1155            json.extend_from_slice(b"\n  ");
1156            serde_json::to_writer(&mut json, operation).unwrap();
1157        }
1158        json.extend_from_slice(b"\n]\n");
1159        std::fs::write(path, &json).unwrap();
1160    }
1161
1162    fn next_server_operation(
1163        &mut self,
1164        clients: &[(Rc<TestClient>, TestAppContext)],
1165    ) -> Option<(Operation, Arc<AtomicBool>)> {
1166        if self.replay {
1167            while let Some(stored_operation) = self.stored_operations.get(self.operation_ix) {
1168                self.operation_ix += 1;
1169                if let (StoredOperation::Server(operation), skipped) = stored_operation {
1170                    return Some((operation.clone(), skipped.clone()));
1171                }
1172            }
1173            None
1174        } else {
1175            let operation = self.generate_server_operation(clients)?;
1176            let skipped = Arc::new(AtomicBool::new(false));
1177            self.stored_operations
1178                .push((StoredOperation::Server(operation.clone()), skipped.clone()));
1179            Some((operation, skipped))
1180        }
1181    }
1182
1183    fn next_client_operation(
1184        &mut self,
1185        client: &TestClient,
1186        current_batch_id: usize,
1187        cx: &TestAppContext,
1188    ) -> Option<(ClientOperation, Arc<AtomicBool>)> {
1189        let current_user_id = client.current_user_id(cx);
1190        let user_ix = self
1191            .users
1192            .iter()
1193            .position(|user| user.user_id == current_user_id)
1194            .unwrap();
1195        let user_plan = &mut self.users[user_ix];
1196
1197        if self.replay {
1198            while let Some(stored_operation) = self.stored_operations.get(user_plan.operation_ix) {
1199                user_plan.operation_ix += 1;
1200                if let (
1201                    StoredOperation::Client {
1202                        user_id, operation, ..
1203                    },
1204                    skipped,
1205                ) = stored_operation
1206                {
1207                    if user_id == &current_user_id {
1208                        return Some((operation.clone(), skipped.clone()));
1209                    }
1210                }
1211            }
1212            None
1213        } else {
1214            let operation = self.generate_client_operation(current_user_id, client, cx)?;
1215            let skipped = Arc::new(AtomicBool::new(false));
1216            self.stored_operations.push((
1217                StoredOperation::Client {
1218                    user_id: current_user_id,
1219                    batch_id: current_batch_id,
1220                    operation: operation.clone(),
1221                },
1222                skipped.clone(),
1223            ));
1224            Some((operation, skipped))
1225        }
1226    }
1227
1228    fn generate_server_operation(
1229        &mut self,
1230        clients: &[(Rc<TestClient>, TestAppContext)],
1231    ) -> Option<Operation> {
1232        if self.operation_ix == self.max_operations {
1233            return None;
1234        }
1235
1236        Some(loop {
1237            break match self.rng.gen_range(0..100) {
1238                0..=29 if clients.len() < self.users.len() => {
1239                    let user = self
1240                        .users
1241                        .iter()
1242                        .filter(|u| !u.online)
1243                        .choose(&mut self.rng)
1244                        .unwrap();
1245                    self.operation_ix += 1;
1246                    Operation::AddConnection {
1247                        user_id: user.user_id,
1248                    }
1249                }
1250                30..=34 if clients.len() > 1 && self.allow_client_disconnection => {
1251                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1252                    let user_id = client.current_user_id(cx);
1253                    self.operation_ix += 1;
1254                    Operation::RemoveConnection { user_id }
1255                }
1256                35..=39 if clients.len() > 1 && self.allow_client_reconnection => {
1257                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1258                    let user_id = client.current_user_id(cx);
1259                    self.operation_ix += 1;
1260                    Operation::BounceConnection { user_id }
1261                }
1262                40..=44 if self.allow_server_restarts && clients.len() > 1 => {
1263                    self.operation_ix += 1;
1264                    Operation::RestartServer
1265                }
1266                _ if !clients.is_empty() => {
1267                    let count = self
1268                        .rng
1269                        .gen_range(1..10)
1270                        .min(self.max_operations - self.operation_ix);
1271                    let batch_id = util::post_inc(&mut self.next_batch_id);
1272                    let mut user_ids = (0..count)
1273                        .map(|_| {
1274                            let ix = self.rng.gen_range(0..clients.len());
1275                            let (client, cx) = &clients[ix];
1276                            client.current_user_id(cx)
1277                        })
1278                        .collect::<Vec<_>>();
1279                    user_ids.sort_unstable();
1280                    Operation::MutateClients {
1281                        user_ids,
1282                        batch_id,
1283                        quiesce: self.rng.gen_bool(0.7),
1284                    }
1285                }
1286                _ => continue,
1287            };
1288        })
1289    }
1290
1291    fn generate_client_operation(
1292        &mut self,
1293        user_id: UserId,
1294        client: &TestClient,
1295        cx: &TestAppContext,
1296    ) -> Option<ClientOperation> {
1297        if self.operation_ix == self.max_operations {
1298            return None;
1299        }
1300
1301        let executor = cx.background();
1302        self.operation_ix += 1;
1303        let call = cx.read(ActiveCall::global);
1304        Some(loop {
1305            match self.rng.gen_range(0..100_u32) {
1306                // Mutate the call
1307                0..=29 => {
1308                    // Respond to an incoming call
1309                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
1310                        break if self.rng.gen_bool(0.7) {
1311                            ClientOperation::AcceptIncomingCall
1312                        } else {
1313                            ClientOperation::RejectIncomingCall
1314                        };
1315                    }
1316
1317                    match self.rng.gen_range(0..100_u32) {
1318                        // Invite a contact to the current call
1319                        0..=70 => {
1320                            let available_contacts =
1321                                client.user_store.read_with(cx, |user_store, _| {
1322                                    user_store
1323                                        .contacts()
1324                                        .iter()
1325                                        .filter(|contact| contact.online && !contact.busy)
1326                                        .cloned()
1327                                        .collect::<Vec<_>>()
1328                                });
1329                            if !available_contacts.is_empty() {
1330                                let contact = available_contacts.choose(&mut self.rng).unwrap();
1331                                break ClientOperation::InviteContactToCall {
1332                                    user_id: UserId(contact.user.id as i32),
1333                                };
1334                            }
1335                        }
1336
1337                        // Leave the current call
1338                        71.. => {
1339                            if self.allow_client_disconnection
1340                                && call.read_with(cx, |call, _| call.room().is_some())
1341                            {
1342                                break ClientOperation::LeaveCall;
1343                            }
1344                        }
1345                    }
1346                }
1347
1348                // Mutate projects
1349                30..=59 => match self.rng.gen_range(0..100_u32) {
1350                    // Open a new project
1351                    0..=70 => {
1352                        // Open a remote project
1353                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
1354                            let existing_remote_project_ids = cx.read(|cx| {
1355                                client
1356                                    .remote_projects()
1357                                    .iter()
1358                                    .map(|p| p.read(cx).remote_id().unwrap())
1359                                    .collect::<Vec<_>>()
1360                            });
1361                            let new_remote_projects = room.read_with(cx, |room, _| {
1362                                room.remote_participants()
1363                                    .values()
1364                                    .flat_map(|participant| {
1365                                        participant.projects.iter().filter_map(|project| {
1366                                            if existing_remote_project_ids.contains(&project.id) {
1367                                                None
1368                                            } else {
1369                                                Some((
1370                                                    UserId::from_proto(participant.user.id),
1371                                                    project.worktree_root_names[0].clone(),
1372                                                ))
1373                                            }
1374                                        })
1375                                    })
1376                                    .collect::<Vec<_>>()
1377                            });
1378                            if !new_remote_projects.is_empty() {
1379                                let (host_id, first_root_name) =
1380                                    new_remote_projects.choose(&mut self.rng).unwrap().clone();
1381                                break ClientOperation::OpenRemoteProject {
1382                                    host_id,
1383                                    first_root_name,
1384                                };
1385                            }
1386                        }
1387                        // Open a local project
1388                        else {
1389                            let first_root_name = self.next_root_dir_name(user_id);
1390                            break ClientOperation::OpenLocalProject { first_root_name };
1391                        }
1392                    }
1393
1394                    // Close a remote project
1395                    71..=80 => {
1396                        if !client.remote_projects().is_empty() {
1397                            let project = client
1398                                .remote_projects()
1399                                .choose(&mut self.rng)
1400                                .unwrap()
1401                                .clone();
1402                            let first_root_name = root_name_for_project(&project, cx);
1403                            break ClientOperation::CloseRemoteProject {
1404                                project_root_name: first_root_name,
1405                            };
1406                        }
1407                    }
1408
1409                    // Mutate project worktrees
1410                    81.. => match self.rng.gen_range(0..100_u32) {
1411                        // Add a worktree to a local project
1412                        0..=50 => {
1413                            let Some(project) = client
1414                                .local_projects()
1415                                .choose(&mut self.rng)
1416                                .cloned() else { continue };
1417                            let project_root_name = root_name_for_project(&project, cx);
1418                            let mut paths = executor.block(client.fs.paths());
1419                            paths.remove(0);
1420                            let new_root_path = if paths.is_empty() || self.rng.gen() {
1421                                Path::new("/").join(&self.next_root_dir_name(user_id))
1422                            } else {
1423                                paths.choose(&mut self.rng).unwrap().clone()
1424                            };
1425                            break ClientOperation::AddWorktreeToProject {
1426                                project_root_name,
1427                                new_root_path,
1428                            };
1429                        }
1430
1431                        // Add an entry to a worktree
1432                        _ => {
1433                            let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1434                            let project_root_name = root_name_for_project(&project, cx);
1435                            let is_local = project.read_with(cx, |project, _| project.is_local());
1436                            let worktree = project.read_with(cx, |project, cx| {
1437                                project
1438                                    .worktrees(cx)
1439                                    .filter(|worktree| {
1440                                        let worktree = worktree.read(cx);
1441                                        worktree.is_visible()
1442                                            && worktree.entries(false).any(|e| e.is_file())
1443                                            && worktree.root_entry().map_or(false, |e| e.is_dir())
1444                                    })
1445                                    .choose(&mut self.rng)
1446                            });
1447                            let Some(worktree) = worktree else { continue };
1448                            let is_dir = self.rng.gen::<bool>();
1449                            let mut full_path =
1450                                worktree.read_with(cx, |w, _| PathBuf::from(w.root_name()));
1451                            full_path.push(gen_file_name(&mut self.rng));
1452                            if !is_dir {
1453                                full_path.set_extension("rs");
1454                            }
1455                            break ClientOperation::CreateWorktreeEntry {
1456                                project_root_name,
1457                                is_local,
1458                                full_path,
1459                                is_dir,
1460                            };
1461                        }
1462                    },
1463                },
1464
1465                // Query and mutate buffers
1466                60..=90 => {
1467                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1468                    let project_root_name = root_name_for_project(&project, cx);
1469                    let is_local = project.read_with(cx, |project, _| project.is_local());
1470
1471                    match self.rng.gen_range(0..100_u32) {
1472                        // Manipulate an existing buffer
1473                        0..=70 => {
1474                            let Some(buffer) = client
1475                                .buffers_for_project(&project)
1476                                .iter()
1477                                .choose(&mut self.rng)
1478                                .cloned() else { continue };
1479
1480                            let full_path = buffer
1481                                .read_with(cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
1482
1483                            match self.rng.gen_range(0..100_u32) {
1484                                // Close the buffer
1485                                0..=15 => {
1486                                    break ClientOperation::CloseBuffer {
1487                                        project_root_name,
1488                                        is_local,
1489                                        full_path,
1490                                    };
1491                                }
1492                                // Save the buffer
1493                                16..=29 if buffer.read_with(cx, |b, _| b.is_dirty()) => {
1494                                    let detach = self.rng.gen_bool(0.3);
1495                                    break ClientOperation::SaveBuffer {
1496                                        project_root_name,
1497                                        is_local,
1498                                        full_path,
1499                                        detach,
1500                                    };
1501                                }
1502                                // Edit the buffer
1503                                30..=69 => {
1504                                    let edits = buffer.read_with(cx, |buffer, _| {
1505                                        buffer.get_random_edits(&mut self.rng, 3)
1506                                    });
1507                                    break ClientOperation::EditBuffer {
1508                                        project_root_name,
1509                                        is_local,
1510                                        full_path,
1511                                        edits,
1512                                    };
1513                                }
1514                                // Make an LSP request
1515                                _ => {
1516                                    let offset = buffer.read_with(cx, |buffer, _| {
1517                                        buffer.clip_offset(
1518                                            self.rng.gen_range(0..=buffer.len()),
1519                                            language::Bias::Left,
1520                                        )
1521                                    });
1522                                    let detach = self.rng.gen();
1523                                    break ClientOperation::RequestLspDataInBuffer {
1524                                        project_root_name,
1525                                        full_path,
1526                                        offset,
1527                                        is_local,
1528                                        kind: match self.rng.gen_range(0..5_u32) {
1529                                            0 => LspRequestKind::Rename,
1530                                            1 => LspRequestKind::Highlights,
1531                                            2 => LspRequestKind::Definition,
1532                                            3 => LspRequestKind::CodeAction,
1533                                            4.. => LspRequestKind::Completion,
1534                                        },
1535                                        detach,
1536                                    };
1537                                }
1538                            }
1539                        }
1540
1541                        71..=80 => {
1542                            let query = self.rng.gen_range('a'..='z').to_string();
1543                            let detach = self.rng.gen_bool(0.3);
1544                            break ClientOperation::SearchProject {
1545                                project_root_name,
1546                                is_local,
1547                                query,
1548                                detach,
1549                            };
1550                        }
1551
1552                        // Open a buffer
1553                        81.. => {
1554                            let worktree = project.read_with(cx, |project, cx| {
1555                                project
1556                                    .worktrees(cx)
1557                                    .filter(|worktree| {
1558                                        let worktree = worktree.read(cx);
1559                                        worktree.is_visible()
1560                                            && worktree.entries(false).any(|e| e.is_file())
1561                                    })
1562                                    .choose(&mut self.rng)
1563                            });
1564                            let Some(worktree) = worktree else { continue };
1565                            let full_path = worktree.read_with(cx, |worktree, _| {
1566                                let entry = worktree
1567                                    .entries(false)
1568                                    .filter(|e| e.is_file())
1569                                    .choose(&mut self.rng)
1570                                    .unwrap();
1571                                if entry.path.as_ref() == Path::new("") {
1572                                    Path::new(worktree.root_name()).into()
1573                                } else {
1574                                    Path::new(worktree.root_name()).join(&entry.path)
1575                                }
1576                            });
1577                            break ClientOperation::OpenBuffer {
1578                                project_root_name,
1579                                is_local,
1580                                full_path,
1581                            };
1582                        }
1583                    }
1584                }
1585
1586                // Update a git index
1587                91..=95 => {
1588                    let repo_path = executor
1589                        .block(client.fs.directories())
1590                        .choose(&mut self.rng)
1591                        .unwrap()
1592                        .clone();
1593
1594                    let mut file_paths = executor
1595                        .block(client.fs.files())
1596                        .into_iter()
1597                        .filter(|path| path.starts_with(&repo_path))
1598                        .collect::<Vec<_>>();
1599                    let count = self.rng.gen_range(0..=file_paths.len());
1600                    file_paths.shuffle(&mut self.rng);
1601                    file_paths.truncate(count);
1602
1603                    let mut contents = Vec::new();
1604                    for abs_child_file_path in &file_paths {
1605                        let child_file_path = abs_child_file_path
1606                            .strip_prefix(&repo_path)
1607                            .unwrap()
1608                            .to_path_buf();
1609                        let new_base = Alphanumeric.sample_string(&mut self.rng, 16);
1610                        contents.push((child_file_path, new_base));
1611                    }
1612
1613                    break ClientOperation::WriteGitIndex {
1614                        repo_path,
1615                        contents,
1616                    };
1617                }
1618
1619                // Create a file or directory
1620                96.. => {
1621                    let is_dir = self.rng.gen::<bool>();
1622                    let mut path = cx
1623                        .background()
1624                        .block(client.fs.directories())
1625                        .choose(&mut self.rng)
1626                        .unwrap()
1627                        .clone();
1628                    path.push(gen_file_name(&mut self.rng));
1629                    if !is_dir {
1630                        path.set_extension("rs");
1631                    }
1632                    break ClientOperation::CreateFsEntry { path, is_dir };
1633                }
1634            }
1635        })
1636    }
1637
1638    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
1639        let user_ix = self
1640            .users
1641            .iter()
1642            .position(|user| user.user_id == user_id)
1643            .unwrap();
1644        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
1645        format!("dir-{user_id}-{root_id}")
1646    }
1647
1648    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
1649        let ix = self
1650            .users
1651            .iter()
1652            .position(|user| user.user_id == user_id)
1653            .unwrap();
1654        &mut self.users[ix]
1655    }
1656}
1657
1658async fn simulate_client(
1659    client: Rc<TestClient>,
1660    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<usize>,
1661    plan: Arc<Mutex<TestPlan>>,
1662    mut cx: TestAppContext,
1663) {
1664    // Setup language server
1665    let mut language = Language::new(
1666        LanguageConfig {
1667            name: "Rust".into(),
1668            path_suffixes: vec!["rs".to_string()],
1669            ..Default::default()
1670        },
1671        None,
1672    );
1673    let _fake_language_servers = language
1674        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
1675            name: "the-fake-language-server",
1676            capabilities: lsp::LanguageServer::full_capabilities(),
1677            initializer: Some(Box::new({
1678                let plan = plan.clone();
1679                let fs = client.fs.clone();
1680                move |fake_server: &mut FakeLanguageServer| {
1681                    fake_server.handle_request::<lsp::request::Completion, _, _>(
1682                        |_, _| async move {
1683                            Ok(Some(lsp::CompletionResponse::Array(vec![
1684                                lsp::CompletionItem {
1685                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
1686                                        range: lsp::Range::new(
1687                                            lsp::Position::new(0, 0),
1688                                            lsp::Position::new(0, 0),
1689                                        ),
1690                                        new_text: "the-new-text".to_string(),
1691                                    })),
1692                                    ..Default::default()
1693                                },
1694                            ])))
1695                        },
1696                    );
1697
1698                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
1699                        |_, _| async move {
1700                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
1701                                lsp::CodeAction {
1702                                    title: "the-code-action".to_string(),
1703                                    ..Default::default()
1704                                },
1705                            )]))
1706                        },
1707                    );
1708
1709                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
1710                        |params, _| async move {
1711                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
1712                                params.position,
1713                                params.position,
1714                            ))))
1715                        },
1716                    );
1717
1718                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
1719                        let fs = fs.clone();
1720                        let plan = plan.clone();
1721                        move |_, _| {
1722                            let fs = fs.clone();
1723                            let plan = plan.clone();
1724                            async move {
1725                                let files = fs.files().await;
1726                                let count = plan.lock().rng.gen_range::<usize, _>(1..3);
1727                                let files = (0..count)
1728                                    .map(|_| files.choose(&mut plan.lock().rng).unwrap())
1729                                    .collect::<Vec<_>>();
1730                                log::info!("LSP: Returning definitions in files {:?}", &files);
1731                                Ok(Some(lsp::GotoDefinitionResponse::Array(
1732                                    files
1733                                        .into_iter()
1734                                        .map(|file| lsp::Location {
1735                                            uri: lsp::Url::from_file_path(file).unwrap(),
1736                                            range: Default::default(),
1737                                        })
1738                                        .collect(),
1739                                )))
1740                            }
1741                        }
1742                    });
1743
1744                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
1745                        let plan = plan.clone();
1746                        move |_, _| {
1747                            let mut highlights = Vec::new();
1748                            let highlight_count = plan.lock().rng.gen_range(1..=5);
1749                            for _ in 0..highlight_count {
1750                                let start_row = plan.lock().rng.gen_range(0..100);
1751                                let start_column = plan.lock().rng.gen_range(0..100);
1752                                let start = PointUtf16::new(start_row, start_column);
1753                                let end_row = plan.lock().rng.gen_range(0..100);
1754                                let end_column = plan.lock().rng.gen_range(0..100);
1755                                let end = PointUtf16::new(end_row, end_column);
1756                                let range = if start > end { end..start } else { start..end };
1757                                highlights.push(lsp::DocumentHighlight {
1758                                    range: range_to_lsp(range.clone()),
1759                                    kind: Some(lsp::DocumentHighlightKind::READ),
1760                                });
1761                            }
1762                            highlights.sort_unstable_by_key(|highlight| {
1763                                (highlight.range.start, highlight.range.end)
1764                            });
1765                            async move { Ok(Some(highlights)) }
1766                        }
1767                    });
1768                }
1769            })),
1770            ..Default::default()
1771        }))
1772        .await;
1773    client.language_registry.add(Arc::new(language));
1774
1775    while let Some(batch_id) = operation_rx.next().await {
1776        let Some((operation, skipped)) = plan.lock().next_client_operation(&client, batch_id, &cx) else { break };
1777        match apply_client_operation(&client, operation, &mut cx).await {
1778            Ok(()) => {}
1779            Err(TestError::Inapplicable) => skipped.store(true, SeqCst),
1780            Err(TestError::Other(error)) => {
1781                log::error!("{} error: {}", client.username, error);
1782            }
1783        }
1784        cx.background().simulate_random_delay().await;
1785    }
1786    log::info!("{}: done", client.username);
1787}
1788
1789fn buffer_for_full_path(
1790    client: &TestClient,
1791    project: &ModelHandle<Project>,
1792    full_path: &PathBuf,
1793    cx: &TestAppContext,
1794) -> Option<ModelHandle<language::Buffer>> {
1795    client
1796        .buffers_for_project(project)
1797        .iter()
1798        .find(|buffer| {
1799            buffer.read_with(cx, |buffer, cx| {
1800                buffer.file().unwrap().full_path(cx) == *full_path
1801            })
1802        })
1803        .cloned()
1804}
1805
1806fn project_for_root_name(
1807    client: &TestClient,
1808    root_name: &str,
1809    cx: &TestAppContext,
1810) -> Option<ModelHandle<Project>> {
1811    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1812        return Some(client.local_projects()[ix].clone());
1813    }
1814    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1815        return Some(client.remote_projects()[ix].clone());
1816    }
1817    None
1818}
1819
1820fn project_ix_for_root_name(
1821    projects: &[ModelHandle<Project>],
1822    root_name: &str,
1823    cx: &TestAppContext,
1824) -> Option<usize> {
1825    projects.iter().position(|project| {
1826        project.read_with(cx, |project, cx| {
1827            let worktree = project.visible_worktrees(cx).next().unwrap();
1828            worktree.read(cx).root_name() == root_name
1829        })
1830    })
1831}
1832
1833fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1834    project.read_with(cx, |project, cx| {
1835        project
1836            .visible_worktrees(cx)
1837            .next()
1838            .unwrap()
1839            .read(cx)
1840            .root_name()
1841            .to_string()
1842    })
1843}
1844
1845fn project_path_for_full_path(
1846    project: &ModelHandle<Project>,
1847    full_path: &Path,
1848    cx: &TestAppContext,
1849) -> Option<ProjectPath> {
1850    let mut components = full_path.components();
1851    let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1852    let path = components.as_path().into();
1853    let worktree_id = project.read_with(cx, |project, cx| {
1854        project.worktrees(cx).find_map(|worktree| {
1855            let worktree = worktree.read(cx);
1856            if worktree.root_name() == root_name {
1857                Some(worktree.id())
1858            } else {
1859                None
1860            }
1861        })
1862    })?;
1863    Some(ProjectPath { worktree_id, path })
1864}
1865
1866async fn ensure_project_shared(
1867    project: &ModelHandle<Project>,
1868    client: &TestClient,
1869    cx: &mut TestAppContext,
1870) {
1871    let first_root_name = root_name_for_project(project, cx);
1872    let active_call = cx.read(ActiveCall::global);
1873    if active_call.read_with(cx, |call, _| call.room().is_some())
1874        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
1875    {
1876        match active_call
1877            .update(cx, |call, cx| call.share_project(project.clone(), cx))
1878            .await
1879        {
1880            Ok(project_id) => {
1881                log::info!(
1882                    "{}: shared project {} with id {}",
1883                    client.username,
1884                    first_root_name,
1885                    project_id
1886                );
1887            }
1888            Err(error) => {
1889                log::error!(
1890                    "{}: error sharing project {}: {:?}",
1891                    client.username,
1892                    first_root_name,
1893                    error
1894                );
1895            }
1896        }
1897    }
1898}
1899
1900fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1901    client
1902        .local_projects()
1903        .iter()
1904        .chain(client.remote_projects().iter())
1905        .choose(rng)
1906        .cloned()
1907}
1908
1909fn gen_file_name(rng: &mut StdRng) -> String {
1910    let mut name = String::new();
1911    for _ in 0..10 {
1912        let letter = rng.gen_range('a'..='z');
1913        name.push(letter);
1914    }
1915    name
1916}
1917
1918fn path_env_var(name: &str) -> Option<PathBuf> {
1919    let value = env::var(name).ok()?;
1920    let mut path = PathBuf::from(value);
1921    if path.is_relative() {
1922        let mut abs_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
1923        abs_path.pop();
1924        abs_path.pop();
1925        abs_path.push(path);
1926        path = abs_path
1927    }
1928    Some(path)
1929}