1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::{BTreeMap, HashSet};
  10use editor::Bias;
  11use fs::{FakeFs, Fs as _};
  12use futures::StreamExt as _;
  13use gpui::{executor::Deterministic, ModelHandle, Task, TestAppContext};
  14use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  15use lsp::FakeLanguageServer;
  16use parking_lot::Mutex;
  17use project::{search::SearchQuery, Project, ProjectPath};
  18use rand::{
  19    distributions::{Alphanumeric, DistString},
  20    prelude::*,
  21};
  22use serde::{Deserialize, Serialize};
  23use std::{
  24    env,
  25    ops::Range,
  26    path::{Path, PathBuf},
  27    rc::Rc,
  28    sync::{
  29        atomic::{AtomicBool, Ordering::SeqCst},
  30        Arc,
  31    },
  32};
  33use util::ResultExt;
  34
  35#[gpui::test(iterations = 100)]
  36async fn test_random_collaboration(
  37    cx: &mut TestAppContext,
  38    deterministic: Arc<Deterministic>,
  39    rng: StdRng,
  40) {
  41    deterministic.forbid_parking();
  42
  43    let max_peers = env::var("MAX_PEERS")
  44        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  45        .unwrap_or(3);
  46    let max_operations = env::var("OPERATIONS")
  47        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  48        .unwrap_or(10);
  49
  50    let plan_load_path = path_env_var("LOAD_PLAN");
  51    let plan_save_path = path_env_var("SAVE_PLAN");
  52
  53    let mut server = TestServer::start(&deterministic).await;
  54    let db = server.app_state.db.clone();
  55
  56    let mut users = Vec::new();
  57    for ix in 0..max_peers {
  58        let username = format!("user-{}", ix + 1);
  59        let user_id = db
  60            .create_user(
  61                &format!("{username}@example.com"),
  62                false,
  63                NewUserParams {
  64                    github_login: username.clone(),
  65                    github_user_id: (ix + 1) as i32,
  66                    invite_count: 0,
  67                },
  68            )
  69            .await
  70            .unwrap()
  71            .user_id;
  72        users.push(UserTestPlan {
  73            user_id,
  74            username,
  75            online: false,
  76            next_root_id: 0,
  77            operation_ix: 0,
  78        });
  79    }
  80
  81    for (ix, user_a) in users.iter().enumerate() {
  82        for user_b in &users[ix + 1..] {
  83            server
  84                .app_state
  85                .db
  86                .send_contact_request(user_a.user_id, user_b.user_id)
  87                .await
  88                .unwrap();
  89            server
  90                .app_state
  91                .db
  92                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  93                .await
  94                .unwrap();
  95        }
  96    }
  97
  98    let plan = Arc::new(Mutex::new(TestPlan::new(rng, users, max_operations)));
  99
 100    if let Some(path) = &plan_load_path {
 101        eprintln!("loaded plan from path {:?}", path);
 102        plan.lock().load(path);
 103    }
 104
 105    let mut clients = Vec::new();
 106    let mut client_tasks = Vec::new();
 107    let mut operation_channels = Vec::new();
 108
 109    loop {
 110        let Some((next_operation, skipped)) = plan.lock().next_server_operation(&clients) else { break };
 111        let applied = apply_server_operation(
 112            deterministic.clone(),
 113            &mut server,
 114            &mut clients,
 115            &mut client_tasks,
 116            &mut operation_channels,
 117            plan.clone(),
 118            next_operation,
 119            cx,
 120        )
 121        .await;
 122        if !applied {
 123            skipped.store(false, SeqCst);
 124        }
 125    }
 126
 127    drop(operation_channels);
 128    deterministic.start_waiting();
 129    futures::future::join_all(client_tasks).await;
 130    deterministic.finish_waiting();
 131    deterministic.run_until_parked();
 132
 133    if let Some(path) = &plan_save_path {
 134        eprintln!("saved test plan to path {:?}", path);
 135        plan.lock().save(path);
 136    }
 137
 138    for (client, client_cx) in &clients {
 139        for guest_project in client.remote_projects().iter() {
 140            guest_project.read_with(client_cx, |guest_project, cx| {
 141                let host_project = clients.iter().find_map(|(client, cx)| {
 142                    let project = client
 143                        .local_projects()
 144                        .iter()
 145                        .find(|host_project| {
 146                            host_project.read_with(cx, |host_project, _| {
 147                                host_project.remote_id() == guest_project.remote_id()
 148                            })
 149                        })?
 150                        .clone();
 151                    Some((project, cx))
 152                });
 153
 154                if !guest_project.is_read_only() {
 155                    if let Some((host_project, host_cx)) = host_project {
 156                        let host_worktree_snapshots =
 157                            host_project.read_with(host_cx, |host_project, cx| {
 158                                host_project
 159                                    .worktrees(cx)
 160                                    .map(|worktree| {
 161                                        let worktree = worktree.read(cx);
 162                                        (worktree.id(), worktree.snapshot())
 163                                    })
 164                                    .collect::<BTreeMap<_, _>>()
 165                            });
 166                        let guest_worktree_snapshots = guest_project
 167                            .worktrees(cx)
 168                            .map(|worktree| {
 169                                let worktree = worktree.read(cx);
 170                                (worktree.id(), worktree.snapshot())
 171                            })
 172                            .collect::<BTreeMap<_, _>>();
 173
 174                        assert_eq!(
 175                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 176                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 177                            "{} has different worktrees than the host",
 178                            client.username
 179                        );
 180
 181                        for (id, host_snapshot) in &host_worktree_snapshots {
 182                            let guest_snapshot = &guest_worktree_snapshots[id];
 183                            assert_eq!(
 184                                guest_snapshot.root_name(),
 185                                host_snapshot.root_name(),
 186                                "{} has different root name than the host for worktree {}",
 187                                client.username,
 188                                id
 189                            );
 190                            assert_eq!(
 191                                guest_snapshot.abs_path(),
 192                                host_snapshot.abs_path(),
 193                                "{} has different abs path than the host for worktree {}",
 194                                client.username,
 195                                id
 196                            );
 197                            assert_eq!(
 198                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 199                                host_snapshot.entries(false).collect::<Vec<_>>(),
 200                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 201                                client.username,
 202                                id,
 203                                host_snapshot.abs_path(),
 204                                host_project.read_with(host_cx, |project, _| project.remote_id())
 205                            );
 206                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 207                        }
 208                    }
 209                }
 210
 211                guest_project.check_invariants(cx);
 212            });
 213        }
 214
 215        let buffers = client.buffers().clone();
 216        for (guest_project, guest_buffers) in &buffers {
 217            let project_id = if guest_project.read_with(client_cx, |project, _| {
 218                project.is_local() || project.is_read_only()
 219            }) {
 220                continue;
 221            } else {
 222                guest_project
 223                    .read_with(client_cx, |project, _| project.remote_id())
 224                    .unwrap()
 225            };
 226            let guest_user_id = client.user_id().unwrap();
 227
 228            let host_project = clients.iter().find_map(|(client, cx)| {
 229                let project = client
 230                    .local_projects()
 231                    .iter()
 232                    .find(|host_project| {
 233                        host_project.read_with(cx, |host_project, _| {
 234                            host_project.remote_id() == Some(project_id)
 235                        })
 236                    })?
 237                    .clone();
 238                Some((client.user_id().unwrap(), project, cx))
 239            });
 240
 241            let (host_user_id, host_project, host_cx) =
 242                if let Some((host_user_id, host_project, host_cx)) = host_project {
 243                    (host_user_id, host_project, host_cx)
 244                } else {
 245                    continue;
 246                };
 247
 248            for guest_buffer in guest_buffers {
 249                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 250                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 251                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 252                        panic!(
 253                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 254                            client.username,
 255                            client.peer_id(),
 256                            buffer_id
 257                        )
 258                    })
 259                });
 260                let path = host_buffer
 261                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 262
 263                assert_eq!(
 264                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 265                    0,
 266                    "{}, buffer {}, path {:?} has deferred operations",
 267                    client.username,
 268                    buffer_id,
 269                    path,
 270                );
 271                assert_eq!(
 272                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 273                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 274                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 275                    client.username,
 276                    buffer_id,
 277                    path
 278                );
 279
 280                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 281                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 282                match (host_file, guest_file) {
 283                    (Some(host_file), Some(guest_file)) => {
 284                        assert_eq!(guest_file.path(), host_file.path());
 285                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 286                        assert_eq!(
 287                            guest_file.mtime(),
 288                            host_file.mtime(),
 289                            "guest {} mtime does not match host {} for path {:?} in project {}",
 290                            guest_user_id,
 291                            host_user_id,
 292                            guest_file.path(),
 293                            project_id,
 294                        );
 295                    }
 296                    (None, None) => {}
 297                    (None, _) => panic!("host's file is None, guest's isn't"),
 298                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 299                }
 300
 301                let host_diff_base =
 302                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 303                let guest_diff_base = guest_buffer
 304                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 305                assert_eq!(guest_diff_base, host_diff_base);
 306            }
 307        }
 308    }
 309
 310    for (client, mut cx) in clients {
 311        cx.update(|cx| {
 312            cx.clear_globals();
 313            drop(client);
 314        });
 315    }
 316}
 317
 318async fn apply_server_operation(
 319    deterministic: Arc<Deterministic>,
 320    server: &mut TestServer,
 321    clients: &mut Vec<(Rc<TestClient>, TestAppContext)>,
 322    client_tasks: &mut Vec<Task<()>>,
 323    operation_channels: &mut Vec<futures::channel::mpsc::UnboundedSender<usize>>,
 324    plan: Arc<Mutex<TestPlan>>,
 325    operation: Operation,
 326    cx: &mut TestAppContext,
 327) -> bool {
 328    match operation {
 329        Operation::AddConnection { user_id } => {
 330            let username;
 331            {
 332                let mut plan = plan.lock();
 333                let mut user = plan.user(user_id);
 334                if user.online {
 335                    return false;
 336                }
 337                user.online = true;
 338                username = user.username.clone();
 339            };
 340            log::info!("Adding new connection for {}", username);
 341            let next_entity_id = (user_id.0 * 10_000) as usize;
 342            let mut client_cx = TestAppContext::new(
 343                cx.foreground_platform(),
 344                cx.platform(),
 345                deterministic.build_foreground(user_id.0 as usize),
 346                deterministic.build_background(),
 347                cx.font_cache(),
 348                cx.leak_detector(),
 349                next_entity_id,
 350                cx.function_name.clone(),
 351            );
 352
 353            let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 354            let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 355            operation_channels.push(operation_tx);
 356            clients.push((client.clone(), client_cx.clone()));
 357            client_tasks.push(client_cx.foreground().spawn(simulate_client(
 358                client,
 359                operation_rx,
 360                plan.clone(),
 361                client_cx,
 362            )));
 363
 364            log::info!("Added connection for {}", username);
 365        }
 366
 367        Operation::RemoveConnection { user_id } => {
 368            log::info!("Simulating full disconnection of user {}", user_id);
 369            let client_ix = clients
 370                .iter()
 371                .position(|(client, cx)| client.current_user_id(cx) == user_id);
 372            let Some(client_ix) = client_ix else { return false };
 373            let user_connection_ids = server
 374                .connection_pool
 375                .lock()
 376                .user_connection_ids(user_id)
 377                .collect::<Vec<_>>();
 378            assert_eq!(user_connection_ids.len(), 1);
 379            let removed_peer_id = user_connection_ids[0].into();
 380            let (client, mut client_cx) = clients.remove(client_ix);
 381            let client_task = client_tasks.remove(client_ix);
 382            operation_channels.remove(client_ix);
 383            server.forbid_connections();
 384            server.disconnect_client(removed_peer_id);
 385            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 386            deterministic.start_waiting();
 387            log::info!("Waiting for user {} to exit...", user_id);
 388            client_task.await;
 389            deterministic.finish_waiting();
 390            server.allow_connections();
 391
 392            for project in client.remote_projects().iter() {
 393                project.read_with(&client_cx, |project, _| {
 394                    assert!(
 395                        project.is_read_only(),
 396                        "project {:?} should be read only",
 397                        project.remote_id()
 398                    )
 399                });
 400            }
 401
 402            for (client, cx) in clients {
 403                let contacts = server
 404                    .app_state
 405                    .db
 406                    .get_contacts(client.current_user_id(cx))
 407                    .await
 408                    .unwrap();
 409                let pool = server.connection_pool.lock();
 410                for contact in contacts {
 411                    if let db::Contact::Accepted { user_id: id, .. } = contact {
 412                        if pool.is_user_online(id) {
 413                            assert_ne!(
 414                                id, user_id,
 415                                "removed client is still a contact of another peer"
 416                            );
 417                        }
 418                    }
 419                }
 420            }
 421
 422            log::info!("{} removed", client.username);
 423            plan.lock().user(user_id).online = false;
 424            client_cx.update(|cx| {
 425                cx.clear_globals();
 426                drop(client);
 427            });
 428        }
 429
 430        Operation::BounceConnection { user_id } => {
 431            log::info!("Simulating temporary disconnection of user {}", user_id);
 432            let user_connection_ids = server
 433                .connection_pool
 434                .lock()
 435                .user_connection_ids(user_id)
 436                .collect::<Vec<_>>();
 437            if user_connection_ids.is_empty() {
 438                return false;
 439            }
 440            assert_eq!(user_connection_ids.len(), 1);
 441            let peer_id = user_connection_ids[0].into();
 442            server.disconnect_client(peer_id);
 443            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 444        }
 445
 446        Operation::RestartServer => {
 447            log::info!("Simulating server restart");
 448            server.reset().await;
 449            deterministic.advance_clock(RECEIVE_TIMEOUT);
 450            server.start().await.unwrap();
 451            deterministic.advance_clock(CLEANUP_TIMEOUT);
 452            let environment = &server.app_state.config.zed_environment;
 453            let stale_room_ids = server
 454                .app_state
 455                .db
 456                .stale_room_ids(environment, server.id())
 457                .await
 458                .unwrap();
 459            assert_eq!(stale_room_ids, vec![]);
 460        }
 461
 462        Operation::MutateClients {
 463            user_ids,
 464            batch_id,
 465            quiesce,
 466        } => {
 467            let mut applied = false;
 468            for user_id in user_ids {
 469                let client_ix = clients
 470                    .iter()
 471                    .position(|(client, cx)| client.current_user_id(cx) == user_id);
 472                let Some(client_ix) = client_ix else { continue };
 473                applied = true;
 474                if let Err(err) = operation_channels[client_ix].unbounded_send(batch_id) {
 475                    // panic!("error signaling user {}, client {}", user_id, client_ix);
 476                }
 477            }
 478
 479            if quiesce && applied {
 480                deterministic.run_until_parked();
 481            }
 482
 483            return applied;
 484        }
 485    }
 486    true
 487}
 488
 489async fn apply_client_operation(
 490    client: &TestClient,
 491    operation: ClientOperation,
 492    cx: &mut TestAppContext,
 493) -> Result<bool> {
 494    match operation {
 495        ClientOperation::AcceptIncomingCall => {
 496            let active_call = cx.read(ActiveCall::global);
 497            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 498                return Ok(false);
 499            }
 500
 501            log::info!("{}: accepting incoming call", client.username);
 502            active_call
 503                .update(cx, |call, cx| call.accept_incoming(cx))
 504                .await?;
 505        }
 506
 507        ClientOperation::RejectIncomingCall => {
 508            let active_call = cx.read(ActiveCall::global);
 509            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 510                return Ok(false);
 511            }
 512
 513            log::info!("{}: declining incoming call", client.username);
 514            active_call.update(cx, |call, _| call.decline_incoming())?;
 515        }
 516
 517        ClientOperation::LeaveCall => {
 518            let active_call = cx.read(ActiveCall::global);
 519            if active_call.read_with(cx, |call, _| call.room().is_none()) {
 520                return Ok(false);
 521            }
 522
 523            log::info!("{}: hanging up", client.username);
 524            active_call.update(cx, |call, cx| call.hang_up(cx))?;
 525        }
 526
 527        ClientOperation::InviteContactToCall { user_id } => {
 528            let active_call = cx.read(ActiveCall::global);
 529
 530            log::info!("{}: inviting {}", client.username, user_id,);
 531            active_call
 532                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 533                .await
 534                .log_err();
 535        }
 536
 537        ClientOperation::OpenLocalProject { first_root_name } => {
 538            log::info!(
 539                "{}: opening local project at {:?}",
 540                client.username,
 541                first_root_name
 542            );
 543
 544            let root_path = Path::new("/").join(&first_root_name);
 545            client.fs.create_dir(&root_path).await.unwrap();
 546            client
 547                .fs
 548                .create_file(&root_path.join("main.rs"), Default::default())
 549                .await
 550                .unwrap();
 551            let project = client.build_local_project(root_path, cx).await.0;
 552            ensure_project_shared(&project, client, cx).await;
 553            client.local_projects_mut().push(project.clone());
 554        }
 555
 556        ClientOperation::AddWorktreeToProject {
 557            project_root_name,
 558            new_root_path,
 559        } => {
 560            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 561                return Ok(false)
 562            };
 563
 564            log::info!(
 565                "{}: finding/creating local worktree at {:?} to project with root path {}",
 566                client.username,
 567                new_root_path,
 568                project_root_name
 569            );
 570
 571            ensure_project_shared(&project, client, cx).await;
 572            if !client.fs.paths().await.contains(&new_root_path) {
 573                client.fs.create_dir(&new_root_path).await.unwrap();
 574            }
 575            project
 576                .update(cx, |project, cx| {
 577                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 578                })
 579                .await
 580                .unwrap();
 581        }
 582
 583        ClientOperation::CloseRemoteProject { project_root_name } => {
 584            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 585                return Ok(false)
 586            };
 587
 588            log::info!(
 589                "{}: closing remote project with root path {}",
 590                client.username,
 591                project_root_name,
 592            );
 593
 594            let ix = client
 595                .remote_projects()
 596                .iter()
 597                .position(|p| p == &project)
 598                .unwrap();
 599            cx.update(|_| {
 600                client.remote_projects_mut().remove(ix);
 601                client.buffers().retain(|project, _| project != project);
 602                drop(project);
 603            });
 604        }
 605
 606        ClientOperation::OpenRemoteProject {
 607            host_id,
 608            first_root_name,
 609        } => {
 610            let active_call = cx.read(ActiveCall::global);
 611            let project = active_call.update(cx, |call, cx| {
 612                let room = call.room().cloned()?;
 613                let participant = room
 614                    .read(cx)
 615                    .remote_participants()
 616                    .get(&host_id.to_proto())?;
 617                let project_id = participant
 618                    .projects
 619                    .iter()
 620                    .find(|project| project.worktree_root_names[0] == first_root_name)?
 621                    .id;
 622                Some(room.update(cx, |room, cx| {
 623                    room.join_project(
 624                        project_id,
 625                        client.language_registry.clone(),
 626                        FakeFs::new(cx.background().clone()),
 627                        cx,
 628                    )
 629                }))
 630            });
 631            let Some(project) = project else {
 632                return Ok(false)
 633            };
 634
 635            log::info!(
 636                "{}: joining remote project of user {}, root name {}",
 637                client.username,
 638                host_id,
 639                first_root_name,
 640            );
 641
 642            let project = project.await?;
 643            client.remote_projects_mut().push(project.clone());
 644        }
 645
 646        ClientOperation::CreateWorktreeEntry {
 647            project_root_name,
 648            is_local,
 649            full_path,
 650            is_dir,
 651        } => {
 652            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 653                return Ok(false);
 654            };
 655            let Some(project_path) = project_path_for_full_path(&project, &full_path, cx) else {
 656                return Ok(false);
 657            };
 658
 659            log::info!(
 660                "{}: creating {} at path {:?} in {} project {}",
 661                client.username,
 662                if is_dir { "dir" } else { "file" },
 663                full_path,
 664                if is_local { "local" } else { "remote" },
 665                project_root_name,
 666            );
 667
 668            ensure_project_shared(&project, client, cx).await;
 669            project
 670                .update(cx, |p, cx| p.create_entry(project_path, is_dir, cx))
 671                .unwrap()
 672                .await?;
 673        }
 674
 675        ClientOperation::OpenBuffer {
 676            project_root_name,
 677            is_local,
 678            full_path,
 679        } => {
 680            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 681                return Ok(false);
 682            };
 683            let Some(project_path) = project_path_for_full_path(&project, &full_path, cx) else {
 684                return Ok(false);
 685            };
 686
 687            log::info!(
 688                "{}: opening buffer {:?} in {} project {}",
 689                client.username,
 690                full_path,
 691                if is_local { "local" } else { "remote" },
 692                project_root_name,
 693            );
 694
 695            ensure_project_shared(&project, client, cx).await;
 696            let buffer = project
 697                .update(cx, |project, cx| project.open_buffer(project_path, cx))
 698                .await?;
 699            client.buffers_for_project(&project).insert(buffer);
 700        }
 701
 702        ClientOperation::EditBuffer {
 703            project_root_name,
 704            is_local,
 705            full_path,
 706            edits,
 707        } => {
 708            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 709                return Ok(false);
 710            };
 711            let Some(buffer) =
 712                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 713                    return Ok(false);
 714                };
 715
 716            log::info!(
 717                "{}: editing buffer {:?} in {} project {} with {:?}",
 718                client.username,
 719                full_path,
 720                if is_local { "local" } else { "remote" },
 721                project_root_name,
 722                edits
 723            );
 724
 725            ensure_project_shared(&project, client, cx).await;
 726            buffer.update(cx, |buffer, cx| {
 727                let snapshot = buffer.snapshot();
 728                buffer.edit(
 729                    edits.into_iter().map(|(range, text)| {
 730                        let start = snapshot.clip_offset(range.start, Bias::Left);
 731                        let end = snapshot.clip_offset(range.end, Bias::Right);
 732                        (start..end, text)
 733                    }),
 734                    None,
 735                    cx,
 736                );
 737            });
 738        }
 739
 740        ClientOperation::CloseBuffer {
 741            project_root_name,
 742            is_local,
 743            full_path,
 744        } => {
 745            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 746                return Ok(false);
 747            };
 748            let Some(buffer) =
 749                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 750                    return Ok(false);
 751                };
 752
 753            log::info!(
 754                "{}: closing buffer {:?} in {} project {}",
 755                client.username,
 756                full_path,
 757                if is_local { "local" } else { "remote" },
 758                project_root_name
 759            );
 760
 761            ensure_project_shared(&project, client, cx).await;
 762            cx.update(|_| {
 763                client.buffers_for_project(&project).remove(&buffer);
 764                drop(buffer);
 765            });
 766        }
 767
 768        ClientOperation::SaveBuffer {
 769            project_root_name,
 770            is_local,
 771            full_path,
 772            detach,
 773        } => {
 774            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 775                return Ok(false);
 776            };
 777            let Some(buffer) =
 778                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 779                    return Ok(false);
 780                };
 781
 782            log::info!(
 783                "{}: saving buffer {:?} in {} project {}{}",
 784                client.username,
 785                full_path,
 786                if is_local { "local" } else { "remote" },
 787                project_root_name,
 788                if detach { ", detaching" } else { ", awaiting" }
 789            );
 790
 791            ensure_project_shared(&project, client, cx).await;
 792            let (requested_version, save) =
 793                buffer.update(cx, |buffer, cx| (buffer.version(), buffer.save(cx)));
 794            let save = cx.background().spawn(async move {
 795                let (saved_version, _, _) = save
 796                    .await
 797                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 798                assert!(saved_version.observed_all(&requested_version));
 799                anyhow::Ok(())
 800            });
 801            if detach {
 802                log::info!("{}: detaching save request", client.username);
 803                cx.update(|cx| save.detach_and_log_err(cx));
 804            } else {
 805                save.await?;
 806            }
 807        }
 808
 809        ClientOperation::RequestLspDataInBuffer {
 810            project_root_name,
 811            is_local,
 812            full_path,
 813            offset,
 814            kind,
 815            detach,
 816        } => {
 817            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 818                return Ok(false);
 819            };
 820            let Some(buffer) =
 821                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 822                    return Ok(false);
 823                };
 824
 825            log::info!(
 826                "{}: request LSP {:?} for buffer {:?} in {} project {}{}",
 827                client.username,
 828                kind,
 829                full_path,
 830                if is_local { "local" } else { "remote" },
 831                project_root_name,
 832                if detach { ", detaching" } else { ", awaiting" }
 833            );
 834
 835            let offset = buffer.read_with(cx, |b, _| b.clip_offset(offset, Bias::Left));
 836            let request = match kind {
 837                LspRequestKind::Rename => cx.spawn(|mut cx| async move {
 838                    project
 839                        .update(&mut cx, |p, cx| p.prepare_rename(buffer, offset, cx))
 840                        .await?;
 841                    anyhow::Ok(())
 842                }),
 843                LspRequestKind::Completion => cx.spawn(|mut cx| async move {
 844                    project
 845                        .update(&mut cx, |p, cx| p.completions(&buffer, offset, cx))
 846                        .await?;
 847                    Ok(())
 848                }),
 849                LspRequestKind::CodeAction => cx.spawn(|mut cx| async move {
 850                    project
 851                        .update(&mut cx, |p, cx| p.code_actions(&buffer, offset..offset, cx))
 852                        .await?;
 853                    Ok(())
 854                }),
 855                LspRequestKind::Definition => cx.spawn(|mut cx| async move {
 856                    project
 857                        .update(&mut cx, |p, cx| p.definition(&buffer, offset, cx))
 858                        .await?;
 859                    Ok(())
 860                }),
 861                LspRequestKind::Highlights => cx.spawn(|mut cx| async move {
 862                    project
 863                        .update(&mut cx, |p, cx| p.document_highlights(&buffer, offset, cx))
 864                        .await?;
 865                    Ok(())
 866                }),
 867            };
 868            if detach {
 869                request.detach();
 870            } else {
 871                request.await?;
 872            }
 873        }
 874
 875        ClientOperation::SearchProject {
 876            project_root_name,
 877            is_local,
 878            query,
 879            detach,
 880        } => {
 881            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 882                return Ok(false);
 883            };
 884
 885            log::info!(
 886                "{}: search {} project {} for {:?}{}",
 887                client.username,
 888                if is_local { "local" } else { "remote" },
 889                project_root_name,
 890                query,
 891                if detach { ", detaching" } else { ", awaiting" }
 892            );
 893
 894            let search = project.update(cx, |project, cx| {
 895                project.search(SearchQuery::text(query, false, false), cx)
 896            });
 897            let search = cx.background().spawn(async move {
 898                search
 899                    .await
 900                    .map_err(|err| anyhow!("search request failed: {:?}", err))
 901            });
 902            if detach {
 903                log::info!("{}: detaching save request", client.username);
 904                cx.update(|cx| search.detach_and_log_err(cx));
 905            } else {
 906                search.await?;
 907            }
 908        }
 909
 910        ClientOperation::CreateFsEntry { path, is_dir } => {
 911            if client.fs.metadata(&path.parent().unwrap()).await?.is_none() {
 912                return Ok(false);
 913            }
 914
 915            log::info!(
 916                "{}: creating {} at {:?}",
 917                client.username,
 918                if is_dir { "dir" } else { "file" },
 919                path
 920            );
 921
 922            if is_dir {
 923                client.fs.create_dir(&path).await.unwrap();
 924            } else {
 925                client
 926                    .fs
 927                    .create_file(&path, Default::default())
 928                    .await
 929                    .unwrap();
 930            }
 931        }
 932
 933        ClientOperation::WriteGitIndex {
 934            repo_path,
 935            contents,
 936        } => {
 937            if !client
 938                .fs
 939                .metadata(&repo_path)
 940                .await?
 941                .map_or(false, |m| m.is_dir)
 942            {
 943                return Ok(false);
 944            }
 945
 946            log::info!(
 947                "{}: writing git index for repo {:?}: {:?}",
 948                client.username,
 949                repo_path,
 950                contents
 951            );
 952
 953            let dot_git_dir = repo_path.join(".git");
 954            let contents = contents
 955                .iter()
 956                .map(|(path, contents)| (path.as_path(), contents.clone()))
 957                .collect::<Vec<_>>();
 958            if client.fs.metadata(&dot_git_dir).await?.is_none() {
 959                client.fs.create_dir(&dot_git_dir).await?;
 960            }
 961            client.fs.set_index_for_repo(&dot_git_dir, &contents).await;
 962        }
 963    }
 964    Ok(true)
 965}
 966
 967struct TestPlan {
 968    rng: StdRng,
 969    replay: bool,
 970    stored_operations: Vec<(StoredOperation, Arc<AtomicBool>)>,
 971    max_operations: usize,
 972    operation_ix: usize,
 973    users: Vec<UserTestPlan>,
 974    next_batch_id: usize,
 975    allow_server_restarts: bool,
 976    allow_client_reconnection: bool,
 977    allow_client_disconnection: bool,
 978}
 979
 980struct UserTestPlan {
 981    user_id: UserId,
 982    username: String,
 983    next_root_id: usize,
 984    operation_ix: usize,
 985    online: bool,
 986}
 987
 988#[derive(Clone, Debug, Serialize, Deserialize)]
 989#[serde(untagged)]
 990enum StoredOperation {
 991    Server(Operation),
 992    Client {
 993        user_id: UserId,
 994        batch_id: usize,
 995        operation: ClientOperation,
 996    },
 997}
 998
 999#[derive(Clone, Debug, Serialize, Deserialize)]
1000enum Operation {
1001    AddConnection {
1002        user_id: UserId,
1003    },
1004    RemoveConnection {
1005        user_id: UserId,
1006    },
1007    BounceConnection {
1008        user_id: UserId,
1009    },
1010    RestartServer,
1011    MutateClients {
1012        batch_id: usize,
1013        #[serde(skip_serializing)]
1014        #[serde(skip_deserializing)]
1015        user_ids: Vec<UserId>,
1016        quiesce: bool,
1017    },
1018}
1019
1020#[derive(Clone, Debug, Serialize, Deserialize)]
1021enum ClientOperation {
1022    AcceptIncomingCall,
1023    RejectIncomingCall,
1024    LeaveCall,
1025    InviteContactToCall {
1026        user_id: UserId,
1027    },
1028    OpenLocalProject {
1029        first_root_name: String,
1030    },
1031    OpenRemoteProject {
1032        host_id: UserId,
1033        first_root_name: String,
1034    },
1035    AddWorktreeToProject {
1036        project_root_name: String,
1037        new_root_path: PathBuf,
1038    },
1039    CloseRemoteProject {
1040        project_root_name: String,
1041    },
1042    OpenBuffer {
1043        project_root_name: String,
1044        is_local: bool,
1045        full_path: PathBuf,
1046    },
1047    SearchProject {
1048        project_root_name: String,
1049        is_local: bool,
1050        query: String,
1051        detach: bool,
1052    },
1053    EditBuffer {
1054        project_root_name: String,
1055        is_local: bool,
1056        full_path: PathBuf,
1057        edits: Vec<(Range<usize>, Arc<str>)>,
1058    },
1059    CloseBuffer {
1060        project_root_name: String,
1061        is_local: bool,
1062        full_path: PathBuf,
1063    },
1064    SaveBuffer {
1065        project_root_name: String,
1066        is_local: bool,
1067        full_path: PathBuf,
1068        detach: bool,
1069    },
1070    RequestLspDataInBuffer {
1071        project_root_name: String,
1072        is_local: bool,
1073        full_path: PathBuf,
1074        offset: usize,
1075        kind: LspRequestKind,
1076        detach: bool,
1077    },
1078    CreateWorktreeEntry {
1079        project_root_name: String,
1080        is_local: bool,
1081        full_path: PathBuf,
1082        is_dir: bool,
1083    },
1084    CreateFsEntry {
1085        path: PathBuf,
1086        is_dir: bool,
1087    },
1088    WriteGitIndex {
1089        repo_path: PathBuf,
1090        contents: Vec<(PathBuf, String)>,
1091    },
1092}
1093
1094#[derive(Clone, Debug, Serialize, Deserialize)]
1095enum LspRequestKind {
1096    Rename,
1097    Completion,
1098    CodeAction,
1099    Definition,
1100    Highlights,
1101}
1102
1103impl TestPlan {
1104    fn new(mut rng: StdRng, users: Vec<UserTestPlan>, max_operations: usize) -> Self {
1105        Self {
1106            replay: false,
1107            allow_server_restarts: rng.gen_bool(0.7),
1108            allow_client_reconnection: rng.gen_bool(0.7),
1109            allow_client_disconnection: rng.gen_bool(0.1),
1110            stored_operations: Vec::new(),
1111            operation_ix: 0,
1112            next_batch_id: 0,
1113            max_operations,
1114            users,
1115            rng,
1116        }
1117    }
1118
1119    fn load(&mut self, path: &Path) {
1120        let json = std::fs::read_to_string(path).unwrap();
1121        self.replay = true;
1122        let stored_operations: Vec<StoredOperation> = serde_json::from_str(&json).unwrap();
1123        self.stored_operations = stored_operations
1124            .iter()
1125            .cloned()
1126            .enumerate()
1127            .map(|(i, mut operation)| {
1128                if let StoredOperation::Server(Operation::MutateClients {
1129                    batch_id: current_batch_id,
1130                    user_ids,
1131                    ..
1132                }) = &mut operation
1133                {
1134                    assert!(user_ids.is_empty());
1135                    user_ids.extend(stored_operations[i + 1..].iter().filter_map(|operation| {
1136                        if let StoredOperation::Client {
1137                            user_id, batch_id, ..
1138                        } = operation
1139                        {
1140                            if batch_id == current_batch_id {
1141                                return Some(user_id);
1142                            }
1143                        }
1144                        None
1145                    }));
1146                    user_ids.sort_unstable();
1147                }
1148                (operation, Arc::new(AtomicBool::new(false)))
1149            })
1150            .collect()
1151    }
1152
1153    fn save(&mut self, path: &Path) {
1154        // Format each operation as one line
1155        let mut json = Vec::new();
1156        json.push(b'[');
1157        for (operation, skipped) in &self.stored_operations {
1158            if skipped.load(SeqCst) {
1159                continue;
1160            }
1161            if json.len() > 1 {
1162                json.push(b',');
1163            }
1164            json.extend_from_slice(b"\n  ");
1165            serde_json::to_writer(&mut json, operation).unwrap();
1166        }
1167        json.extend_from_slice(b"\n]\n");
1168        std::fs::write(path, &json).unwrap();
1169    }
1170
1171    fn next_server_operation(
1172        &mut self,
1173        clients: &[(Rc<TestClient>, TestAppContext)],
1174    ) -> Option<(Operation, Arc<AtomicBool>)> {
1175        if self.replay {
1176            while let Some(stored_operation) = self.stored_operations.get(self.operation_ix) {
1177                self.operation_ix += 1;
1178                if let (StoredOperation::Server(operation), skipped) = stored_operation {
1179                    return Some((operation.clone(), skipped.clone()));
1180                }
1181            }
1182            None
1183        } else {
1184            let operation = self.generate_server_operation(clients)?;
1185            let skipped = Arc::new(AtomicBool::new(false));
1186            self.stored_operations
1187                .push((StoredOperation::Server(operation.clone()), skipped.clone()));
1188            Some((operation, skipped))
1189        }
1190    }
1191
1192    fn next_client_operation(
1193        &mut self,
1194        client: &TestClient,
1195        current_batch_id: usize,
1196        cx: &TestAppContext,
1197    ) -> Option<(ClientOperation, Arc<AtomicBool>)> {
1198        let current_user_id = client.current_user_id(cx);
1199        let user_ix = self
1200            .users
1201            .iter()
1202            .position(|user| user.user_id == current_user_id)
1203            .unwrap();
1204        let user_plan = &mut self.users[user_ix];
1205
1206        if self.replay {
1207            while let Some(stored_operation) = self.stored_operations.get(user_plan.operation_ix) {
1208                user_plan.operation_ix += 1;
1209                if let (
1210                    StoredOperation::Client {
1211                        user_id, operation, ..
1212                    },
1213                    skipped,
1214                ) = stored_operation
1215                {
1216                    if user_id == &current_user_id {
1217                        return Some((operation.clone(), skipped.clone()));
1218                    }
1219                }
1220            }
1221            None
1222        } else {
1223            let operation = self.generate_client_operation(current_user_id, client, cx)?;
1224            let skipped = Arc::new(AtomicBool::new(false));
1225            self.stored_operations.push((
1226                StoredOperation::Client {
1227                    user_id: current_user_id,
1228                    batch_id: current_batch_id,
1229                    operation: operation.clone(),
1230                },
1231                skipped.clone(),
1232            ));
1233            Some((operation, skipped))
1234        }
1235    }
1236
1237    fn generate_server_operation(
1238        &mut self,
1239        clients: &[(Rc<TestClient>, TestAppContext)],
1240    ) -> Option<Operation> {
1241        if self.operation_ix == self.max_operations {
1242            return None;
1243        }
1244
1245        Some(loop {
1246            break match self.rng.gen_range(0..100) {
1247                0..=29 if clients.len() < self.users.len() => {
1248                    let user = self
1249                        .users
1250                        .iter()
1251                        .filter(|u| !u.online)
1252                        .choose(&mut self.rng)
1253                        .unwrap();
1254                    self.operation_ix += 1;
1255                    Operation::AddConnection {
1256                        user_id: user.user_id,
1257                    }
1258                }
1259                30..=34 if clients.len() > 1 && self.allow_client_disconnection => {
1260                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1261                    let user_id = client.current_user_id(cx);
1262                    self.operation_ix += 1;
1263                    Operation::RemoveConnection { user_id }
1264                }
1265                35..=39 if clients.len() > 1 && self.allow_client_reconnection => {
1266                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1267                    let user_id = client.current_user_id(cx);
1268                    self.operation_ix += 1;
1269                    Operation::BounceConnection { user_id }
1270                }
1271                40..=44 if self.allow_server_restarts && clients.len() > 1 => {
1272                    self.operation_ix += 1;
1273                    Operation::RestartServer
1274                }
1275                _ if !clients.is_empty() => {
1276                    let count = self
1277                        .rng
1278                        .gen_range(1..10)
1279                        .min(self.max_operations - self.operation_ix);
1280                    let batch_id = util::post_inc(&mut self.next_batch_id);
1281                    let mut user_ids = (0..count)
1282                        .map(|_| {
1283                            let ix = self.rng.gen_range(0..clients.len());
1284                            let (client, cx) = &clients[ix];
1285                            client.current_user_id(cx)
1286                        })
1287                        .collect::<Vec<_>>();
1288                    user_ids.sort_unstable();
1289                    Operation::MutateClients {
1290                        user_ids,
1291                        batch_id,
1292                        quiesce: self.rng.gen_bool(0.7),
1293                    }
1294                }
1295                _ => continue,
1296            };
1297        })
1298    }
1299
1300    fn generate_client_operation(
1301        &mut self,
1302        user_id: UserId,
1303        client: &TestClient,
1304        cx: &TestAppContext,
1305    ) -> Option<ClientOperation> {
1306        if self.operation_ix == self.max_operations {
1307            return None;
1308        }
1309
1310        let executor = cx.background();
1311        self.operation_ix += 1;
1312        let call = cx.read(ActiveCall::global);
1313        Some(loop {
1314            match self.rng.gen_range(0..100_u32) {
1315                // Mutate the call
1316                0..=29 => {
1317                    // Respond to an incoming call
1318                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
1319                        break if self.rng.gen_bool(0.7) {
1320                            ClientOperation::AcceptIncomingCall
1321                        } else {
1322                            ClientOperation::RejectIncomingCall
1323                        };
1324                    }
1325
1326                    match self.rng.gen_range(0..100_u32) {
1327                        // Invite a contact to the current call
1328                        0..=70 => {
1329                            let available_contacts =
1330                                client.user_store.read_with(cx, |user_store, _| {
1331                                    user_store
1332                                        .contacts()
1333                                        .iter()
1334                                        .filter(|contact| contact.online && !contact.busy)
1335                                        .cloned()
1336                                        .collect::<Vec<_>>()
1337                                });
1338                            if !available_contacts.is_empty() {
1339                                let contact = available_contacts.choose(&mut self.rng).unwrap();
1340                                break ClientOperation::InviteContactToCall {
1341                                    user_id: UserId(contact.user.id as i32),
1342                                };
1343                            }
1344                        }
1345
1346                        // Leave the current call
1347                        71.. => {
1348                            if self.allow_client_disconnection
1349                                && call.read_with(cx, |call, _| call.room().is_some())
1350                            {
1351                                break ClientOperation::LeaveCall;
1352                            }
1353                        }
1354                    }
1355                }
1356
1357                // Mutate projects
1358                30..=59 => match self.rng.gen_range(0..100_u32) {
1359                    // Open a new project
1360                    0..=70 => {
1361                        // Open a remote project
1362                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
1363                            let existing_remote_project_ids = cx.read(|cx| {
1364                                client
1365                                    .remote_projects()
1366                                    .iter()
1367                                    .map(|p| p.read(cx).remote_id().unwrap())
1368                                    .collect::<Vec<_>>()
1369                            });
1370                            let new_remote_projects = room.read_with(cx, |room, _| {
1371                                room.remote_participants()
1372                                    .values()
1373                                    .flat_map(|participant| {
1374                                        participant.projects.iter().filter_map(|project| {
1375                                            if existing_remote_project_ids.contains(&project.id) {
1376                                                None
1377                                            } else {
1378                                                Some((
1379                                                    UserId::from_proto(participant.user.id),
1380                                                    project.worktree_root_names[0].clone(),
1381                                                ))
1382                                            }
1383                                        })
1384                                    })
1385                                    .collect::<Vec<_>>()
1386                            });
1387                            if !new_remote_projects.is_empty() {
1388                                let (host_id, first_root_name) =
1389                                    new_remote_projects.choose(&mut self.rng).unwrap().clone();
1390                                break ClientOperation::OpenRemoteProject {
1391                                    host_id,
1392                                    first_root_name,
1393                                };
1394                            }
1395                        }
1396                        // Open a local project
1397                        else {
1398                            let first_root_name = self.next_root_dir_name(user_id);
1399                            break ClientOperation::OpenLocalProject { first_root_name };
1400                        }
1401                    }
1402
1403                    // Close a remote project
1404                    71..=80 => {
1405                        if !client.remote_projects().is_empty() {
1406                            let project = client
1407                                .remote_projects()
1408                                .choose(&mut self.rng)
1409                                .unwrap()
1410                                .clone();
1411                            let first_root_name = root_name_for_project(&project, cx);
1412                            break ClientOperation::CloseRemoteProject {
1413                                project_root_name: first_root_name,
1414                            };
1415                        }
1416                    }
1417
1418                    // Mutate project worktrees
1419                    81.. => match self.rng.gen_range(0..100_u32) {
1420                        // Add a worktree to a local project
1421                        0..=50 => {
1422                            let Some(project) = client
1423                                .local_projects()
1424                                .choose(&mut self.rng)
1425                                .cloned() else { continue };
1426                            let project_root_name = root_name_for_project(&project, cx);
1427                            let mut paths = executor.block(client.fs.paths());
1428                            paths.remove(0);
1429                            let new_root_path = if paths.is_empty() || self.rng.gen() {
1430                                Path::new("/").join(&self.next_root_dir_name(user_id))
1431                            } else {
1432                                paths.choose(&mut self.rng).unwrap().clone()
1433                            };
1434                            break ClientOperation::AddWorktreeToProject {
1435                                project_root_name,
1436                                new_root_path,
1437                            };
1438                        }
1439
1440                        // Add an entry to a worktree
1441                        _ => {
1442                            let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1443                            let project_root_name = root_name_for_project(&project, cx);
1444                            let is_local = project.read_with(cx, |project, _| project.is_local());
1445                            let worktree = project.read_with(cx, |project, cx| {
1446                                project
1447                                    .worktrees(cx)
1448                                    .filter(|worktree| {
1449                                        let worktree = worktree.read(cx);
1450                                        worktree.is_visible()
1451                                            && worktree.entries(false).any(|e| e.is_file())
1452                                            && worktree.root_entry().map_or(false, |e| e.is_dir())
1453                                    })
1454                                    .choose(&mut self.rng)
1455                            });
1456                            let Some(worktree) = worktree else { continue };
1457                            let is_dir = self.rng.gen::<bool>();
1458                            let mut full_path =
1459                                worktree.read_with(cx, |w, _| PathBuf::from(w.root_name()));
1460                            full_path.push(gen_file_name(&mut self.rng));
1461                            if !is_dir {
1462                                full_path.set_extension("rs");
1463                            }
1464                            break ClientOperation::CreateWorktreeEntry {
1465                                project_root_name,
1466                                is_local,
1467                                full_path,
1468                                is_dir,
1469                            };
1470                        }
1471                    },
1472                },
1473
1474                // Query and mutate buffers
1475                60..=90 => {
1476                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1477                    let project_root_name = root_name_for_project(&project, cx);
1478                    let is_local = project.read_with(cx, |project, _| project.is_local());
1479
1480                    match self.rng.gen_range(0..100_u32) {
1481                        // Manipulate an existing buffer
1482                        0..=70 => {
1483                            let Some(buffer) = client
1484                                .buffers_for_project(&project)
1485                                .iter()
1486                                .choose(&mut self.rng)
1487                                .cloned() else { continue };
1488
1489                            let full_path = buffer
1490                                .read_with(cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
1491
1492                            match self.rng.gen_range(0..100_u32) {
1493                                // Close the buffer
1494                                0..=15 => {
1495                                    break ClientOperation::CloseBuffer {
1496                                        project_root_name,
1497                                        is_local,
1498                                        full_path,
1499                                    };
1500                                }
1501                                // Save the buffer
1502                                16..=29 if buffer.read_with(cx, |b, _| b.is_dirty()) => {
1503                                    let detach = self.rng.gen_bool(0.3);
1504                                    break ClientOperation::SaveBuffer {
1505                                        project_root_name,
1506                                        is_local,
1507                                        full_path,
1508                                        detach,
1509                                    };
1510                                }
1511                                // Edit the buffer
1512                                30..=69 => {
1513                                    let edits = buffer.read_with(cx, |buffer, _| {
1514                                        buffer.get_random_edits(&mut self.rng, 3)
1515                                    });
1516                                    break ClientOperation::EditBuffer {
1517                                        project_root_name,
1518                                        is_local,
1519                                        full_path,
1520                                        edits,
1521                                    };
1522                                }
1523                                // Make an LSP request
1524                                _ => {
1525                                    let offset = buffer.read_with(cx, |buffer, _| {
1526                                        buffer.clip_offset(
1527                                            self.rng.gen_range(0..=buffer.len()),
1528                                            language::Bias::Left,
1529                                        )
1530                                    });
1531                                    let detach = self.rng.gen();
1532                                    break ClientOperation::RequestLspDataInBuffer {
1533                                        project_root_name,
1534                                        full_path,
1535                                        offset,
1536                                        is_local,
1537                                        kind: match self.rng.gen_range(0..5_u32) {
1538                                            0 => LspRequestKind::Rename,
1539                                            1 => LspRequestKind::Highlights,
1540                                            2 => LspRequestKind::Definition,
1541                                            3 => LspRequestKind::CodeAction,
1542                                            4.. => LspRequestKind::Completion,
1543                                        },
1544                                        detach,
1545                                    };
1546                                }
1547                            }
1548                        }
1549
1550                        71..=80 => {
1551                            let query = self.rng.gen_range('a'..='z').to_string();
1552                            let detach = self.rng.gen_bool(0.3);
1553                            break ClientOperation::SearchProject {
1554                                project_root_name,
1555                                is_local,
1556                                query,
1557                                detach,
1558                            };
1559                        }
1560
1561                        // Open a buffer
1562                        81.. => {
1563                            let worktree = project.read_with(cx, |project, cx| {
1564                                project
1565                                    .worktrees(cx)
1566                                    .filter(|worktree| {
1567                                        let worktree = worktree.read(cx);
1568                                        worktree.is_visible()
1569                                            && worktree.entries(false).any(|e| e.is_file())
1570                                    })
1571                                    .choose(&mut self.rng)
1572                            });
1573                            let Some(worktree) = worktree else { continue };
1574                            let full_path = worktree.read_with(cx, |worktree, _| {
1575                                let entry = worktree
1576                                    .entries(false)
1577                                    .filter(|e| e.is_file())
1578                                    .choose(&mut self.rng)
1579                                    .unwrap();
1580                                if entry.path.as_ref() == Path::new("") {
1581                                    Path::new(worktree.root_name()).into()
1582                                } else {
1583                                    Path::new(worktree.root_name()).join(&entry.path)
1584                                }
1585                            });
1586                            break ClientOperation::OpenBuffer {
1587                                project_root_name,
1588                                is_local,
1589                                full_path,
1590                            };
1591                        }
1592                    }
1593                }
1594
1595                // Update a git index
1596                91..=95 => {
1597                    let repo_path = executor
1598                        .block(client.fs.directories())
1599                        .choose(&mut self.rng)
1600                        .unwrap()
1601                        .clone();
1602
1603                    let mut file_paths = executor
1604                        .block(client.fs.files())
1605                        .into_iter()
1606                        .filter(|path| path.starts_with(&repo_path))
1607                        .collect::<Vec<_>>();
1608                    let count = self.rng.gen_range(0..=file_paths.len());
1609                    file_paths.shuffle(&mut self.rng);
1610                    file_paths.truncate(count);
1611
1612                    let mut contents = Vec::new();
1613                    for abs_child_file_path in &file_paths {
1614                        let child_file_path = abs_child_file_path
1615                            .strip_prefix(&repo_path)
1616                            .unwrap()
1617                            .to_path_buf();
1618                        let new_base = Alphanumeric.sample_string(&mut self.rng, 16);
1619                        contents.push((child_file_path, new_base));
1620                    }
1621
1622                    break ClientOperation::WriteGitIndex {
1623                        repo_path,
1624                        contents,
1625                    };
1626                }
1627
1628                // Create a file or directory
1629                96.. => {
1630                    let is_dir = self.rng.gen::<bool>();
1631                    let mut path = cx
1632                        .background()
1633                        .block(client.fs.directories())
1634                        .choose(&mut self.rng)
1635                        .unwrap()
1636                        .clone();
1637                    path.push(gen_file_name(&mut self.rng));
1638                    if !is_dir {
1639                        path.set_extension("rs");
1640                    }
1641                    break ClientOperation::CreateFsEntry { path, is_dir };
1642                }
1643            }
1644        })
1645    }
1646
1647    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
1648        let user_ix = self
1649            .users
1650            .iter()
1651            .position(|user| user.user_id == user_id)
1652            .unwrap();
1653        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
1654        format!("dir-{user_id}-{root_id}")
1655    }
1656
1657    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
1658        let ix = self
1659            .users
1660            .iter()
1661            .position(|user| user.user_id == user_id)
1662            .unwrap();
1663        &mut self.users[ix]
1664    }
1665}
1666
1667async fn simulate_client(
1668    client: Rc<TestClient>,
1669    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<usize>,
1670    plan: Arc<Mutex<TestPlan>>,
1671    mut cx: TestAppContext,
1672) {
1673    // Setup language server
1674    let mut language = Language::new(
1675        LanguageConfig {
1676            name: "Rust".into(),
1677            path_suffixes: vec!["rs".to_string()],
1678            ..Default::default()
1679        },
1680        None,
1681    );
1682    let _fake_language_servers = language
1683        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
1684            name: "the-fake-language-server",
1685            capabilities: lsp::LanguageServer::full_capabilities(),
1686            initializer: Some(Box::new({
1687                let plan = plan.clone();
1688                let fs = client.fs.clone();
1689                move |fake_server: &mut FakeLanguageServer| {
1690                    fake_server.handle_request::<lsp::request::Completion, _, _>(
1691                        |_, _| async move {
1692                            Ok(Some(lsp::CompletionResponse::Array(vec![
1693                                lsp::CompletionItem {
1694                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
1695                                        range: lsp::Range::new(
1696                                            lsp::Position::new(0, 0),
1697                                            lsp::Position::new(0, 0),
1698                                        ),
1699                                        new_text: "the-new-text".to_string(),
1700                                    })),
1701                                    ..Default::default()
1702                                },
1703                            ])))
1704                        },
1705                    );
1706
1707                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
1708                        |_, _| async move {
1709                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
1710                                lsp::CodeAction {
1711                                    title: "the-code-action".to_string(),
1712                                    ..Default::default()
1713                                },
1714                            )]))
1715                        },
1716                    );
1717
1718                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
1719                        |params, _| async move {
1720                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
1721                                params.position,
1722                                params.position,
1723                            ))))
1724                        },
1725                    );
1726
1727                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
1728                        let fs = fs.clone();
1729                        let plan = plan.clone();
1730                        move |_, _| {
1731                            let fs = fs.clone();
1732                            let plan = plan.clone();
1733                            async move {
1734                                let files = fs.files().await;
1735                                let count = plan.lock().rng.gen_range::<usize, _>(1..3);
1736                                let files = (0..count)
1737                                    .map(|_| files.choose(&mut plan.lock().rng).unwrap())
1738                                    .collect::<Vec<_>>();
1739                                log::info!("LSP: Returning definitions in files {:?}", &files);
1740                                Ok(Some(lsp::GotoDefinitionResponse::Array(
1741                                    files
1742                                        .into_iter()
1743                                        .map(|file| lsp::Location {
1744                                            uri: lsp::Url::from_file_path(file).unwrap(),
1745                                            range: Default::default(),
1746                                        })
1747                                        .collect(),
1748                                )))
1749                            }
1750                        }
1751                    });
1752
1753                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
1754                        let plan = plan.clone();
1755                        move |_, _| {
1756                            let mut highlights = Vec::new();
1757                            let highlight_count = plan.lock().rng.gen_range(1..=5);
1758                            for _ in 0..highlight_count {
1759                                let start_row = plan.lock().rng.gen_range(0..100);
1760                                let start_column = plan.lock().rng.gen_range(0..100);
1761                                let start = PointUtf16::new(start_row, start_column);
1762                                let end_row = plan.lock().rng.gen_range(0..100);
1763                                let end_column = plan.lock().rng.gen_range(0..100);
1764                                let end = PointUtf16::new(end_row, end_column);
1765                                let range = if start > end { end..start } else { start..end };
1766                                highlights.push(lsp::DocumentHighlight {
1767                                    range: range_to_lsp(range.clone()),
1768                                    kind: Some(lsp::DocumentHighlightKind::READ),
1769                                });
1770                            }
1771                            highlights.sort_unstable_by_key(|highlight| {
1772                                (highlight.range.start, highlight.range.end)
1773                            });
1774                            async move { Ok(Some(highlights)) }
1775                        }
1776                    });
1777                }
1778            })),
1779            ..Default::default()
1780        }))
1781        .await;
1782    client.language_registry.add(Arc::new(language));
1783
1784    while let Some(batch_id) = operation_rx.next().await {
1785        let Some((operation, skipped)) = plan.lock().next_client_operation(&client, batch_id, &cx) else { break };
1786        match apply_client_operation(&client, operation, &mut cx).await {
1787            Err(error) => {
1788                log::error!("{} error: {}", client.username, error);
1789            }
1790            Ok(applied) => {
1791                if !applied {
1792                    skipped.store(true, SeqCst);
1793                }
1794            }
1795        }
1796        cx.background().simulate_random_delay().await;
1797    }
1798    log::info!("{}: done", client.username);
1799}
1800
1801fn buffer_for_full_path(
1802    buffers: &HashSet<ModelHandle<language::Buffer>>,
1803    full_path: &PathBuf,
1804    cx: &TestAppContext,
1805) -> Option<ModelHandle<language::Buffer>> {
1806    buffers
1807        .iter()
1808        .find(|buffer| {
1809            buffer.read_with(cx, |buffer, cx| {
1810                buffer.file().unwrap().full_path(cx) == *full_path
1811            })
1812        })
1813        .cloned()
1814}
1815
1816fn project_for_root_name(
1817    client: &TestClient,
1818    root_name: &str,
1819    cx: &TestAppContext,
1820) -> Option<ModelHandle<Project>> {
1821    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1822        return Some(client.local_projects()[ix].clone());
1823    }
1824    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1825        return Some(client.remote_projects()[ix].clone());
1826    }
1827    None
1828}
1829
1830fn project_ix_for_root_name(
1831    projects: &[ModelHandle<Project>],
1832    root_name: &str,
1833    cx: &TestAppContext,
1834) -> Option<usize> {
1835    projects.iter().position(|project| {
1836        project.read_with(cx, |project, cx| {
1837            let worktree = project.visible_worktrees(cx).next().unwrap();
1838            worktree.read(cx).root_name() == root_name
1839        })
1840    })
1841}
1842
1843fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1844    project.read_with(cx, |project, cx| {
1845        project
1846            .visible_worktrees(cx)
1847            .next()
1848            .unwrap()
1849            .read(cx)
1850            .root_name()
1851            .to_string()
1852    })
1853}
1854
1855fn project_path_for_full_path(
1856    project: &ModelHandle<Project>,
1857    full_path: &Path,
1858    cx: &TestAppContext,
1859) -> Option<ProjectPath> {
1860    let mut components = full_path.components();
1861    let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1862    let path = components.as_path().into();
1863    let worktree_id = project.read_with(cx, |project, cx| {
1864        project.worktrees(cx).find_map(|worktree| {
1865            let worktree = worktree.read(cx);
1866            if worktree.root_name() == root_name {
1867                Some(worktree.id())
1868            } else {
1869                None
1870            }
1871        })
1872    })?;
1873    Some(ProjectPath { worktree_id, path })
1874}
1875
1876async fn ensure_project_shared(
1877    project: &ModelHandle<Project>,
1878    client: &TestClient,
1879    cx: &mut TestAppContext,
1880) {
1881    let first_root_name = root_name_for_project(project, cx);
1882    let active_call = cx.read(ActiveCall::global);
1883    if active_call.read_with(cx, |call, _| call.room().is_some())
1884        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
1885    {
1886        match active_call
1887            .update(cx, |call, cx| call.share_project(project.clone(), cx))
1888            .await
1889        {
1890            Ok(project_id) => {
1891                log::info!(
1892                    "{}: shared project {} with id {}",
1893                    client.username,
1894                    first_root_name,
1895                    project_id
1896                );
1897            }
1898            Err(error) => {
1899                log::error!(
1900                    "{}: error sharing project {}: {:?}",
1901                    client.username,
1902                    first_root_name,
1903                    error
1904                );
1905            }
1906        }
1907    }
1908}
1909
1910fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1911    client
1912        .local_projects()
1913        .iter()
1914        .chain(client.remote_projects().iter())
1915        .choose(rng)
1916        .cloned()
1917}
1918
1919fn gen_file_name(rng: &mut StdRng) -> String {
1920    let mut name = String::new();
1921    for _ in 0..10 {
1922        let letter = rng.gen_range('a'..='z');
1923        name.push(letter);
1924    }
1925    name
1926}
1927
1928fn path_env_var(name: &str) -> Option<PathBuf> {
1929    let value = env::var(name).ok()?;
1930    let mut path = PathBuf::from(value);
1931    if path.is_relative() {
1932        let mut abs_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
1933        abs_path.pop();
1934        abs_path.pop();
1935        abs_path.push(path);
1936        path = abs_path
1937    }
1938    Some(path)
1939}