1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::{BTreeMap, HashSet};
  10use editor::Bias;
  11use fs::{FakeFs, Fs as _};
  12use futures::StreamExt as _;
  13use gpui::{executor::Deterministic, ModelHandle, Task, TestAppContext};
  14use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  15use lsp::FakeLanguageServer;
  16use parking_lot::Mutex;
  17use project::{search::SearchQuery, Project, ProjectPath};
  18use rand::{
  19    distributions::{Alphanumeric, DistString},
  20    prelude::*,
  21};
  22use serde::{Deserialize, Serialize};
  23use std::{
  24    env,
  25    ops::Range,
  26    path::{Path, PathBuf},
  27    rc::Rc,
  28    sync::{
  29        atomic::{AtomicBool, Ordering::SeqCst},
  30        Arc,
  31    },
  32};
  33use util::ResultExt;
  34
  35#[gpui::test(iterations = 100)]
  36async fn test_random_collaboration(
  37    cx: &mut TestAppContext,
  38    deterministic: Arc<Deterministic>,
  39    rng: StdRng,
  40) {
  41    deterministic.forbid_parking();
  42
  43    let max_peers = env::var("MAX_PEERS")
  44        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  45        .unwrap_or(3);
  46    let max_operations = env::var("OPERATIONS")
  47        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  48        .unwrap_or(10);
  49
  50    let plan_load_path = path_env_var("LOAD_PLAN");
  51    let plan_save_path = path_env_var("SAVE_PLAN");
  52
  53    let mut server = TestServer::start(&deterministic).await;
  54    let db = server.app_state.db.clone();
  55
  56    let mut users = Vec::new();
  57    for ix in 0..max_peers {
  58        let username = format!("user-{}", ix + 1);
  59        let user_id = db
  60            .create_user(
  61                &format!("{username}@example.com"),
  62                false,
  63                NewUserParams {
  64                    github_login: username.clone(),
  65                    github_user_id: (ix + 1) as i32,
  66                    invite_count: 0,
  67                },
  68            )
  69            .await
  70            .unwrap()
  71            .user_id;
  72        users.push(UserTestPlan {
  73            user_id,
  74            username,
  75            online: false,
  76            next_root_id: 0,
  77            operation_ix: 0,
  78        });
  79    }
  80
  81    for (ix, user_a) in users.iter().enumerate() {
  82        for user_b in &users[ix + 1..] {
  83            server
  84                .app_state
  85                .db
  86                .send_contact_request(user_a.user_id, user_b.user_id)
  87                .await
  88                .unwrap();
  89            server
  90                .app_state
  91                .db
  92                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  93                .await
  94                .unwrap();
  95        }
  96    }
  97
  98    let plan = Arc::new(Mutex::new(TestPlan::new(rng, users, max_operations)));
  99
 100    if let Some(path) = &plan_load_path {
 101        eprintln!("loaded plan from path {:?}", path);
 102        plan.lock().load(path);
 103    }
 104
 105    let mut clients = Vec::new();
 106    let mut client_tasks = Vec::new();
 107    let mut operation_channels = Vec::new();
 108    let mut next_entity_id = 100000;
 109
 110    loop {
 111        let Some((next_operation, skipped)) = plan.lock().next_server_operation(&clients) else { break };
 112        let applied = apply_server_operation(
 113            deterministic.clone(),
 114            &mut server,
 115            &mut clients,
 116            &mut client_tasks,
 117            &mut operation_channels,
 118            &mut next_entity_id,
 119            plan.clone(),
 120            next_operation,
 121            cx,
 122        )
 123        .await;
 124        if !applied {
 125            skipped.store(false, SeqCst);
 126        }
 127    }
 128
 129    drop(operation_channels);
 130    deterministic.start_waiting();
 131    futures::future::join_all(client_tasks).await;
 132    deterministic.finish_waiting();
 133    deterministic.run_until_parked();
 134
 135    if let Some(path) = &plan_save_path {
 136        eprintln!("saved test plan to path {:?}", path);
 137        plan.lock().save(path);
 138    }
 139
 140    for (client, client_cx) in &clients {
 141        for guest_project in client.remote_projects().iter() {
 142            guest_project.read_with(client_cx, |guest_project, cx| {
 143                let host_project = clients.iter().find_map(|(client, cx)| {
 144                    let project = client
 145                        .local_projects()
 146                        .iter()
 147                        .find(|host_project| {
 148                            host_project.read_with(cx, |host_project, _| {
 149                                host_project.remote_id() == guest_project.remote_id()
 150                            })
 151                        })?
 152                        .clone();
 153                    Some((project, cx))
 154                });
 155
 156                if !guest_project.is_read_only() {
 157                    if let Some((host_project, host_cx)) = host_project {
 158                        let host_worktree_snapshots =
 159                            host_project.read_with(host_cx, |host_project, cx| {
 160                                host_project
 161                                    .worktrees(cx)
 162                                    .map(|worktree| {
 163                                        let worktree = worktree.read(cx);
 164                                        (worktree.id(), worktree.snapshot())
 165                                    })
 166                                    .collect::<BTreeMap<_, _>>()
 167                            });
 168                        let guest_worktree_snapshots = guest_project
 169                            .worktrees(cx)
 170                            .map(|worktree| {
 171                                let worktree = worktree.read(cx);
 172                                (worktree.id(), worktree.snapshot())
 173                            })
 174                            .collect::<BTreeMap<_, _>>();
 175
 176                        assert_eq!(
 177                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 178                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 179                            "{} has different worktrees than the host",
 180                            client.username
 181                        );
 182
 183                        for (id, host_snapshot) in &host_worktree_snapshots {
 184                            let guest_snapshot = &guest_worktree_snapshots[id];
 185                            assert_eq!(
 186                                guest_snapshot.root_name(),
 187                                host_snapshot.root_name(),
 188                                "{} has different root name than the host for worktree {}",
 189                                client.username,
 190                                id
 191                            );
 192                            assert_eq!(
 193                                guest_snapshot.abs_path(),
 194                                host_snapshot.abs_path(),
 195                                "{} has different abs path than the host for worktree {}",
 196                                client.username,
 197                                id
 198                            );
 199                            assert_eq!(
 200                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 201                                host_snapshot.entries(false).collect::<Vec<_>>(),
 202                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 203                                client.username,
 204                                id,
 205                                host_snapshot.abs_path(),
 206                                host_project.read_with(host_cx, |project, _| project.remote_id())
 207                            );
 208                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 209                        }
 210                    }
 211                }
 212
 213                guest_project.check_invariants(cx);
 214            });
 215        }
 216
 217        let buffers = client.buffers().clone();
 218        for (guest_project, guest_buffers) in &buffers {
 219            let project_id = if guest_project.read_with(client_cx, |project, _| {
 220                project.is_local() || project.is_read_only()
 221            }) {
 222                continue;
 223            } else {
 224                guest_project
 225                    .read_with(client_cx, |project, _| project.remote_id())
 226                    .unwrap()
 227            };
 228            let guest_user_id = client.user_id().unwrap();
 229
 230            let host_project = clients.iter().find_map(|(client, cx)| {
 231                let project = client
 232                    .local_projects()
 233                    .iter()
 234                    .find(|host_project| {
 235                        host_project.read_with(cx, |host_project, _| {
 236                            host_project.remote_id() == Some(project_id)
 237                        })
 238                    })?
 239                    .clone();
 240                Some((client.user_id().unwrap(), project, cx))
 241            });
 242
 243            let (host_user_id, host_project, host_cx) =
 244                if let Some((host_user_id, host_project, host_cx)) = host_project {
 245                    (host_user_id, host_project, host_cx)
 246                } else {
 247                    continue;
 248                };
 249
 250            for guest_buffer in guest_buffers {
 251                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 252                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 253                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 254                        panic!(
 255                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 256                            client.username,
 257                            client.peer_id(),
 258                            buffer_id
 259                        )
 260                    })
 261                });
 262                let path = host_buffer
 263                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 264
 265                assert_eq!(
 266                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 267                    0,
 268                    "{}, buffer {}, path {:?} has deferred operations",
 269                    client.username,
 270                    buffer_id,
 271                    path,
 272                );
 273                assert_eq!(
 274                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 275                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 276                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 277                    client.username,
 278                    buffer_id,
 279                    path
 280                );
 281
 282                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 283                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 284                match (host_file, guest_file) {
 285                    (Some(host_file), Some(guest_file)) => {
 286                        assert_eq!(guest_file.path(), host_file.path());
 287                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 288                        assert_eq!(
 289                            guest_file.mtime(),
 290                            host_file.mtime(),
 291                            "guest {} mtime does not match host {} for path {:?} in project {}",
 292                            guest_user_id,
 293                            host_user_id,
 294                            guest_file.path(),
 295                            project_id,
 296                        );
 297                    }
 298                    (None, None) => {}
 299                    (None, _) => panic!("host's file is None, guest's isn't"),
 300                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 301                }
 302
 303                let host_diff_base =
 304                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 305                let guest_diff_base = guest_buffer
 306                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 307                assert_eq!(guest_diff_base, host_diff_base);
 308            }
 309        }
 310    }
 311
 312    for (client, mut cx) in clients {
 313        cx.update(|cx| {
 314            cx.clear_globals();
 315            drop(client);
 316        });
 317    }
 318}
 319
 320async fn apply_server_operation(
 321    deterministic: Arc<Deterministic>,
 322    server: &mut TestServer,
 323    clients: &mut Vec<(Rc<TestClient>, TestAppContext)>,
 324    client_tasks: &mut Vec<Task<()>>,
 325    operation_channels: &mut Vec<futures::channel::mpsc::UnboundedSender<usize>>,
 326    next_entity_id: &mut usize,
 327    plan: Arc<Mutex<TestPlan>>,
 328    operation: Operation,
 329    cx: &mut TestAppContext,
 330) -> bool {
 331    match operation {
 332        Operation::AddConnection { user_id } => {
 333            let username;
 334            {
 335                let mut plan = plan.lock();
 336                let mut user = plan.user(user_id);
 337                if user.online {
 338                    return false;
 339                }
 340                user.online = true;
 341                username = user.username.clone();
 342            };
 343            log::info!("Adding new connection for {}", username);
 344            *next_entity_id += 100000;
 345            let mut client_cx = TestAppContext::new(
 346                cx.foreground_platform(),
 347                cx.platform(),
 348                deterministic.build_foreground(*next_entity_id),
 349                deterministic.build_background(),
 350                cx.font_cache(),
 351                cx.leak_detector(),
 352                *next_entity_id,
 353                cx.function_name.clone(),
 354            );
 355
 356            let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 357            let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 358            operation_channels.push(operation_tx);
 359            clients.push((client.clone(), client_cx.clone()));
 360            client_tasks.push(client_cx.foreground().spawn(simulate_client(
 361                client,
 362                operation_rx,
 363                plan.clone(),
 364                client_cx,
 365            )));
 366
 367            log::info!("Added connection for {}", username);
 368        }
 369
 370        Operation::RemoveConnection { user_id } => {
 371            log::info!("Simulating full disconnection of user {}", user_id);
 372            let client_ix = clients
 373                .iter()
 374                .position(|(client, cx)| client.current_user_id(cx) == user_id);
 375            let Some(client_ix) = client_ix else { return false };
 376            let user_connection_ids = server
 377                .connection_pool
 378                .lock()
 379                .user_connection_ids(user_id)
 380                .collect::<Vec<_>>();
 381            assert_eq!(user_connection_ids.len(), 1);
 382            let removed_peer_id = user_connection_ids[0].into();
 383            let (client, mut client_cx) = clients.remove(client_ix);
 384            let client_task = client_tasks.remove(client_ix);
 385            operation_channels.remove(client_ix);
 386            server.forbid_connections();
 387            server.disconnect_client(removed_peer_id);
 388            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 389            deterministic.start_waiting();
 390            log::info!("Waiting for user {} to exit...", user_id);
 391            client_task.await;
 392            deterministic.finish_waiting();
 393            server.allow_connections();
 394
 395            for project in client.remote_projects().iter() {
 396                project.read_with(&client_cx, |project, _| {
 397                    assert!(
 398                        project.is_read_only(),
 399                        "project {:?} should be read only",
 400                        project.remote_id()
 401                    )
 402                });
 403            }
 404
 405            for (client, cx) in clients {
 406                let contacts = server
 407                    .app_state
 408                    .db
 409                    .get_contacts(client.current_user_id(cx))
 410                    .await
 411                    .unwrap();
 412                let pool = server.connection_pool.lock();
 413                for contact in contacts {
 414                    if let db::Contact::Accepted { user_id: id, .. } = contact {
 415                        if pool.is_user_online(id) {
 416                            assert_ne!(
 417                                id, user_id,
 418                                "removed client is still a contact of another peer"
 419                            );
 420                        }
 421                    }
 422                }
 423            }
 424
 425            log::info!("{} removed", client.username);
 426            plan.lock().user(user_id).online = false;
 427            client_cx.update(|cx| {
 428                cx.clear_globals();
 429                drop(client);
 430            });
 431        }
 432
 433        Operation::BounceConnection { user_id } => {
 434            log::info!("Simulating temporary disconnection of user {}", user_id);
 435            let user_connection_ids = server
 436                .connection_pool
 437                .lock()
 438                .user_connection_ids(user_id)
 439                .collect::<Vec<_>>();
 440            if user_connection_ids.is_empty() {
 441                return false;
 442            }
 443            assert_eq!(user_connection_ids.len(), 1);
 444            let peer_id = user_connection_ids[0].into();
 445            server.disconnect_client(peer_id);
 446            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 447        }
 448
 449        Operation::RestartServer => {
 450            log::info!("Simulating server restart");
 451            server.reset().await;
 452            deterministic.advance_clock(RECEIVE_TIMEOUT);
 453            server.start().await.unwrap();
 454            deterministic.advance_clock(CLEANUP_TIMEOUT);
 455            let environment = &server.app_state.config.zed_environment;
 456            let stale_room_ids = server
 457                .app_state
 458                .db
 459                .stale_room_ids(environment, server.id())
 460                .await
 461                .unwrap();
 462            assert_eq!(stale_room_ids, vec![]);
 463        }
 464
 465        Operation::MutateClients {
 466            user_ids,
 467            batch_id,
 468            quiesce,
 469        } => {
 470            let mut applied = false;
 471            for user_id in user_ids {
 472                let client_ix = clients
 473                    .iter()
 474                    .position(|(client, cx)| client.current_user_id(cx) == user_id);
 475                let Some(client_ix) = client_ix else { continue };
 476                applied = true;
 477                if let Err(err) = operation_channels[client_ix].unbounded_send(batch_id) {
 478                    // panic!("error signaling user {}, client {}", user_id, client_ix);
 479                }
 480            }
 481
 482            if quiesce && applied {
 483                deterministic.run_until_parked();
 484            }
 485
 486            return applied;
 487        }
 488    }
 489    true
 490}
 491
 492async fn apply_client_operation(
 493    client: &TestClient,
 494    operation: ClientOperation,
 495    cx: &mut TestAppContext,
 496) -> Result<bool> {
 497    match operation {
 498        ClientOperation::AcceptIncomingCall => {
 499            let active_call = cx.read(ActiveCall::global);
 500            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 501                return Ok(false);
 502            }
 503
 504            log::info!("{}: accepting incoming call", client.username);
 505            active_call
 506                .update(cx, |call, cx| call.accept_incoming(cx))
 507                .await?;
 508        }
 509
 510        ClientOperation::RejectIncomingCall => {
 511            let active_call = cx.read(ActiveCall::global);
 512            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 513                return Ok(false);
 514            }
 515
 516            log::info!("{}: declining incoming call", client.username);
 517            active_call.update(cx, |call, _| call.decline_incoming())?;
 518        }
 519
 520        ClientOperation::LeaveCall => {
 521            let active_call = cx.read(ActiveCall::global);
 522            if active_call.read_with(cx, |call, _| call.room().is_none()) {
 523                return Ok(false);
 524            }
 525
 526            log::info!("{}: hanging up", client.username);
 527            active_call.update(cx, |call, cx| call.hang_up(cx))?;
 528        }
 529
 530        ClientOperation::InviteContactToCall { user_id } => {
 531            let active_call = cx.read(ActiveCall::global);
 532
 533            log::info!("{}: inviting {}", client.username, user_id,);
 534            active_call
 535                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 536                .await
 537                .log_err();
 538        }
 539
 540        ClientOperation::OpenLocalProject { first_root_name } => {
 541            log::info!(
 542                "{}: opening local project at {:?}",
 543                client.username,
 544                first_root_name
 545            );
 546
 547            let root_path = Path::new("/").join(&first_root_name);
 548            client.fs.create_dir(&root_path).await.unwrap();
 549            client
 550                .fs
 551                .create_file(&root_path.join("main.rs"), Default::default())
 552                .await
 553                .unwrap();
 554            let project = client.build_local_project(root_path, cx).await.0;
 555            ensure_project_shared(&project, client, cx).await;
 556            client.local_projects_mut().push(project.clone());
 557        }
 558
 559        ClientOperation::AddWorktreeToProject {
 560            project_root_name,
 561            new_root_path,
 562        } => {
 563            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 564                return Ok(false)
 565            };
 566
 567            log::info!(
 568                "{}: finding/creating local worktree at {:?} to project with root path {}",
 569                client.username,
 570                new_root_path,
 571                project_root_name
 572            );
 573
 574            ensure_project_shared(&project, client, cx).await;
 575            if !client.fs.paths().await.contains(&new_root_path) {
 576                client.fs.create_dir(&new_root_path).await.unwrap();
 577            }
 578            project
 579                .update(cx, |project, cx| {
 580                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 581                })
 582                .await
 583                .unwrap();
 584        }
 585
 586        ClientOperation::CloseRemoteProject { project_root_name } => {
 587            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 588                return Ok(false)
 589            };
 590
 591            log::info!(
 592                "{}: closing remote project with root path {}",
 593                client.username,
 594                project_root_name,
 595            );
 596
 597            let ix = client
 598                .remote_projects()
 599                .iter()
 600                .position(|p| p == &project)
 601                .unwrap();
 602            cx.update(|_| {
 603                client.remote_projects_mut().remove(ix);
 604                client.buffers().retain(|project, _| project != project);
 605                drop(project);
 606            });
 607        }
 608
 609        ClientOperation::OpenRemoteProject {
 610            host_id,
 611            first_root_name,
 612        } => {
 613            let active_call = cx.read(ActiveCall::global);
 614            let project = active_call.update(cx, |call, cx| {
 615                let room = call.room().cloned()?;
 616                let participant = room
 617                    .read(cx)
 618                    .remote_participants()
 619                    .get(&host_id.to_proto())?;
 620                let project_id = participant
 621                    .projects
 622                    .iter()
 623                    .find(|project| project.worktree_root_names[0] == first_root_name)?
 624                    .id;
 625                Some(room.update(cx, |room, cx| {
 626                    room.join_project(
 627                        project_id,
 628                        client.language_registry.clone(),
 629                        FakeFs::new(cx.background().clone()),
 630                        cx,
 631                    )
 632                }))
 633            });
 634            let Some(project) = project else {
 635                return Ok(false)
 636            };
 637
 638            log::info!(
 639                "{}: joining remote project of user {}, root name {}",
 640                client.username,
 641                host_id,
 642                first_root_name,
 643            );
 644
 645            let project = project.await?;
 646            client.remote_projects_mut().push(project.clone());
 647        }
 648
 649        ClientOperation::CreateWorktreeEntry {
 650            project_root_name,
 651            is_local,
 652            full_path,
 653            is_dir,
 654        } => {
 655            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 656                return Ok(false);
 657            };
 658            let Some(project_path) = project_path_for_full_path(&project, &full_path, cx) else {
 659                return Ok(false);
 660            };
 661
 662            log::info!(
 663                "{}: creating {} at path {:?} in {} project {}",
 664                client.username,
 665                if is_dir { "dir" } else { "file" },
 666                full_path,
 667                if is_local { "local" } else { "remote" },
 668                project_root_name,
 669            );
 670
 671            ensure_project_shared(&project, client, cx).await;
 672            project
 673                .update(cx, |p, cx| p.create_entry(project_path, is_dir, cx))
 674                .unwrap()
 675                .await?;
 676        }
 677
 678        ClientOperation::OpenBuffer {
 679            project_root_name,
 680            is_local,
 681            full_path,
 682        } => {
 683            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 684                return Ok(false);
 685            };
 686            let Some(project_path) = project_path_for_full_path(&project, &full_path, cx) else {
 687                return Ok(false);
 688            };
 689
 690            log::info!(
 691                "{}: opening buffer {:?} in {} project {}",
 692                client.username,
 693                full_path,
 694                if is_local { "local" } else { "remote" },
 695                project_root_name,
 696            );
 697
 698            ensure_project_shared(&project, client, cx).await;
 699            let buffer = project
 700                .update(cx, |project, cx| project.open_buffer(project_path, cx))
 701                .await?;
 702            client.buffers_for_project(&project).insert(buffer);
 703        }
 704
 705        ClientOperation::EditBuffer {
 706            project_root_name,
 707            is_local,
 708            full_path,
 709            edits,
 710        } => {
 711            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 712                return Ok(false);
 713            };
 714            let Some(buffer) =
 715                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 716                    return Ok(false);
 717                };
 718
 719            log::info!(
 720                "{}: editing buffer {:?} in {} project {} with {:?}",
 721                client.username,
 722                full_path,
 723                if is_local { "local" } else { "remote" },
 724                project_root_name,
 725                edits
 726            );
 727
 728            ensure_project_shared(&project, client, cx).await;
 729            buffer.update(cx, |buffer, cx| {
 730                let snapshot = buffer.snapshot();
 731                buffer.edit(
 732                    edits.into_iter().map(|(range, text)| {
 733                        let start = snapshot.clip_offset(range.start, Bias::Left);
 734                        let end = snapshot.clip_offset(range.end, Bias::Right);
 735                        (start..end, text)
 736                    }),
 737                    None,
 738                    cx,
 739                );
 740            });
 741        }
 742
 743        ClientOperation::CloseBuffer {
 744            project_root_name,
 745            is_local,
 746            full_path,
 747        } => {
 748            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 749                return Ok(false);
 750            };
 751            let Some(buffer) =
 752                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 753                    return Ok(false);
 754                };
 755
 756            log::info!(
 757                "{}: closing buffer {:?} in {} project {}",
 758                client.username,
 759                full_path,
 760                if is_local { "local" } else { "remote" },
 761                project_root_name
 762            );
 763
 764            ensure_project_shared(&project, client, cx).await;
 765            cx.update(|_| {
 766                client.buffers_for_project(&project).remove(&buffer);
 767                drop(buffer);
 768            });
 769        }
 770
 771        ClientOperation::SaveBuffer {
 772            project_root_name,
 773            is_local,
 774            full_path,
 775            detach,
 776        } => {
 777            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 778                return Ok(false);
 779            };
 780            let Some(buffer) =
 781                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 782                    return Ok(false);
 783                };
 784
 785            log::info!(
 786                "{}: saving buffer {:?} in {} project {}{}",
 787                client.username,
 788                full_path,
 789                if is_local { "local" } else { "remote" },
 790                project_root_name,
 791                if detach { ", detaching" } else { ", awaiting" }
 792            );
 793
 794            ensure_project_shared(&project, client, cx).await;
 795            let (requested_version, save) =
 796                buffer.update(cx, |buffer, cx| (buffer.version(), buffer.save(cx)));
 797            let save = cx.background().spawn(async move {
 798                let (saved_version, _, _) = save
 799                    .await
 800                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 801                assert!(saved_version.observed_all(&requested_version));
 802                anyhow::Ok(())
 803            });
 804            if detach {
 805                log::info!("{}: detaching save request", client.username);
 806                cx.update(|cx| save.detach_and_log_err(cx));
 807            } else {
 808                save.await?;
 809            }
 810        }
 811
 812        ClientOperation::RequestLspDataInBuffer {
 813            project_root_name,
 814            is_local,
 815            full_path,
 816            offset,
 817            kind,
 818            detach,
 819        } => {
 820            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 821                return Ok(false);
 822            };
 823            let Some(buffer) =
 824                buffer_for_full_path(&*client.buffers_for_project(&project), &full_path, cx) else {
 825                    return Ok(false);
 826                };
 827
 828            log::info!(
 829                "{}: request LSP {:?} for buffer {:?} in {} project {}{}",
 830                client.username,
 831                kind,
 832                full_path,
 833                if is_local { "local" } else { "remote" },
 834                project_root_name,
 835                if detach { ", detaching" } else { ", awaiting" }
 836            );
 837
 838            let offset = buffer.read_with(cx, |b, _| b.clip_offset(offset, Bias::Left));
 839            let request = match kind {
 840                LspRequestKind::Rename => cx.spawn(|mut cx| async move {
 841                    project
 842                        .update(&mut cx, |p, cx| p.prepare_rename(buffer, offset, cx))
 843                        .await?;
 844                    anyhow::Ok(())
 845                }),
 846                LspRequestKind::Completion => cx.spawn(|mut cx| async move {
 847                    project
 848                        .update(&mut cx, |p, cx| p.completions(&buffer, offset, cx))
 849                        .await?;
 850                    Ok(())
 851                }),
 852                LspRequestKind::CodeAction => cx.spawn(|mut cx| async move {
 853                    project
 854                        .update(&mut cx, |p, cx| p.code_actions(&buffer, offset..offset, cx))
 855                        .await?;
 856                    Ok(())
 857                }),
 858                LspRequestKind::Definition => cx.spawn(|mut cx| async move {
 859                    project
 860                        .update(&mut cx, |p, cx| p.definition(&buffer, offset, cx))
 861                        .await?;
 862                    Ok(())
 863                }),
 864                LspRequestKind::Highlights => cx.spawn(|mut cx| async move {
 865                    project
 866                        .update(&mut cx, |p, cx| p.document_highlights(&buffer, offset, cx))
 867                        .await?;
 868                    Ok(())
 869                }),
 870            };
 871            if detach {
 872                request.detach();
 873            } else {
 874                request.await?;
 875            }
 876        }
 877
 878        ClientOperation::SearchProject {
 879            project_root_name,
 880            is_local,
 881            query,
 882            detach,
 883        } => {
 884            let Some(project) = project_for_root_name(client, &project_root_name, cx) else {
 885                return Ok(false);
 886            };
 887
 888            log::info!(
 889                "{}: search {} project {} for {:?}{}",
 890                client.username,
 891                if is_local { "local" } else { "remote" },
 892                project_root_name,
 893                query,
 894                if detach { ", detaching" } else { ", awaiting" }
 895            );
 896
 897            let search = project.update(cx, |project, cx| {
 898                project.search(SearchQuery::text(query, false, false), cx)
 899            });
 900            let search = cx.background().spawn(async move {
 901                search
 902                    .await
 903                    .map_err(|err| anyhow!("search request failed: {:?}", err))
 904            });
 905            if detach {
 906                log::info!("{}: detaching save request", client.username);
 907                cx.update(|cx| search.detach_and_log_err(cx));
 908            } else {
 909                search.await?;
 910            }
 911        }
 912
 913        ClientOperation::CreateFsEntry { path, is_dir } => {
 914            if client.fs.metadata(&path.parent().unwrap()).await?.is_none() {
 915                return Ok(false);
 916            }
 917
 918            log::info!(
 919                "{}: creating {} at {:?}",
 920                client.username,
 921                if is_dir { "dir" } else { "file" },
 922                path
 923            );
 924
 925            if is_dir {
 926                client.fs.create_dir(&path).await.unwrap();
 927            } else {
 928                client
 929                    .fs
 930                    .create_file(&path, Default::default())
 931                    .await
 932                    .unwrap();
 933            }
 934        }
 935
 936        ClientOperation::WriteGitIndex {
 937            repo_path,
 938            contents,
 939        } => {
 940            if !client
 941                .fs
 942                .metadata(&repo_path)
 943                .await?
 944                .map_or(false, |m| m.is_dir)
 945            {
 946                return Ok(false);
 947            }
 948
 949            log::info!(
 950                "{}: writing git index for repo {:?}: {:?}",
 951                client.username,
 952                repo_path,
 953                contents
 954            );
 955
 956            let dot_git_dir = repo_path.join(".git");
 957            let contents = contents
 958                .iter()
 959                .map(|(path, contents)| (path.as_path(), contents.clone()))
 960                .collect::<Vec<_>>();
 961            if client.fs.metadata(&dot_git_dir).await?.is_none() {
 962                client.fs.create_dir(&dot_git_dir).await?;
 963            }
 964            client.fs.set_index_for_repo(&dot_git_dir, &contents).await;
 965        }
 966    }
 967    Ok(true)
 968}
 969
 970struct TestPlan {
 971    rng: StdRng,
 972    replay: bool,
 973    stored_operations: Vec<(StoredOperation, Arc<AtomicBool>)>,
 974    max_operations: usize,
 975    operation_ix: usize,
 976    users: Vec<UserTestPlan>,
 977    next_batch_id: usize,
 978    allow_server_restarts: bool,
 979    allow_client_reconnection: bool,
 980    allow_client_disconnection: bool,
 981}
 982
 983struct UserTestPlan {
 984    user_id: UserId,
 985    username: String,
 986    next_root_id: usize,
 987    operation_ix: usize,
 988    online: bool,
 989}
 990
 991#[derive(Clone, Debug, Serialize, Deserialize)]
 992#[serde(untagged)]
 993enum StoredOperation {
 994    Server(Operation),
 995    Client {
 996        user_id: UserId,
 997        batch_id: usize,
 998        operation: ClientOperation,
 999    },
1000}
1001
1002#[derive(Clone, Debug, Serialize, Deserialize)]
1003enum Operation {
1004    AddConnection {
1005        user_id: UserId,
1006    },
1007    RemoveConnection {
1008        user_id: UserId,
1009    },
1010    BounceConnection {
1011        user_id: UserId,
1012    },
1013    RestartServer,
1014    MutateClients {
1015        batch_id: usize,
1016        #[serde(skip_serializing)]
1017        #[serde(skip_deserializing)]
1018        user_ids: Vec<UserId>,
1019        quiesce: bool,
1020    },
1021}
1022
1023#[derive(Clone, Debug, Serialize, Deserialize)]
1024enum ClientOperation {
1025    AcceptIncomingCall,
1026    RejectIncomingCall,
1027    LeaveCall,
1028    InviteContactToCall {
1029        user_id: UserId,
1030    },
1031    OpenLocalProject {
1032        first_root_name: String,
1033    },
1034    OpenRemoteProject {
1035        host_id: UserId,
1036        first_root_name: String,
1037    },
1038    AddWorktreeToProject {
1039        project_root_name: String,
1040        new_root_path: PathBuf,
1041    },
1042    CloseRemoteProject {
1043        project_root_name: String,
1044    },
1045    OpenBuffer {
1046        project_root_name: String,
1047        is_local: bool,
1048        full_path: PathBuf,
1049    },
1050    SearchProject {
1051        project_root_name: String,
1052        is_local: bool,
1053        query: String,
1054        detach: bool,
1055    },
1056    EditBuffer {
1057        project_root_name: String,
1058        is_local: bool,
1059        full_path: PathBuf,
1060        edits: Vec<(Range<usize>, Arc<str>)>,
1061    },
1062    CloseBuffer {
1063        project_root_name: String,
1064        is_local: bool,
1065        full_path: PathBuf,
1066    },
1067    SaveBuffer {
1068        project_root_name: String,
1069        is_local: bool,
1070        full_path: PathBuf,
1071        detach: bool,
1072    },
1073    RequestLspDataInBuffer {
1074        project_root_name: String,
1075        is_local: bool,
1076        full_path: PathBuf,
1077        offset: usize,
1078        kind: LspRequestKind,
1079        detach: bool,
1080    },
1081    CreateWorktreeEntry {
1082        project_root_name: String,
1083        is_local: bool,
1084        full_path: PathBuf,
1085        is_dir: bool,
1086    },
1087    CreateFsEntry {
1088        path: PathBuf,
1089        is_dir: bool,
1090    },
1091    WriteGitIndex {
1092        repo_path: PathBuf,
1093        contents: Vec<(PathBuf, String)>,
1094    },
1095}
1096
1097#[derive(Clone, Debug, Serialize, Deserialize)]
1098enum LspRequestKind {
1099    Rename,
1100    Completion,
1101    CodeAction,
1102    Definition,
1103    Highlights,
1104}
1105
1106impl TestPlan {
1107    fn new(mut rng: StdRng, users: Vec<UserTestPlan>, max_operations: usize) -> Self {
1108        Self {
1109            replay: false,
1110            allow_server_restarts: rng.gen_bool(0.7),
1111            allow_client_reconnection: rng.gen_bool(0.7),
1112            allow_client_disconnection: rng.gen_bool(0.1),
1113            stored_operations: Vec::new(),
1114            operation_ix: 0,
1115            next_batch_id: 0,
1116            max_operations,
1117            users,
1118            rng,
1119        }
1120    }
1121
1122    fn load(&mut self, path: &Path) {
1123        let json = std::fs::read_to_string(path).unwrap();
1124        self.replay = true;
1125        let stored_operations: Vec<StoredOperation> = serde_json::from_str(&json).unwrap();
1126        self.stored_operations = stored_operations
1127            .iter()
1128            .cloned()
1129            .enumerate()
1130            .map(|(i, mut operation)| {
1131                if let StoredOperation::Server(Operation::MutateClients {
1132                    batch_id: current_batch_id,
1133                    user_ids,
1134                    ..
1135                }) = &mut operation
1136                {
1137                    assert!(user_ids.is_empty());
1138                    user_ids.extend(stored_operations[i + 1..].iter().filter_map(|operation| {
1139                        if let StoredOperation::Client {
1140                            user_id, batch_id, ..
1141                        } = operation
1142                        {
1143                            if batch_id == current_batch_id {
1144                                return Some(user_id);
1145                            }
1146                        }
1147                        None
1148                    }));
1149                    user_ids.sort_unstable();
1150                }
1151                (operation, Arc::new(AtomicBool::new(false)))
1152            })
1153            .collect()
1154    }
1155
1156    fn save(&mut self, path: &Path) {
1157        // Format each operation as one line
1158        let mut json = Vec::new();
1159        json.push(b'[');
1160        for (operation, skipped) in &self.stored_operations {
1161            if skipped.load(SeqCst) {
1162                continue;
1163            }
1164            if json.len() > 1 {
1165                json.push(b',');
1166            }
1167            json.extend_from_slice(b"\n  ");
1168            serde_json::to_writer(&mut json, operation).unwrap();
1169        }
1170        json.extend_from_slice(b"\n]\n");
1171        std::fs::write(path, &json).unwrap();
1172    }
1173
1174    fn next_server_operation(
1175        &mut self,
1176        clients: &[(Rc<TestClient>, TestAppContext)],
1177    ) -> Option<(Operation, Arc<AtomicBool>)> {
1178        if self.replay {
1179            while let Some(stored_operation) = self.stored_operations.get(self.operation_ix) {
1180                self.operation_ix += 1;
1181                if let (StoredOperation::Server(operation), skipped) = stored_operation {
1182                    return Some((operation.clone(), skipped.clone()));
1183                }
1184            }
1185            None
1186        } else {
1187            let operation = self.generate_server_operation(clients)?;
1188            let skipped = Arc::new(AtomicBool::new(false));
1189            self.stored_operations
1190                .push((StoredOperation::Server(operation.clone()), skipped.clone()));
1191            Some((operation, skipped))
1192        }
1193    }
1194
1195    fn next_client_operation(
1196        &mut self,
1197        client: &TestClient,
1198        current_batch_id: usize,
1199        cx: &TestAppContext,
1200    ) -> Option<(ClientOperation, Arc<AtomicBool>)> {
1201        let current_user_id = client.current_user_id(cx);
1202        let user_ix = self
1203            .users
1204            .iter()
1205            .position(|user| user.user_id == current_user_id)
1206            .unwrap();
1207        let user_plan = &mut self.users[user_ix];
1208
1209        if self.replay {
1210            while let Some(stored_operation) = self.stored_operations.get(user_plan.operation_ix) {
1211                user_plan.operation_ix += 1;
1212                if let (
1213                    StoredOperation::Client {
1214                        user_id, operation, ..
1215                    },
1216                    skipped,
1217                ) = stored_operation
1218                {
1219                    if user_id == &current_user_id {
1220                        return Some((operation.clone(), skipped.clone()));
1221                    }
1222                }
1223            }
1224            None
1225        } else {
1226            let operation = self.generate_client_operation(current_user_id, client, cx)?;
1227            let skipped = Arc::new(AtomicBool::new(false));
1228            self.stored_operations.push((
1229                StoredOperation::Client {
1230                    user_id: current_user_id,
1231                    batch_id: current_batch_id,
1232                    operation: operation.clone(),
1233                },
1234                skipped.clone(),
1235            ));
1236            Some((operation, skipped))
1237        }
1238    }
1239
1240    fn generate_server_operation(
1241        &mut self,
1242        clients: &[(Rc<TestClient>, TestAppContext)],
1243    ) -> Option<Operation> {
1244        if self.operation_ix == self.max_operations {
1245            return None;
1246        }
1247
1248        Some(loop {
1249            break match self.rng.gen_range(0..100) {
1250                0..=29 if clients.len() < self.users.len() => {
1251                    let user = self
1252                        .users
1253                        .iter()
1254                        .filter(|u| !u.online)
1255                        .choose(&mut self.rng)
1256                        .unwrap();
1257                    self.operation_ix += 1;
1258                    Operation::AddConnection {
1259                        user_id: user.user_id,
1260                    }
1261                }
1262                30..=34 if clients.len() > 1 && self.allow_client_disconnection => {
1263                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1264                    let user_id = client.current_user_id(cx);
1265                    self.operation_ix += 1;
1266                    Operation::RemoveConnection { user_id }
1267                }
1268                35..=39 if clients.len() > 1 && self.allow_client_reconnection => {
1269                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1270                    let user_id = client.current_user_id(cx);
1271                    self.operation_ix += 1;
1272                    Operation::BounceConnection { user_id }
1273                }
1274                40..=44 if self.allow_server_restarts && clients.len() > 1 => {
1275                    self.operation_ix += 1;
1276                    Operation::RestartServer
1277                }
1278                _ if !clients.is_empty() => {
1279                    let count = self
1280                        .rng
1281                        .gen_range(1..10)
1282                        .min(self.max_operations - self.operation_ix);
1283                    let batch_id = util::post_inc(&mut self.next_batch_id);
1284                    let mut user_ids = (0..count)
1285                        .map(|_| {
1286                            let ix = self.rng.gen_range(0..clients.len());
1287                            let (client, cx) = &clients[ix];
1288                            client.current_user_id(cx)
1289                        })
1290                        .collect::<Vec<_>>();
1291                    user_ids.sort_unstable();
1292                    Operation::MutateClients {
1293                        user_ids,
1294                        batch_id,
1295                        quiesce: self.rng.gen_bool(0.7),
1296                    }
1297                }
1298                _ => continue,
1299            };
1300        })
1301    }
1302
1303    fn generate_client_operation(
1304        &mut self,
1305        user_id: UserId,
1306        client: &TestClient,
1307        cx: &TestAppContext,
1308    ) -> Option<ClientOperation> {
1309        if self.operation_ix == self.max_operations {
1310            return None;
1311        }
1312
1313        let executor = cx.background();
1314        self.operation_ix += 1;
1315        let call = cx.read(ActiveCall::global);
1316        Some(loop {
1317            match self.rng.gen_range(0..100_u32) {
1318                // Mutate the call
1319                0..=29 => {
1320                    // Respond to an incoming call
1321                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
1322                        break if self.rng.gen_bool(0.7) {
1323                            ClientOperation::AcceptIncomingCall
1324                        } else {
1325                            ClientOperation::RejectIncomingCall
1326                        };
1327                    }
1328
1329                    match self.rng.gen_range(0..100_u32) {
1330                        // Invite a contact to the current call
1331                        0..=70 => {
1332                            let available_contacts =
1333                                client.user_store.read_with(cx, |user_store, _| {
1334                                    user_store
1335                                        .contacts()
1336                                        .iter()
1337                                        .filter(|contact| contact.online && !contact.busy)
1338                                        .cloned()
1339                                        .collect::<Vec<_>>()
1340                                });
1341                            if !available_contacts.is_empty() {
1342                                let contact = available_contacts.choose(&mut self.rng).unwrap();
1343                                break ClientOperation::InviteContactToCall {
1344                                    user_id: UserId(contact.user.id as i32),
1345                                };
1346                            }
1347                        }
1348
1349                        // Leave the current call
1350                        71.. => {
1351                            if self.allow_client_disconnection
1352                                && call.read_with(cx, |call, _| call.room().is_some())
1353                            {
1354                                break ClientOperation::LeaveCall;
1355                            }
1356                        }
1357                    }
1358                }
1359
1360                // Mutate projects
1361                30..=59 => match self.rng.gen_range(0..100_u32) {
1362                    // Open a new project
1363                    0..=70 => {
1364                        // Open a remote project
1365                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
1366                            let existing_remote_project_ids = cx.read(|cx| {
1367                                client
1368                                    .remote_projects()
1369                                    .iter()
1370                                    .map(|p| p.read(cx).remote_id().unwrap())
1371                                    .collect::<Vec<_>>()
1372                            });
1373                            let new_remote_projects = room.read_with(cx, |room, _| {
1374                                room.remote_participants()
1375                                    .values()
1376                                    .flat_map(|participant| {
1377                                        participant.projects.iter().filter_map(|project| {
1378                                            if existing_remote_project_ids.contains(&project.id) {
1379                                                None
1380                                            } else {
1381                                                Some((
1382                                                    UserId::from_proto(participant.user.id),
1383                                                    project.worktree_root_names[0].clone(),
1384                                                ))
1385                                            }
1386                                        })
1387                                    })
1388                                    .collect::<Vec<_>>()
1389                            });
1390                            if !new_remote_projects.is_empty() {
1391                                let (host_id, first_root_name) =
1392                                    new_remote_projects.choose(&mut self.rng).unwrap().clone();
1393                                break ClientOperation::OpenRemoteProject {
1394                                    host_id,
1395                                    first_root_name,
1396                                };
1397                            }
1398                        }
1399                        // Open a local project
1400                        else {
1401                            let first_root_name = self.next_root_dir_name(user_id);
1402                            break ClientOperation::OpenLocalProject { first_root_name };
1403                        }
1404                    }
1405
1406                    // Close a remote project
1407                    71..=80 => {
1408                        if !client.remote_projects().is_empty() {
1409                            let project = client
1410                                .remote_projects()
1411                                .choose(&mut self.rng)
1412                                .unwrap()
1413                                .clone();
1414                            let first_root_name = root_name_for_project(&project, cx);
1415                            break ClientOperation::CloseRemoteProject {
1416                                project_root_name: first_root_name,
1417                            };
1418                        }
1419                    }
1420
1421                    // Mutate project worktrees
1422                    81.. => match self.rng.gen_range(0..100_u32) {
1423                        // Add a worktree to a local project
1424                        0..=50 => {
1425                            let Some(project) = client
1426                                .local_projects()
1427                                .choose(&mut self.rng)
1428                                .cloned() else { continue };
1429                            let project_root_name = root_name_for_project(&project, cx);
1430                            let mut paths = executor.block(client.fs.paths());
1431                            paths.remove(0);
1432                            let new_root_path = if paths.is_empty() || self.rng.gen() {
1433                                Path::new("/").join(&self.next_root_dir_name(user_id))
1434                            } else {
1435                                paths.choose(&mut self.rng).unwrap().clone()
1436                            };
1437                            break ClientOperation::AddWorktreeToProject {
1438                                project_root_name,
1439                                new_root_path,
1440                            };
1441                        }
1442
1443                        // Add an entry to a worktree
1444                        _ => {
1445                            let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1446                            let project_root_name = root_name_for_project(&project, cx);
1447                            let is_local = project.read_with(cx, |project, _| project.is_local());
1448                            let worktree = project.read_with(cx, |project, cx| {
1449                                project
1450                                    .worktrees(cx)
1451                                    .filter(|worktree| {
1452                                        let worktree = worktree.read(cx);
1453                                        worktree.is_visible()
1454                                            && worktree.entries(false).any(|e| e.is_file())
1455                                            && worktree.root_entry().map_or(false, |e| e.is_dir())
1456                                    })
1457                                    .choose(&mut self.rng)
1458                            });
1459                            let Some(worktree) = worktree else { continue };
1460                            let is_dir = self.rng.gen::<bool>();
1461                            let mut full_path =
1462                                worktree.read_with(cx, |w, _| PathBuf::from(w.root_name()));
1463                            full_path.push(gen_file_name(&mut self.rng));
1464                            if !is_dir {
1465                                full_path.set_extension("rs");
1466                            }
1467                            break ClientOperation::CreateWorktreeEntry {
1468                                project_root_name,
1469                                is_local,
1470                                full_path,
1471                                is_dir,
1472                            };
1473                        }
1474                    },
1475                },
1476
1477                // Query and mutate buffers
1478                60..=90 => {
1479                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1480                    let project_root_name = root_name_for_project(&project, cx);
1481                    let is_local = project.read_with(cx, |project, _| project.is_local());
1482
1483                    match self.rng.gen_range(0..100_u32) {
1484                        // Manipulate an existing buffer
1485                        0..=70 => {
1486                            let Some(buffer) = client
1487                                .buffers_for_project(&project)
1488                                .iter()
1489                                .choose(&mut self.rng)
1490                                .cloned() else { continue };
1491
1492                            let full_path = buffer
1493                                .read_with(cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
1494
1495                            match self.rng.gen_range(0..100_u32) {
1496                                // Close the buffer
1497                                0..=15 => {
1498                                    break ClientOperation::CloseBuffer {
1499                                        project_root_name,
1500                                        is_local,
1501                                        full_path,
1502                                    };
1503                                }
1504                                // Save the buffer
1505                                16..=29 if buffer.read_with(cx, |b, _| b.is_dirty()) => {
1506                                    let detach = self.rng.gen_bool(0.3);
1507                                    break ClientOperation::SaveBuffer {
1508                                        project_root_name,
1509                                        is_local,
1510                                        full_path,
1511                                        detach,
1512                                    };
1513                                }
1514                                // Edit the buffer
1515                                30..=69 => {
1516                                    let edits = buffer.read_with(cx, |buffer, _| {
1517                                        buffer.get_random_edits(&mut self.rng, 3)
1518                                    });
1519                                    break ClientOperation::EditBuffer {
1520                                        project_root_name,
1521                                        is_local,
1522                                        full_path,
1523                                        edits,
1524                                    };
1525                                }
1526                                // Make an LSP request
1527                                _ => {
1528                                    let offset = buffer.read_with(cx, |buffer, _| {
1529                                        buffer.clip_offset(
1530                                            self.rng.gen_range(0..=buffer.len()),
1531                                            language::Bias::Left,
1532                                        )
1533                                    });
1534                                    let detach = self.rng.gen();
1535                                    break ClientOperation::RequestLspDataInBuffer {
1536                                        project_root_name,
1537                                        full_path,
1538                                        offset,
1539                                        is_local,
1540                                        kind: match self.rng.gen_range(0..5_u32) {
1541                                            0 => LspRequestKind::Rename,
1542                                            1 => LspRequestKind::Highlights,
1543                                            2 => LspRequestKind::Definition,
1544                                            3 => LspRequestKind::CodeAction,
1545                                            4.. => LspRequestKind::Completion,
1546                                        },
1547                                        detach,
1548                                    };
1549                                }
1550                            }
1551                        }
1552
1553                        71..=80 => {
1554                            let query = self.rng.gen_range('a'..='z').to_string();
1555                            let detach = self.rng.gen_bool(0.3);
1556                            break ClientOperation::SearchProject {
1557                                project_root_name,
1558                                is_local,
1559                                query,
1560                                detach,
1561                            };
1562                        }
1563
1564                        // Open a buffer
1565                        81.. => {
1566                            let worktree = project.read_with(cx, |project, cx| {
1567                                project
1568                                    .worktrees(cx)
1569                                    .filter(|worktree| {
1570                                        let worktree = worktree.read(cx);
1571                                        worktree.is_visible()
1572                                            && worktree.entries(false).any(|e| e.is_file())
1573                                    })
1574                                    .choose(&mut self.rng)
1575                            });
1576                            let Some(worktree) = worktree else { continue };
1577                            let full_path = worktree.read_with(cx, |worktree, _| {
1578                                let entry = worktree
1579                                    .entries(false)
1580                                    .filter(|e| e.is_file())
1581                                    .choose(&mut self.rng)
1582                                    .unwrap();
1583                                if entry.path.as_ref() == Path::new("") {
1584                                    Path::new(worktree.root_name()).into()
1585                                } else {
1586                                    Path::new(worktree.root_name()).join(&entry.path)
1587                                }
1588                            });
1589                            break ClientOperation::OpenBuffer {
1590                                project_root_name,
1591                                is_local,
1592                                full_path,
1593                            };
1594                        }
1595                    }
1596                }
1597
1598                // Update a git index
1599                91..=95 => {
1600                    let repo_path = executor
1601                        .block(client.fs.directories())
1602                        .choose(&mut self.rng)
1603                        .unwrap()
1604                        .clone();
1605
1606                    let mut file_paths = executor
1607                        .block(client.fs.files())
1608                        .into_iter()
1609                        .filter(|path| path.starts_with(&repo_path))
1610                        .collect::<Vec<_>>();
1611                    let count = self.rng.gen_range(0..=file_paths.len());
1612                    file_paths.shuffle(&mut self.rng);
1613                    file_paths.truncate(count);
1614
1615                    let mut contents = Vec::new();
1616                    for abs_child_file_path in &file_paths {
1617                        let child_file_path = abs_child_file_path
1618                            .strip_prefix(&repo_path)
1619                            .unwrap()
1620                            .to_path_buf();
1621                        let new_base = Alphanumeric.sample_string(&mut self.rng, 16);
1622                        contents.push((child_file_path, new_base));
1623                    }
1624
1625                    break ClientOperation::WriteGitIndex {
1626                        repo_path,
1627                        contents,
1628                    };
1629                }
1630
1631                // Create a file or directory
1632                96.. => {
1633                    let is_dir = self.rng.gen::<bool>();
1634                    let mut path = cx
1635                        .background()
1636                        .block(client.fs.directories())
1637                        .choose(&mut self.rng)
1638                        .unwrap()
1639                        .clone();
1640                    path.push(gen_file_name(&mut self.rng));
1641                    if !is_dir {
1642                        path.set_extension("rs");
1643                    }
1644                    break ClientOperation::CreateFsEntry { path, is_dir };
1645                }
1646            }
1647        })
1648    }
1649
1650    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
1651        let user_ix = self
1652            .users
1653            .iter()
1654            .position(|user| user.user_id == user_id)
1655            .unwrap();
1656        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
1657        format!("dir-{user_id}-{root_id}")
1658    }
1659
1660    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
1661        let ix = self
1662            .users
1663            .iter()
1664            .position(|user| user.user_id == user_id)
1665            .unwrap();
1666        &mut self.users[ix]
1667    }
1668}
1669
1670async fn simulate_client(
1671    client: Rc<TestClient>,
1672    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<usize>,
1673    plan: Arc<Mutex<TestPlan>>,
1674    mut cx: TestAppContext,
1675) {
1676    // Setup language server
1677    let mut language = Language::new(
1678        LanguageConfig {
1679            name: "Rust".into(),
1680            path_suffixes: vec!["rs".to_string()],
1681            ..Default::default()
1682        },
1683        None,
1684    );
1685    let _fake_language_servers = language
1686        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
1687            name: "the-fake-language-server",
1688            capabilities: lsp::LanguageServer::full_capabilities(),
1689            initializer: Some(Box::new({
1690                let plan = plan.clone();
1691                let fs = client.fs.clone();
1692                move |fake_server: &mut FakeLanguageServer| {
1693                    fake_server.handle_request::<lsp::request::Completion, _, _>(
1694                        |_, _| async move {
1695                            Ok(Some(lsp::CompletionResponse::Array(vec![
1696                                lsp::CompletionItem {
1697                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
1698                                        range: lsp::Range::new(
1699                                            lsp::Position::new(0, 0),
1700                                            lsp::Position::new(0, 0),
1701                                        ),
1702                                        new_text: "the-new-text".to_string(),
1703                                    })),
1704                                    ..Default::default()
1705                                },
1706                            ])))
1707                        },
1708                    );
1709
1710                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
1711                        |_, _| async move {
1712                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
1713                                lsp::CodeAction {
1714                                    title: "the-code-action".to_string(),
1715                                    ..Default::default()
1716                                },
1717                            )]))
1718                        },
1719                    );
1720
1721                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
1722                        |params, _| async move {
1723                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
1724                                params.position,
1725                                params.position,
1726                            ))))
1727                        },
1728                    );
1729
1730                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
1731                        let fs = fs.clone();
1732                        let plan = plan.clone();
1733                        move |_, _| {
1734                            let fs = fs.clone();
1735                            let plan = plan.clone();
1736                            async move {
1737                                let files = fs.files().await;
1738                                let count = plan.lock().rng.gen_range::<usize, _>(1..3);
1739                                let files = (0..count)
1740                                    .map(|_| files.choose(&mut plan.lock().rng).unwrap())
1741                                    .collect::<Vec<_>>();
1742                                log::info!("LSP: Returning definitions in files {:?}", &files);
1743                                Ok(Some(lsp::GotoDefinitionResponse::Array(
1744                                    files
1745                                        .into_iter()
1746                                        .map(|file| lsp::Location {
1747                                            uri: lsp::Url::from_file_path(file).unwrap(),
1748                                            range: Default::default(),
1749                                        })
1750                                        .collect(),
1751                                )))
1752                            }
1753                        }
1754                    });
1755
1756                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
1757                        let plan = plan.clone();
1758                        move |_, _| {
1759                            let mut highlights = Vec::new();
1760                            let highlight_count = plan.lock().rng.gen_range(1..=5);
1761                            for _ in 0..highlight_count {
1762                                let start_row = plan.lock().rng.gen_range(0..100);
1763                                let start_column = plan.lock().rng.gen_range(0..100);
1764                                let start = PointUtf16::new(start_row, start_column);
1765                                let end_row = plan.lock().rng.gen_range(0..100);
1766                                let end_column = plan.lock().rng.gen_range(0..100);
1767                                let end = PointUtf16::new(end_row, end_column);
1768                                let range = if start > end { end..start } else { start..end };
1769                                highlights.push(lsp::DocumentHighlight {
1770                                    range: range_to_lsp(range.clone()),
1771                                    kind: Some(lsp::DocumentHighlightKind::READ),
1772                                });
1773                            }
1774                            highlights.sort_unstable_by_key(|highlight| {
1775                                (highlight.range.start, highlight.range.end)
1776                            });
1777                            async move { Ok(Some(highlights)) }
1778                        }
1779                    });
1780                }
1781            })),
1782            ..Default::default()
1783        }))
1784        .await;
1785    client.language_registry.add(Arc::new(language));
1786
1787    while let Some(batch_id) = operation_rx.next().await {
1788        let Some((operation, skipped)) = plan.lock().next_client_operation(&client, batch_id, &cx) else { break };
1789        match apply_client_operation(&client, operation, &mut cx).await {
1790            Err(error) => {
1791                log::error!("{} error: {}", client.username, error);
1792            }
1793            Ok(applied) => {
1794                if !applied {
1795                    skipped.store(true, SeqCst);
1796                }
1797            }
1798        }
1799        cx.background().simulate_random_delay().await;
1800    }
1801    log::info!("{}: done", client.username);
1802}
1803
1804fn buffer_for_full_path(
1805    buffers: &HashSet<ModelHandle<language::Buffer>>,
1806    full_path: &PathBuf,
1807    cx: &TestAppContext,
1808) -> Option<ModelHandle<language::Buffer>> {
1809    buffers
1810        .iter()
1811        .find(|buffer| {
1812            buffer.read_with(cx, |buffer, cx| {
1813                buffer.file().unwrap().full_path(cx) == *full_path
1814            })
1815        })
1816        .cloned()
1817}
1818
1819fn project_for_root_name(
1820    client: &TestClient,
1821    root_name: &str,
1822    cx: &TestAppContext,
1823) -> Option<ModelHandle<Project>> {
1824    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1825        return Some(client.local_projects()[ix].clone());
1826    }
1827    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1828        return Some(client.remote_projects()[ix].clone());
1829    }
1830    None
1831}
1832
1833fn project_ix_for_root_name(
1834    projects: &[ModelHandle<Project>],
1835    root_name: &str,
1836    cx: &TestAppContext,
1837) -> Option<usize> {
1838    projects.iter().position(|project| {
1839        project.read_with(cx, |project, cx| {
1840            let worktree = project.visible_worktrees(cx).next().unwrap();
1841            worktree.read(cx).root_name() == root_name
1842        })
1843    })
1844}
1845
1846fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1847    project.read_with(cx, |project, cx| {
1848        project
1849            .visible_worktrees(cx)
1850            .next()
1851            .unwrap()
1852            .read(cx)
1853            .root_name()
1854            .to_string()
1855    })
1856}
1857
1858fn project_path_for_full_path(
1859    project: &ModelHandle<Project>,
1860    full_path: &Path,
1861    cx: &TestAppContext,
1862) -> Option<ProjectPath> {
1863    let mut components = full_path.components();
1864    let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1865    let path = components.as_path().into();
1866    let worktree_id = project.read_with(cx, |project, cx| {
1867        project.worktrees(cx).find_map(|worktree| {
1868            let worktree = worktree.read(cx);
1869            if worktree.root_name() == root_name {
1870                Some(worktree.id())
1871            } else {
1872                None
1873            }
1874        })
1875    })?;
1876    Some(ProjectPath { worktree_id, path })
1877}
1878
1879async fn ensure_project_shared(
1880    project: &ModelHandle<Project>,
1881    client: &TestClient,
1882    cx: &mut TestAppContext,
1883) {
1884    let first_root_name = root_name_for_project(project, cx);
1885    let active_call = cx.read(ActiveCall::global);
1886    if active_call.read_with(cx, |call, _| call.room().is_some())
1887        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
1888    {
1889        match active_call
1890            .update(cx, |call, cx| call.share_project(project.clone(), cx))
1891            .await
1892        {
1893            Ok(project_id) => {
1894                log::info!(
1895                    "{}: shared project {} with id {}",
1896                    client.username,
1897                    first_root_name,
1898                    project_id
1899                );
1900            }
1901            Err(error) => {
1902                log::error!(
1903                    "{}: error sharing project {}: {:?}",
1904                    client.username,
1905                    first_root_name,
1906                    error
1907                );
1908            }
1909        }
1910    }
1911}
1912
1913fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1914    client
1915        .local_projects()
1916        .iter()
1917        .chain(client.remote_projects().iter())
1918        .choose(rng)
1919        .cloned()
1920}
1921
1922fn gen_file_name(rng: &mut StdRng) -> String {
1923    let mut name = String::new();
1924    for _ in 0..10 {
1925        let letter = rng.gen_range('a'..='z');
1926        name.push(letter);
1927    }
1928    name
1929}
1930
1931fn path_env_var(name: &str) -> Option<PathBuf> {
1932    let value = env::var(name).ok()?;
1933    let mut path = PathBuf::from(value);
1934    if path.is_relative() {
1935        let mut abs_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
1936        abs_path.pop();
1937        abs_path.pop();
1938        abs_path.push(path);
1939        path = abs_path
1940    }
1941    Some(path)
1942}