1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use editor::Bias;
  11use fs::{FakeFs, Fs as _};
  12use futures::StreamExt as _;
  13use gpui::{executor::Deterministic, ModelHandle, Task, TestAppContext};
  14use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  15use lsp::FakeLanguageServer;
  16use parking_lot::Mutex;
  17use project::{search::SearchQuery, Project, ProjectPath};
  18use rand::{
  19    distributions::{Alphanumeric, DistString},
  20    prelude::*,
  21};
  22use serde::{Deserialize, Serialize};
  23use std::{
  24    env,
  25    ops::Range,
  26    path::{Path, PathBuf},
  27    rc::Rc,
  28    sync::{
  29        atomic::{AtomicBool, Ordering::SeqCst},
  30        Arc,
  31    },
  32};
  33use util::ResultExt;
  34
  35#[gpui::test(iterations = 100)]
  36async fn test_random_collaboration(
  37    cx: &mut TestAppContext,
  38    deterministic: Arc<Deterministic>,
  39    rng: StdRng,
  40) {
  41    deterministic.forbid_parking();
  42
  43    let max_peers = env::var("MAX_PEERS")
  44        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  45        .unwrap_or(3);
  46    let max_operations = env::var("OPERATIONS")
  47        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  48        .unwrap_or(10);
  49
  50    let plan_load_path = path_env_var("LOAD_PLAN");
  51    let plan_save_path = path_env_var("SAVE_PLAN");
  52
  53    let mut server = TestServer::start(&deterministic).await;
  54    let db = server.app_state.db.clone();
  55
  56    let mut users = Vec::new();
  57    for ix in 0..max_peers {
  58        let username = format!("user-{}", ix + 1);
  59        let user_id = db
  60            .create_user(
  61                &format!("{username}@example.com"),
  62                false,
  63                NewUserParams {
  64                    github_login: username.clone(),
  65                    github_user_id: (ix + 1) as i32,
  66                    invite_count: 0,
  67                },
  68            )
  69            .await
  70            .unwrap()
  71            .user_id;
  72        users.push(UserTestPlan {
  73            user_id,
  74            username,
  75            online: false,
  76            next_root_id: 0,
  77            operation_ix: 0,
  78        });
  79    }
  80
  81    for (ix, user_a) in users.iter().enumerate() {
  82        for user_b in &users[ix + 1..] {
  83            server
  84                .app_state
  85                .db
  86                .send_contact_request(user_a.user_id, user_b.user_id)
  87                .await
  88                .unwrap();
  89            server
  90                .app_state
  91                .db
  92                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  93                .await
  94                .unwrap();
  95        }
  96    }
  97
  98    let plan = Arc::new(Mutex::new(TestPlan::new(rng, users, max_operations)));
  99
 100    if let Some(path) = &plan_load_path {
 101        eprintln!("loaded plan from path {:?}", path);
 102        plan.lock().load(path);
 103    }
 104
 105    let mut clients = Vec::new();
 106    let mut client_tasks = Vec::new();
 107    let mut operation_channels = Vec::new();
 108
 109    loop {
 110        let Some((next_operation, skipped)) = plan.lock().next_server_operation(&clients) else { break };
 111        let applied = apply_server_operation(
 112            deterministic.clone(),
 113            &mut server,
 114            &mut clients,
 115            &mut client_tasks,
 116            &mut operation_channels,
 117            plan.clone(),
 118            next_operation,
 119            cx,
 120        )
 121        .await;
 122        if !applied {
 123            skipped.store(false, SeqCst);
 124        }
 125    }
 126
 127    drop(operation_channels);
 128    deterministic.start_waiting();
 129    futures::future::join_all(client_tasks).await;
 130    deterministic.finish_waiting();
 131    deterministic.run_until_parked();
 132
 133    if let Some(path) = &plan_save_path {
 134        eprintln!("saved test plan to path {:?}", path);
 135        plan.lock().save(path);
 136    }
 137
 138    for (client, client_cx) in &clients {
 139        for guest_project in client.remote_projects().iter() {
 140            guest_project.read_with(client_cx, |guest_project, cx| {
 141                let host_project = clients.iter().find_map(|(client, cx)| {
 142                    let project = client
 143                        .local_projects()
 144                        .iter()
 145                        .find(|host_project| {
 146                            host_project.read_with(cx, |host_project, _| {
 147                                host_project.remote_id() == guest_project.remote_id()
 148                            })
 149                        })?
 150                        .clone();
 151                    Some((project, cx))
 152                });
 153
 154                if !guest_project.is_read_only() {
 155                    if let Some((host_project, host_cx)) = host_project {
 156                        let host_worktree_snapshots =
 157                            host_project.read_with(host_cx, |host_project, cx| {
 158                                host_project
 159                                    .worktrees(cx)
 160                                    .map(|worktree| {
 161                                        let worktree = worktree.read(cx);
 162                                        (worktree.id(), worktree.snapshot())
 163                                    })
 164                                    .collect::<BTreeMap<_, _>>()
 165                            });
 166                        let guest_worktree_snapshots = guest_project
 167                            .worktrees(cx)
 168                            .map(|worktree| {
 169                                let worktree = worktree.read(cx);
 170                                (worktree.id(), worktree.snapshot())
 171                            })
 172                            .collect::<BTreeMap<_, _>>();
 173
 174                        assert_eq!(
 175                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 176                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 177                            "{} has different worktrees than the host",
 178                            client.username
 179                        );
 180
 181                        for (id, host_snapshot) in &host_worktree_snapshots {
 182                            let guest_snapshot = &guest_worktree_snapshots[id];
 183                            assert_eq!(
 184                                guest_snapshot.root_name(),
 185                                host_snapshot.root_name(),
 186                                "{} has different root name than the host for worktree {}",
 187                                client.username,
 188                                id
 189                            );
 190                            assert_eq!(
 191                                guest_snapshot.abs_path(),
 192                                host_snapshot.abs_path(),
 193                                "{} has different abs path than the host for worktree {}",
 194                                client.username,
 195                                id
 196                            );
 197                            assert_eq!(
 198                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 199                                host_snapshot.entries(false).collect::<Vec<_>>(),
 200                                "{} has different snapshot than the host for worktree {:?} and project {:?}",
 201                                client.username,
 202                                host_snapshot.abs_path(),
 203                                host_project.read_with(host_cx, |project, _| project.remote_id())
 204                            );
 205                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id(),
 206                                "{} has different scan id than the host for worktree {:?} and project {:?}",
 207                                client.username,
 208                                host_snapshot.abs_path(),
 209                                host_project.read_with(host_cx, |project, _| project.remote_id())
 210                            );
 211                        }
 212                    }
 213                }
 214
 215                guest_project.check_invariants(cx);
 216            });
 217        }
 218
 219        let buffers = client.buffers().clone();
 220        for (guest_project, guest_buffers) in &buffers {
 221            let project_id = if guest_project.read_with(client_cx, |project, _| {
 222                project.is_local() || project.is_read_only()
 223            }) {
 224                continue;
 225            } else {
 226                guest_project
 227                    .read_with(client_cx, |project, _| project.remote_id())
 228                    .unwrap()
 229            };
 230            let guest_user_id = client.user_id().unwrap();
 231
 232            let host_project = clients.iter().find_map(|(client, cx)| {
 233                let project = client
 234                    .local_projects()
 235                    .iter()
 236                    .find(|host_project| {
 237                        host_project.read_with(cx, |host_project, _| {
 238                            host_project.remote_id() == Some(project_id)
 239                        })
 240                    })?
 241                    .clone();
 242                Some((client.user_id().unwrap(), project, cx))
 243            });
 244
 245            let (host_user_id, host_project, host_cx) =
 246                if let Some((host_user_id, host_project, host_cx)) = host_project {
 247                    (host_user_id, host_project, host_cx)
 248                } else {
 249                    continue;
 250                };
 251
 252            for guest_buffer in guest_buffers {
 253                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 254                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 255                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 256                        panic!(
 257                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 258                            client.username,
 259                            client.peer_id(),
 260                            buffer_id
 261                        )
 262                    })
 263                });
 264                let path = host_buffer
 265                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 266
 267                assert_eq!(
 268                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 269                    0,
 270                    "{}, buffer {}, path {:?} has deferred operations",
 271                    client.username,
 272                    buffer_id,
 273                    path,
 274                );
 275                assert_eq!(
 276                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 277                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 278                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 279                    client.username,
 280                    buffer_id,
 281                    path
 282                );
 283
 284                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 285                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 286                match (host_file, guest_file) {
 287                    (Some(host_file), Some(guest_file)) => {
 288                        assert_eq!(guest_file.path(), host_file.path());
 289                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 290                        assert_eq!(
 291                            guest_file.mtime(),
 292                            host_file.mtime(),
 293                            "guest {} mtime does not match host {} for path {:?} in project {}",
 294                            guest_user_id,
 295                            host_user_id,
 296                            guest_file.path(),
 297                            project_id,
 298                        );
 299                    }
 300                    (None, None) => {}
 301                    (None, _) => panic!("host's file is None, guest's isn't"),
 302                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 303                }
 304
 305                let host_diff_base =
 306                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 307                let guest_diff_base = guest_buffer
 308                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 309                assert_eq!(guest_diff_base, host_diff_base);
 310            }
 311        }
 312    }
 313
 314    for (client, mut cx) in clients {
 315        cx.update(|cx| {
 316            cx.clear_globals();
 317            drop(client);
 318        });
 319    }
 320}
 321
 322async fn apply_server_operation(
 323    deterministic: Arc<Deterministic>,
 324    server: &mut TestServer,
 325    clients: &mut Vec<(Rc<TestClient>, TestAppContext)>,
 326    client_tasks: &mut Vec<Task<()>>,
 327    operation_channels: &mut Vec<futures::channel::mpsc::UnboundedSender<usize>>,
 328    plan: Arc<Mutex<TestPlan>>,
 329    operation: Operation,
 330    cx: &mut TestAppContext,
 331) -> bool {
 332    match operation {
 333        Operation::AddConnection { user_id } => {
 334            let username;
 335            {
 336                let mut plan = plan.lock();
 337                let mut user = plan.user(user_id);
 338                if user.online {
 339                    return false;
 340                }
 341                user.online = true;
 342                username = user.username.clone();
 343            };
 344            log::info!("Adding new connection for {}", username);
 345            let next_entity_id = (user_id.0 * 10_000) as usize;
 346            let mut client_cx = TestAppContext::new(
 347                cx.foreground_platform(),
 348                cx.platform(),
 349                deterministic.build_foreground(user_id.0 as usize),
 350                deterministic.build_background(),
 351                cx.font_cache(),
 352                cx.leak_detector(),
 353                next_entity_id,
 354                cx.function_name.clone(),
 355            );
 356
 357            let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 358            let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 359            operation_channels.push(operation_tx);
 360            clients.push((client.clone(), client_cx.clone()));
 361            client_tasks.push(client_cx.foreground().spawn(simulate_client(
 362                client,
 363                operation_rx,
 364                plan.clone(),
 365                client_cx,
 366            )));
 367
 368            log::info!("Added connection for {}", username);
 369        }
 370
 371        Operation::RemoveConnection { user_id } => {
 372            log::info!("Simulating full disconnection of user {}", user_id);
 373            let client_ix = clients
 374                .iter()
 375                .position(|(client, cx)| client.current_user_id(cx) == user_id);
 376            let Some(client_ix) = client_ix else { return false };
 377            let user_connection_ids = server
 378                .connection_pool
 379                .lock()
 380                .user_connection_ids(user_id)
 381                .collect::<Vec<_>>();
 382            assert_eq!(user_connection_ids.len(), 1);
 383            let removed_peer_id = user_connection_ids[0].into();
 384            let (client, mut client_cx) = clients.remove(client_ix);
 385            let client_task = client_tasks.remove(client_ix);
 386            operation_channels.remove(client_ix);
 387            server.forbid_connections();
 388            server.disconnect_client(removed_peer_id);
 389            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 390            deterministic.start_waiting();
 391            log::info!("Waiting for user {} to exit...", user_id);
 392            client_task.await;
 393            deterministic.finish_waiting();
 394            server.allow_connections();
 395
 396            for project in client.remote_projects().iter() {
 397                project.read_with(&client_cx, |project, _| {
 398                    assert!(
 399                        project.is_read_only(),
 400                        "project {:?} should be read only",
 401                        project.remote_id()
 402                    )
 403                });
 404            }
 405
 406            for (client, cx) in clients {
 407                let contacts = server
 408                    .app_state
 409                    .db
 410                    .get_contacts(client.current_user_id(cx))
 411                    .await
 412                    .unwrap();
 413                let pool = server.connection_pool.lock();
 414                for contact in contacts {
 415                    if let db::Contact::Accepted { user_id: id, .. } = contact {
 416                        if pool.is_user_online(id) {
 417                            assert_ne!(
 418                                id, user_id,
 419                                "removed client is still a contact of another peer"
 420                            );
 421                        }
 422                    }
 423                }
 424            }
 425
 426            log::info!("{} removed", client.username);
 427            plan.lock().user(user_id).online = false;
 428            client_cx.update(|cx| {
 429                cx.clear_globals();
 430                drop(client);
 431            });
 432        }
 433
 434        Operation::BounceConnection { user_id } => {
 435            log::info!("Simulating temporary disconnection of user {}", user_id);
 436            let user_connection_ids = server
 437                .connection_pool
 438                .lock()
 439                .user_connection_ids(user_id)
 440                .collect::<Vec<_>>();
 441            if user_connection_ids.is_empty() {
 442                return false;
 443            }
 444            assert_eq!(user_connection_ids.len(), 1);
 445            let peer_id = user_connection_ids[0].into();
 446            server.disconnect_client(peer_id);
 447            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 448        }
 449
 450        Operation::RestartServer => {
 451            log::info!("Simulating server restart");
 452            server.reset().await;
 453            deterministic.advance_clock(RECEIVE_TIMEOUT);
 454            server.start().await.unwrap();
 455            deterministic.advance_clock(CLEANUP_TIMEOUT);
 456            let environment = &server.app_state.config.zed_environment;
 457            let stale_room_ids = server
 458                .app_state
 459                .db
 460                .stale_room_ids(environment, server.id())
 461                .await
 462                .unwrap();
 463            assert_eq!(stale_room_ids, vec![]);
 464        }
 465
 466        Operation::MutateClients {
 467            user_ids,
 468            batch_id,
 469            quiesce,
 470        } => {
 471            let mut applied = false;
 472            for user_id in user_ids {
 473                let client_ix = clients
 474                    .iter()
 475                    .position(|(client, cx)| client.current_user_id(cx) == user_id);
 476                let Some(client_ix) = client_ix else { continue };
 477                applied = true;
 478                if let Err(err) = operation_channels[client_ix].unbounded_send(batch_id) {
 479                    // panic!("error signaling user {}, client {}", user_id, client_ix);
 480                }
 481            }
 482
 483            if quiesce && applied {
 484                deterministic.run_until_parked();
 485            }
 486
 487            return applied;
 488        }
 489    }
 490    true
 491}
 492
 493async fn apply_client_operation(
 494    client: &TestClient,
 495    operation: ClientOperation,
 496    cx: &mut TestAppContext,
 497) -> Result<(), TestError> {
 498    match operation {
 499        ClientOperation::AcceptIncomingCall => {
 500            let active_call = cx.read(ActiveCall::global);
 501            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 502                Err(TestError::Inapplicable)?;
 503            }
 504
 505            log::info!("{}: accepting incoming call", client.username);
 506            active_call
 507                .update(cx, |call, cx| call.accept_incoming(cx))
 508                .await?;
 509        }
 510
 511        ClientOperation::RejectIncomingCall => {
 512            let active_call = cx.read(ActiveCall::global);
 513            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 514                Err(TestError::Inapplicable)?;
 515            }
 516
 517            log::info!("{}: declining incoming call", client.username);
 518            active_call.update(cx, |call, _| call.decline_incoming())?;
 519        }
 520
 521        ClientOperation::LeaveCall => {
 522            let active_call = cx.read(ActiveCall::global);
 523            if active_call.read_with(cx, |call, _| call.room().is_none()) {
 524                Err(TestError::Inapplicable)?;
 525            }
 526
 527            log::info!("{}: hanging up", client.username);
 528            active_call.update(cx, |call, cx| call.hang_up(cx))?;
 529        }
 530
 531        ClientOperation::InviteContactToCall { user_id } => {
 532            let active_call = cx.read(ActiveCall::global);
 533
 534            log::info!("{}: inviting {}", client.username, user_id,);
 535            active_call
 536                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 537                .await
 538                .log_err();
 539        }
 540
 541        ClientOperation::OpenLocalProject { first_root_name } => {
 542            log::info!(
 543                "{}: opening local project at {:?}",
 544                client.username,
 545                first_root_name
 546            );
 547
 548            let root_path = Path::new("/").join(&first_root_name);
 549            client.fs.create_dir(&root_path).await.unwrap();
 550            client
 551                .fs
 552                .create_file(&root_path.join("main.rs"), Default::default())
 553                .await
 554                .unwrap();
 555            let project = client.build_local_project(root_path, cx).await.0;
 556            ensure_project_shared(&project, client, cx).await;
 557            client.local_projects_mut().push(project.clone());
 558        }
 559
 560        ClientOperation::AddWorktreeToProject {
 561            project_root_name,
 562            new_root_path,
 563        } => {
 564            let project = project_for_root_name(client, &project_root_name, cx)
 565                .ok_or(TestError::Inapplicable)?;
 566
 567            log::info!(
 568                "{}: finding/creating local worktree at {:?} to project with root path {}",
 569                client.username,
 570                new_root_path,
 571                project_root_name
 572            );
 573
 574            ensure_project_shared(&project, client, cx).await;
 575            if !client.fs.paths().await.contains(&new_root_path) {
 576                client.fs.create_dir(&new_root_path).await.unwrap();
 577            }
 578            project
 579                .update(cx, |project, cx| {
 580                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 581                })
 582                .await
 583                .unwrap();
 584        }
 585
 586        ClientOperation::CloseRemoteProject { project_root_name } => {
 587            let project = project_for_root_name(client, &project_root_name, cx)
 588                .ok_or(TestError::Inapplicable)?;
 589
 590            log::info!(
 591                "{}: closing remote project with root path {}",
 592                client.username,
 593                project_root_name,
 594            );
 595
 596            let ix = client
 597                .remote_projects()
 598                .iter()
 599                .position(|p| p == &project)
 600                .unwrap();
 601            cx.update(|_| {
 602                client.remote_projects_mut().remove(ix);
 603                client.buffers().retain(|project, _| project != project);
 604                drop(project);
 605            });
 606        }
 607
 608        ClientOperation::OpenRemoteProject {
 609            host_id,
 610            first_root_name,
 611        } => {
 612            let active_call = cx.read(ActiveCall::global);
 613            let project = active_call
 614                .update(cx, |call, cx| {
 615                    let room = call.room().cloned()?;
 616                    let participant = room
 617                        .read(cx)
 618                        .remote_participants()
 619                        .get(&host_id.to_proto())?;
 620                    let project_id = participant
 621                        .projects
 622                        .iter()
 623                        .find(|project| project.worktree_root_names[0] == first_root_name)?
 624                        .id;
 625                    Some(room.update(cx, |room, cx| {
 626                        room.join_project(
 627                            project_id,
 628                            client.language_registry.clone(),
 629                            FakeFs::new(cx.background().clone()),
 630                            cx,
 631                        )
 632                    }))
 633                })
 634                .ok_or(TestError::Inapplicable)?;
 635
 636            log::info!(
 637                "{}: joining remote project of user {}, root name {}",
 638                client.username,
 639                host_id,
 640                first_root_name,
 641            );
 642
 643            let project = project.await?;
 644            client.remote_projects_mut().push(project.clone());
 645        }
 646
 647        ClientOperation::CreateWorktreeEntry {
 648            project_root_name,
 649            is_local,
 650            full_path,
 651            is_dir,
 652        } => {
 653            let project = project_for_root_name(client, &project_root_name, cx)
 654                .ok_or(TestError::Inapplicable)?;
 655            let project_path = project_path_for_full_path(&project, &full_path, cx)
 656                .ok_or(TestError::Inapplicable)?;
 657
 658            log::info!(
 659                "{}: creating {} at path {:?} in {} project {}",
 660                client.username,
 661                if is_dir { "dir" } else { "file" },
 662                full_path,
 663                if is_local { "local" } else { "remote" },
 664                project_root_name,
 665            );
 666
 667            ensure_project_shared(&project, client, cx).await;
 668            project
 669                .update(cx, |p, cx| p.create_entry(project_path, is_dir, cx))
 670                .unwrap()
 671                .await?;
 672        }
 673
 674        ClientOperation::OpenBuffer {
 675            project_root_name,
 676            is_local,
 677            full_path,
 678        } => {
 679            let project = project_for_root_name(client, &project_root_name, cx)
 680                .ok_or(TestError::Inapplicable)?;
 681            let project_path = project_path_for_full_path(&project, &full_path, cx)
 682                .ok_or(TestError::Inapplicable)?;
 683
 684            log::info!(
 685                "{}: opening buffer {:?} in {} project {}",
 686                client.username,
 687                full_path,
 688                if is_local { "local" } else { "remote" },
 689                project_root_name,
 690            );
 691
 692            ensure_project_shared(&project, client, cx).await;
 693            let buffer = project
 694                .update(cx, |project, cx| project.open_buffer(project_path, cx))
 695                .await?;
 696            client.buffers_for_project(&project).insert(buffer);
 697        }
 698
 699        ClientOperation::EditBuffer {
 700            project_root_name,
 701            is_local,
 702            full_path,
 703            edits,
 704        } => {
 705            let project = project_for_root_name(client, &project_root_name, cx)
 706                .ok_or(TestError::Inapplicable)?;
 707            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 708                .ok_or(TestError::Inapplicable)?;
 709
 710            log::info!(
 711                "{}: editing buffer {:?} in {} project {} with {:?}",
 712                client.username,
 713                full_path,
 714                if is_local { "local" } else { "remote" },
 715                project_root_name,
 716                edits
 717            );
 718
 719            ensure_project_shared(&project, client, cx).await;
 720            buffer.update(cx, |buffer, cx| {
 721                let snapshot = buffer.snapshot();
 722                buffer.edit(
 723                    edits.into_iter().map(|(range, text)| {
 724                        let start = snapshot.clip_offset(range.start, Bias::Left);
 725                        let end = snapshot.clip_offset(range.end, Bias::Right);
 726                        (start..end, text)
 727                    }),
 728                    None,
 729                    cx,
 730                );
 731            });
 732        }
 733
 734        ClientOperation::CloseBuffer {
 735            project_root_name,
 736            is_local,
 737            full_path,
 738        } => {
 739            let project = project_for_root_name(client, &project_root_name, cx)
 740                .ok_or(TestError::Inapplicable)?;
 741            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 742                .ok_or(TestError::Inapplicable)?;
 743
 744            log::info!(
 745                "{}: closing buffer {:?} in {} project {}",
 746                client.username,
 747                full_path,
 748                if is_local { "local" } else { "remote" },
 749                project_root_name
 750            );
 751
 752            ensure_project_shared(&project, client, cx).await;
 753            cx.update(|_| {
 754                client.buffers_for_project(&project).remove(&buffer);
 755                drop(buffer);
 756            });
 757        }
 758
 759        ClientOperation::SaveBuffer {
 760            project_root_name,
 761            is_local,
 762            full_path,
 763            detach,
 764        } => {
 765            let project = project_for_root_name(client, &project_root_name, cx)
 766                .ok_or(TestError::Inapplicable)?;
 767            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 768                .ok_or(TestError::Inapplicable)?;
 769
 770            log::info!(
 771                "{}: saving buffer {:?} in {} project {}{}",
 772                client.username,
 773                full_path,
 774                if is_local { "local" } else { "remote" },
 775                project_root_name,
 776                if detach { ", detaching" } else { ", awaiting" }
 777            );
 778
 779            ensure_project_shared(&project, client, cx).await;
 780            let (requested_version, save) =
 781                buffer.update(cx, |buffer, cx| (buffer.version(), buffer.save(cx)));
 782            let save = cx.background().spawn(async move {
 783                let (saved_version, _, _) = save
 784                    .await
 785                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 786                assert!(saved_version.observed_all(&requested_version));
 787                anyhow::Ok(())
 788            });
 789            if detach {
 790                cx.update(|cx| save.detach_and_log_err(cx));
 791            } else {
 792                save.await?;
 793            }
 794        }
 795
 796        ClientOperation::RequestLspDataInBuffer {
 797            project_root_name,
 798            is_local,
 799            full_path,
 800            offset,
 801            kind,
 802            detach,
 803        } => {
 804            let project = project_for_root_name(client, &project_root_name, cx)
 805                .ok_or(TestError::Inapplicable)?;
 806            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 807                .ok_or(TestError::Inapplicable)?;
 808
 809            log::info!(
 810                "{}: request LSP {:?} for buffer {:?} in {} project {}{}",
 811                client.username,
 812                kind,
 813                full_path,
 814                if is_local { "local" } else { "remote" },
 815                project_root_name,
 816                if detach { ", detaching" } else { ", awaiting" }
 817            );
 818
 819            use futures::{FutureExt as _, TryFutureExt as _};
 820            let offset = buffer.read_with(cx, |b, _| b.clip_offset(offset, Bias::Left));
 821            let request = cx.foreground().spawn(project.update(cx, |project, cx| {
 822                match kind {
 823                    LspRequestKind::Rename => project
 824                        .prepare_rename(buffer, offset, cx)
 825                        .map_ok(|_| ())
 826                        .boxed(),
 827                    LspRequestKind::Completion => project
 828                        .completions(&buffer, offset, cx)
 829                        .map_ok(|_| ())
 830                        .boxed(),
 831                    LspRequestKind::CodeAction => project
 832                        .code_actions(&buffer, offset..offset, cx)
 833                        .map_ok(|_| ())
 834                        .boxed(),
 835                    LspRequestKind::Definition => project
 836                        .definition(&buffer, offset, cx)
 837                        .map_ok(|_| ())
 838                        .boxed(),
 839                    LspRequestKind::Highlights => project
 840                        .document_highlights(&buffer, offset, cx)
 841                        .map_ok(|_| ())
 842                        .boxed(),
 843                }
 844            }));
 845            if detach {
 846                request.detach();
 847            } else {
 848                request.await?;
 849            }
 850        }
 851
 852        ClientOperation::SearchProject {
 853            project_root_name,
 854            is_local,
 855            query,
 856            detach,
 857        } => {
 858            let project = project_for_root_name(client, &project_root_name, cx)
 859                .ok_or(TestError::Inapplicable)?;
 860
 861            log::info!(
 862                "{}: search {} project {} for {:?}{}",
 863                client.username,
 864                if is_local { "local" } else { "remote" },
 865                project_root_name,
 866                query,
 867                if detach { ", detaching" } else { ", awaiting" }
 868            );
 869
 870            let search = project.update(cx, |project, cx| {
 871                project.search(SearchQuery::text(query, false, false), cx)
 872            });
 873            drop(project);
 874            let search = cx.background().spawn(async move {
 875                search
 876                    .await
 877                    .map_err(|err| anyhow!("search request failed: {:?}", err))
 878            });
 879            if detach {
 880                cx.update(|cx| search.detach_and_log_err(cx));
 881            } else {
 882                search.await?;
 883            }
 884        }
 885
 886        ClientOperation::CreateFsEntry { path, is_dir } => {
 887            client
 888                .fs
 889                .metadata(&path.parent().unwrap())
 890                .await?
 891                .ok_or(TestError::Inapplicable)?;
 892
 893            log::info!(
 894                "{}: creating {} at {:?}",
 895                client.username,
 896                if is_dir { "dir" } else { "file" },
 897                path
 898            );
 899
 900            if is_dir {
 901                client.fs.create_dir(&path).await.unwrap();
 902            } else {
 903                client
 904                    .fs
 905                    .create_file(&path, Default::default())
 906                    .await
 907                    .unwrap();
 908            }
 909        }
 910
 911        ClientOperation::WriteGitIndex {
 912            repo_path,
 913            contents,
 914        } => {
 915            if !client
 916                .fs
 917                .metadata(&repo_path)
 918                .await?
 919                .map_or(false, |m| m.is_dir)
 920            {
 921                Err(TestError::Inapplicable)?;
 922            }
 923
 924            log::info!(
 925                "{}: writing git index for repo {:?}: {:?}",
 926                client.username,
 927                repo_path,
 928                contents
 929            );
 930
 931            let dot_git_dir = repo_path.join(".git");
 932            let contents = contents
 933                .iter()
 934                .map(|(path, contents)| (path.as_path(), contents.clone()))
 935                .collect::<Vec<_>>();
 936            if client.fs.metadata(&dot_git_dir).await?.is_none() {
 937                client.fs.create_dir(&dot_git_dir).await?;
 938            }
 939            client.fs.set_index_for_repo(&dot_git_dir, &contents).await;
 940        }
 941    }
 942    Ok(())
 943}
 944
 945struct TestPlan {
 946    rng: StdRng,
 947    replay: bool,
 948    stored_operations: Vec<(StoredOperation, Arc<AtomicBool>)>,
 949    max_operations: usize,
 950    operation_ix: usize,
 951    users: Vec<UserTestPlan>,
 952    next_batch_id: usize,
 953    allow_server_restarts: bool,
 954    allow_client_reconnection: bool,
 955    allow_client_disconnection: bool,
 956}
 957
 958struct UserTestPlan {
 959    user_id: UserId,
 960    username: String,
 961    next_root_id: usize,
 962    operation_ix: usize,
 963    online: bool,
 964}
 965
 966#[derive(Clone, Debug, Serialize, Deserialize)]
 967#[serde(untagged)]
 968enum StoredOperation {
 969    Server(Operation),
 970    Client {
 971        user_id: UserId,
 972        batch_id: usize,
 973        operation: ClientOperation,
 974    },
 975}
 976
 977#[derive(Clone, Debug, Serialize, Deserialize)]
 978enum Operation {
 979    AddConnection {
 980        user_id: UserId,
 981    },
 982    RemoveConnection {
 983        user_id: UserId,
 984    },
 985    BounceConnection {
 986        user_id: UserId,
 987    },
 988    RestartServer,
 989    MutateClients {
 990        batch_id: usize,
 991        #[serde(skip_serializing)]
 992        #[serde(skip_deserializing)]
 993        user_ids: Vec<UserId>,
 994        quiesce: bool,
 995    },
 996}
 997
 998#[derive(Clone, Debug, Serialize, Deserialize)]
 999enum ClientOperation {
1000    AcceptIncomingCall,
1001    RejectIncomingCall,
1002    LeaveCall,
1003    InviteContactToCall {
1004        user_id: UserId,
1005    },
1006    OpenLocalProject {
1007        first_root_name: String,
1008    },
1009    OpenRemoteProject {
1010        host_id: UserId,
1011        first_root_name: String,
1012    },
1013    AddWorktreeToProject {
1014        project_root_name: String,
1015        new_root_path: PathBuf,
1016    },
1017    CloseRemoteProject {
1018        project_root_name: String,
1019    },
1020    OpenBuffer {
1021        project_root_name: String,
1022        is_local: bool,
1023        full_path: PathBuf,
1024    },
1025    SearchProject {
1026        project_root_name: String,
1027        is_local: bool,
1028        query: String,
1029        detach: bool,
1030    },
1031    EditBuffer {
1032        project_root_name: String,
1033        is_local: bool,
1034        full_path: PathBuf,
1035        edits: Vec<(Range<usize>, Arc<str>)>,
1036    },
1037    CloseBuffer {
1038        project_root_name: String,
1039        is_local: bool,
1040        full_path: PathBuf,
1041    },
1042    SaveBuffer {
1043        project_root_name: String,
1044        is_local: bool,
1045        full_path: PathBuf,
1046        detach: bool,
1047    },
1048    RequestLspDataInBuffer {
1049        project_root_name: String,
1050        is_local: bool,
1051        full_path: PathBuf,
1052        offset: usize,
1053        kind: LspRequestKind,
1054        detach: bool,
1055    },
1056    CreateWorktreeEntry {
1057        project_root_name: String,
1058        is_local: bool,
1059        full_path: PathBuf,
1060        is_dir: bool,
1061    },
1062    CreateFsEntry {
1063        path: PathBuf,
1064        is_dir: bool,
1065    },
1066    WriteGitIndex {
1067        repo_path: PathBuf,
1068        contents: Vec<(PathBuf, String)>,
1069    },
1070}
1071
1072#[derive(Clone, Debug, Serialize, Deserialize)]
1073enum LspRequestKind {
1074    Rename,
1075    Completion,
1076    CodeAction,
1077    Definition,
1078    Highlights,
1079}
1080
1081enum TestError {
1082    Inapplicable,
1083    Other(anyhow::Error),
1084}
1085
1086impl From<anyhow::Error> for TestError {
1087    fn from(value: anyhow::Error) -> Self {
1088        Self::Other(value)
1089    }
1090}
1091
1092impl TestPlan {
1093    fn new(mut rng: StdRng, users: Vec<UserTestPlan>, max_operations: usize) -> Self {
1094        Self {
1095            replay: false,
1096            allow_server_restarts: rng.gen_bool(0.7),
1097            allow_client_reconnection: rng.gen_bool(0.7),
1098            allow_client_disconnection: rng.gen_bool(0.1),
1099            stored_operations: Vec::new(),
1100            operation_ix: 0,
1101            next_batch_id: 0,
1102            max_operations,
1103            users,
1104            rng,
1105        }
1106    }
1107
1108    fn load(&mut self, path: &Path) {
1109        let json = std::fs::read_to_string(path).unwrap();
1110        self.replay = true;
1111        let stored_operations: Vec<StoredOperation> = serde_json::from_str(&json).unwrap();
1112        self.stored_operations = stored_operations
1113            .iter()
1114            .cloned()
1115            .enumerate()
1116            .map(|(i, mut operation)| {
1117                if let StoredOperation::Server(Operation::MutateClients {
1118                    batch_id: current_batch_id,
1119                    user_ids,
1120                    ..
1121                }) = &mut operation
1122                {
1123                    assert!(user_ids.is_empty());
1124                    user_ids.extend(stored_operations[i + 1..].iter().filter_map(|operation| {
1125                        if let StoredOperation::Client {
1126                            user_id, batch_id, ..
1127                        } = operation
1128                        {
1129                            if batch_id == current_batch_id {
1130                                return Some(user_id);
1131                            }
1132                        }
1133                        None
1134                    }));
1135                    user_ids.sort_unstable();
1136                }
1137                (operation, Arc::new(AtomicBool::new(false)))
1138            })
1139            .collect()
1140    }
1141
1142    fn save(&mut self, path: &Path) {
1143        // Format each operation as one line
1144        let mut json = Vec::new();
1145        json.push(b'[');
1146        for (operation, skipped) in &self.stored_operations {
1147            if skipped.load(SeqCst) {
1148                continue;
1149            }
1150            if json.len() > 1 {
1151                json.push(b',');
1152            }
1153            json.extend_from_slice(b"\n  ");
1154            serde_json::to_writer(&mut json, operation).unwrap();
1155        }
1156        json.extend_from_slice(b"\n]\n");
1157        std::fs::write(path, &json).unwrap();
1158    }
1159
1160    fn next_server_operation(
1161        &mut self,
1162        clients: &[(Rc<TestClient>, TestAppContext)],
1163    ) -> Option<(Operation, Arc<AtomicBool>)> {
1164        if self.replay {
1165            while let Some(stored_operation) = self.stored_operations.get(self.operation_ix) {
1166                self.operation_ix += 1;
1167                if let (StoredOperation::Server(operation), skipped) = stored_operation {
1168                    return Some((operation.clone(), skipped.clone()));
1169                }
1170            }
1171            None
1172        } else {
1173            let operation = self.generate_server_operation(clients)?;
1174            let skipped = Arc::new(AtomicBool::new(false));
1175            self.stored_operations
1176                .push((StoredOperation::Server(operation.clone()), skipped.clone()));
1177            Some((operation, skipped))
1178        }
1179    }
1180
1181    fn next_client_operation(
1182        &mut self,
1183        client: &TestClient,
1184        current_batch_id: usize,
1185        cx: &TestAppContext,
1186    ) -> Option<(ClientOperation, Arc<AtomicBool>)> {
1187        let current_user_id = client.current_user_id(cx);
1188        let user_ix = self
1189            .users
1190            .iter()
1191            .position(|user| user.user_id == current_user_id)
1192            .unwrap();
1193        let user_plan = &mut self.users[user_ix];
1194
1195        if self.replay {
1196            while let Some(stored_operation) = self.stored_operations.get(user_plan.operation_ix) {
1197                user_plan.operation_ix += 1;
1198                if let (
1199                    StoredOperation::Client {
1200                        user_id, operation, ..
1201                    },
1202                    skipped,
1203                ) = stored_operation
1204                {
1205                    if user_id == &current_user_id {
1206                        return Some((operation.clone(), skipped.clone()));
1207                    }
1208                }
1209            }
1210            None
1211        } else {
1212            let operation = self.generate_client_operation(current_user_id, client, cx)?;
1213            let skipped = Arc::new(AtomicBool::new(false));
1214            self.stored_operations.push((
1215                StoredOperation::Client {
1216                    user_id: current_user_id,
1217                    batch_id: current_batch_id,
1218                    operation: operation.clone(),
1219                },
1220                skipped.clone(),
1221            ));
1222            Some((operation, skipped))
1223        }
1224    }
1225
1226    fn generate_server_operation(
1227        &mut self,
1228        clients: &[(Rc<TestClient>, TestAppContext)],
1229    ) -> Option<Operation> {
1230        if self.operation_ix == self.max_operations {
1231            return None;
1232        }
1233
1234        Some(loop {
1235            break match self.rng.gen_range(0..100) {
1236                0..=29 if clients.len() < self.users.len() => {
1237                    let user = self
1238                        .users
1239                        .iter()
1240                        .filter(|u| !u.online)
1241                        .choose(&mut self.rng)
1242                        .unwrap();
1243                    self.operation_ix += 1;
1244                    Operation::AddConnection {
1245                        user_id: user.user_id,
1246                    }
1247                }
1248                30..=34 if clients.len() > 1 && self.allow_client_disconnection => {
1249                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1250                    let user_id = client.current_user_id(cx);
1251                    self.operation_ix += 1;
1252                    Operation::RemoveConnection { user_id }
1253                }
1254                35..=39 if clients.len() > 1 && self.allow_client_reconnection => {
1255                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1256                    let user_id = client.current_user_id(cx);
1257                    self.operation_ix += 1;
1258                    Operation::BounceConnection { user_id }
1259                }
1260                40..=44 if self.allow_server_restarts && clients.len() > 1 => {
1261                    self.operation_ix += 1;
1262                    Operation::RestartServer
1263                }
1264                _ if !clients.is_empty() => {
1265                    let count = self
1266                        .rng
1267                        .gen_range(1..10)
1268                        .min(self.max_operations - self.operation_ix);
1269                    let batch_id = util::post_inc(&mut self.next_batch_id);
1270                    let mut user_ids = (0..count)
1271                        .map(|_| {
1272                            let ix = self.rng.gen_range(0..clients.len());
1273                            let (client, cx) = &clients[ix];
1274                            client.current_user_id(cx)
1275                        })
1276                        .collect::<Vec<_>>();
1277                    user_ids.sort_unstable();
1278                    Operation::MutateClients {
1279                        user_ids,
1280                        batch_id,
1281                        quiesce: self.rng.gen_bool(0.7),
1282                    }
1283                }
1284                _ => continue,
1285            };
1286        })
1287    }
1288
1289    fn generate_client_operation(
1290        &mut self,
1291        user_id: UserId,
1292        client: &TestClient,
1293        cx: &TestAppContext,
1294    ) -> Option<ClientOperation> {
1295        if self.operation_ix == self.max_operations {
1296            return None;
1297        }
1298
1299        let executor = cx.background();
1300        self.operation_ix += 1;
1301        let call = cx.read(ActiveCall::global);
1302        Some(loop {
1303            match self.rng.gen_range(0..100_u32) {
1304                // Mutate the call
1305                0..=29 => {
1306                    // Respond to an incoming call
1307                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
1308                        break if self.rng.gen_bool(0.7) {
1309                            ClientOperation::AcceptIncomingCall
1310                        } else {
1311                            ClientOperation::RejectIncomingCall
1312                        };
1313                    }
1314
1315                    match self.rng.gen_range(0..100_u32) {
1316                        // Invite a contact to the current call
1317                        0..=70 => {
1318                            let available_contacts =
1319                                client.user_store.read_with(cx, |user_store, _| {
1320                                    user_store
1321                                        .contacts()
1322                                        .iter()
1323                                        .filter(|contact| contact.online && !contact.busy)
1324                                        .cloned()
1325                                        .collect::<Vec<_>>()
1326                                });
1327                            if !available_contacts.is_empty() {
1328                                let contact = available_contacts.choose(&mut self.rng).unwrap();
1329                                break ClientOperation::InviteContactToCall {
1330                                    user_id: UserId(contact.user.id as i32),
1331                                };
1332                            }
1333                        }
1334
1335                        // Leave the current call
1336                        71.. => {
1337                            if self.allow_client_disconnection
1338                                && call.read_with(cx, |call, _| call.room().is_some())
1339                            {
1340                                break ClientOperation::LeaveCall;
1341                            }
1342                        }
1343                    }
1344                }
1345
1346                // Mutate projects
1347                30..=59 => match self.rng.gen_range(0..100_u32) {
1348                    // Open a new project
1349                    0..=70 => {
1350                        // Open a remote project
1351                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
1352                            let existing_remote_project_ids = cx.read(|cx| {
1353                                client
1354                                    .remote_projects()
1355                                    .iter()
1356                                    .map(|p| p.read(cx).remote_id().unwrap())
1357                                    .collect::<Vec<_>>()
1358                            });
1359                            let new_remote_projects = room.read_with(cx, |room, _| {
1360                                room.remote_participants()
1361                                    .values()
1362                                    .flat_map(|participant| {
1363                                        participant.projects.iter().filter_map(|project| {
1364                                            if existing_remote_project_ids.contains(&project.id) {
1365                                                None
1366                                            } else {
1367                                                Some((
1368                                                    UserId::from_proto(participant.user.id),
1369                                                    project.worktree_root_names[0].clone(),
1370                                                ))
1371                                            }
1372                                        })
1373                                    })
1374                                    .collect::<Vec<_>>()
1375                            });
1376                            if !new_remote_projects.is_empty() {
1377                                let (host_id, first_root_name) =
1378                                    new_remote_projects.choose(&mut self.rng).unwrap().clone();
1379                                break ClientOperation::OpenRemoteProject {
1380                                    host_id,
1381                                    first_root_name,
1382                                };
1383                            }
1384                        }
1385                        // Open a local project
1386                        else {
1387                            let first_root_name = self.next_root_dir_name(user_id);
1388                            break ClientOperation::OpenLocalProject { first_root_name };
1389                        }
1390                    }
1391
1392                    // Close a remote project
1393                    71..=80 => {
1394                        if !client.remote_projects().is_empty() {
1395                            let project = client
1396                                .remote_projects()
1397                                .choose(&mut self.rng)
1398                                .unwrap()
1399                                .clone();
1400                            let first_root_name = root_name_for_project(&project, cx);
1401                            break ClientOperation::CloseRemoteProject {
1402                                project_root_name: first_root_name,
1403                            };
1404                        }
1405                    }
1406
1407                    // Mutate project worktrees
1408                    81.. => match self.rng.gen_range(0..100_u32) {
1409                        // Add a worktree to a local project
1410                        0..=50 => {
1411                            let Some(project) = client
1412                                .local_projects()
1413                                .choose(&mut self.rng)
1414                                .cloned() else { continue };
1415                            let project_root_name = root_name_for_project(&project, cx);
1416                            let mut paths = executor.block(client.fs.paths());
1417                            paths.remove(0);
1418                            let new_root_path = if paths.is_empty() || self.rng.gen() {
1419                                Path::new("/").join(&self.next_root_dir_name(user_id))
1420                            } else {
1421                                paths.choose(&mut self.rng).unwrap().clone()
1422                            };
1423                            break ClientOperation::AddWorktreeToProject {
1424                                project_root_name,
1425                                new_root_path,
1426                            };
1427                        }
1428
1429                        // Add an entry to a worktree
1430                        _ => {
1431                            let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1432                            let project_root_name = root_name_for_project(&project, cx);
1433                            let is_local = project.read_with(cx, |project, _| project.is_local());
1434                            let worktree = project.read_with(cx, |project, cx| {
1435                                project
1436                                    .worktrees(cx)
1437                                    .filter(|worktree| {
1438                                        let worktree = worktree.read(cx);
1439                                        worktree.is_visible()
1440                                            && worktree.entries(false).any(|e| e.is_file())
1441                                            && worktree.root_entry().map_or(false, |e| e.is_dir())
1442                                    })
1443                                    .choose(&mut self.rng)
1444                            });
1445                            let Some(worktree) = worktree else { continue };
1446                            let is_dir = self.rng.gen::<bool>();
1447                            let mut full_path =
1448                                worktree.read_with(cx, |w, _| PathBuf::from(w.root_name()));
1449                            full_path.push(gen_file_name(&mut self.rng));
1450                            if !is_dir {
1451                                full_path.set_extension("rs");
1452                            }
1453                            break ClientOperation::CreateWorktreeEntry {
1454                                project_root_name,
1455                                is_local,
1456                                full_path,
1457                                is_dir,
1458                            };
1459                        }
1460                    },
1461                },
1462
1463                // Query and mutate buffers
1464                60..=90 => {
1465                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1466                    let project_root_name = root_name_for_project(&project, cx);
1467                    let is_local = project.read_with(cx, |project, _| project.is_local());
1468
1469                    match self.rng.gen_range(0..100_u32) {
1470                        // Manipulate an existing buffer
1471                        0..=70 => {
1472                            let Some(buffer) = client
1473                                .buffers_for_project(&project)
1474                                .iter()
1475                                .choose(&mut self.rng)
1476                                .cloned() else { continue };
1477
1478                            let full_path = buffer
1479                                .read_with(cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
1480
1481                            match self.rng.gen_range(0..100_u32) {
1482                                // Close the buffer
1483                                0..=15 => {
1484                                    break ClientOperation::CloseBuffer {
1485                                        project_root_name,
1486                                        is_local,
1487                                        full_path,
1488                                    };
1489                                }
1490                                // Save the buffer
1491                                16..=29 if buffer.read_with(cx, |b, _| b.is_dirty()) => {
1492                                    let detach = self.rng.gen_bool(0.3);
1493                                    break ClientOperation::SaveBuffer {
1494                                        project_root_name,
1495                                        is_local,
1496                                        full_path,
1497                                        detach,
1498                                    };
1499                                }
1500                                // Edit the buffer
1501                                30..=69 => {
1502                                    let edits = buffer.read_with(cx, |buffer, _| {
1503                                        buffer.get_random_edits(&mut self.rng, 3)
1504                                    });
1505                                    break ClientOperation::EditBuffer {
1506                                        project_root_name,
1507                                        is_local,
1508                                        full_path,
1509                                        edits,
1510                                    };
1511                                }
1512                                // Make an LSP request
1513                                _ => {
1514                                    let offset = buffer.read_with(cx, |buffer, _| {
1515                                        buffer.clip_offset(
1516                                            self.rng.gen_range(0..=buffer.len()),
1517                                            language::Bias::Left,
1518                                        )
1519                                    });
1520                                    let detach = self.rng.gen();
1521                                    break ClientOperation::RequestLspDataInBuffer {
1522                                        project_root_name,
1523                                        full_path,
1524                                        offset,
1525                                        is_local,
1526                                        kind: match self.rng.gen_range(0..5_u32) {
1527                                            0 => LspRequestKind::Rename,
1528                                            1 => LspRequestKind::Highlights,
1529                                            2 => LspRequestKind::Definition,
1530                                            3 => LspRequestKind::CodeAction,
1531                                            4.. => LspRequestKind::Completion,
1532                                        },
1533                                        detach,
1534                                    };
1535                                }
1536                            }
1537                        }
1538
1539                        71..=80 => {
1540                            let query = self.rng.gen_range('a'..='z').to_string();
1541                            let detach = self.rng.gen_bool(0.3);
1542                            break ClientOperation::SearchProject {
1543                                project_root_name,
1544                                is_local,
1545                                query,
1546                                detach,
1547                            };
1548                        }
1549
1550                        // Open a buffer
1551                        81.. => {
1552                            let worktree = project.read_with(cx, |project, cx| {
1553                                project
1554                                    .worktrees(cx)
1555                                    .filter(|worktree| {
1556                                        let worktree = worktree.read(cx);
1557                                        worktree.is_visible()
1558                                            && worktree.entries(false).any(|e| e.is_file())
1559                                    })
1560                                    .choose(&mut self.rng)
1561                            });
1562                            let Some(worktree) = worktree else { continue };
1563                            let full_path = worktree.read_with(cx, |worktree, _| {
1564                                let entry = worktree
1565                                    .entries(false)
1566                                    .filter(|e| e.is_file())
1567                                    .choose(&mut self.rng)
1568                                    .unwrap();
1569                                if entry.path.as_ref() == Path::new("") {
1570                                    Path::new(worktree.root_name()).into()
1571                                } else {
1572                                    Path::new(worktree.root_name()).join(&entry.path)
1573                                }
1574                            });
1575                            break ClientOperation::OpenBuffer {
1576                                project_root_name,
1577                                is_local,
1578                                full_path,
1579                            };
1580                        }
1581                    }
1582                }
1583
1584                // Update a git index
1585                91..=95 => {
1586                    let repo_path = executor
1587                        .block(client.fs.directories())
1588                        .choose(&mut self.rng)
1589                        .unwrap()
1590                        .clone();
1591
1592                    let mut file_paths = executor
1593                        .block(client.fs.files())
1594                        .into_iter()
1595                        .filter(|path| path.starts_with(&repo_path))
1596                        .collect::<Vec<_>>();
1597                    let count = self.rng.gen_range(0..=file_paths.len());
1598                    file_paths.shuffle(&mut self.rng);
1599                    file_paths.truncate(count);
1600
1601                    let mut contents = Vec::new();
1602                    for abs_child_file_path in &file_paths {
1603                        let child_file_path = abs_child_file_path
1604                            .strip_prefix(&repo_path)
1605                            .unwrap()
1606                            .to_path_buf();
1607                        let new_base = Alphanumeric.sample_string(&mut self.rng, 16);
1608                        contents.push((child_file_path, new_base));
1609                    }
1610
1611                    break ClientOperation::WriteGitIndex {
1612                        repo_path,
1613                        contents,
1614                    };
1615                }
1616
1617                // Create a file or directory
1618                96.. => {
1619                    let is_dir = self.rng.gen::<bool>();
1620                    let mut path = cx
1621                        .background()
1622                        .block(client.fs.directories())
1623                        .choose(&mut self.rng)
1624                        .unwrap()
1625                        .clone();
1626                    path.push(gen_file_name(&mut self.rng));
1627                    if !is_dir {
1628                        path.set_extension("rs");
1629                    }
1630                    break ClientOperation::CreateFsEntry { path, is_dir };
1631                }
1632            }
1633        })
1634    }
1635
1636    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
1637        let user_ix = self
1638            .users
1639            .iter()
1640            .position(|user| user.user_id == user_id)
1641            .unwrap();
1642        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
1643        format!("dir-{user_id}-{root_id}")
1644    }
1645
1646    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
1647        let ix = self
1648            .users
1649            .iter()
1650            .position(|user| user.user_id == user_id)
1651            .unwrap();
1652        &mut self.users[ix]
1653    }
1654}
1655
1656async fn simulate_client(
1657    client: Rc<TestClient>,
1658    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<usize>,
1659    plan: Arc<Mutex<TestPlan>>,
1660    mut cx: TestAppContext,
1661) {
1662    // Setup language server
1663    let mut language = Language::new(
1664        LanguageConfig {
1665            name: "Rust".into(),
1666            path_suffixes: vec!["rs".to_string()],
1667            ..Default::default()
1668        },
1669        None,
1670    );
1671    let _fake_language_servers = language
1672        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
1673            name: "the-fake-language-server",
1674            capabilities: lsp::LanguageServer::full_capabilities(),
1675            initializer: Some(Box::new({
1676                let plan = plan.clone();
1677                let fs = client.fs.clone();
1678                move |fake_server: &mut FakeLanguageServer| {
1679                    fake_server.handle_request::<lsp::request::Completion, _, _>(
1680                        |_, _| async move {
1681                            Ok(Some(lsp::CompletionResponse::Array(vec![
1682                                lsp::CompletionItem {
1683                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
1684                                        range: lsp::Range::new(
1685                                            lsp::Position::new(0, 0),
1686                                            lsp::Position::new(0, 0),
1687                                        ),
1688                                        new_text: "the-new-text".to_string(),
1689                                    })),
1690                                    ..Default::default()
1691                                },
1692                            ])))
1693                        },
1694                    );
1695
1696                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
1697                        |_, _| async move {
1698                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
1699                                lsp::CodeAction {
1700                                    title: "the-code-action".to_string(),
1701                                    ..Default::default()
1702                                },
1703                            )]))
1704                        },
1705                    );
1706
1707                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
1708                        |params, _| async move {
1709                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
1710                                params.position,
1711                                params.position,
1712                            ))))
1713                        },
1714                    );
1715
1716                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
1717                        let fs = fs.clone();
1718                        let plan = plan.clone();
1719                        move |_, _| {
1720                            let fs = fs.clone();
1721                            let plan = plan.clone();
1722                            async move {
1723                                let files = fs.files().await;
1724                                let count = plan.lock().rng.gen_range::<usize, _>(1..3);
1725                                let files = (0..count)
1726                                    .map(|_| files.choose(&mut plan.lock().rng).unwrap())
1727                                    .collect::<Vec<_>>();
1728                                log::info!("LSP: Returning definitions in files {:?}", &files);
1729                                Ok(Some(lsp::GotoDefinitionResponse::Array(
1730                                    files
1731                                        .into_iter()
1732                                        .map(|file| lsp::Location {
1733                                            uri: lsp::Url::from_file_path(file).unwrap(),
1734                                            range: Default::default(),
1735                                        })
1736                                        .collect(),
1737                                )))
1738                            }
1739                        }
1740                    });
1741
1742                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
1743                        let plan = plan.clone();
1744                        move |_, _| {
1745                            let mut highlights = Vec::new();
1746                            let highlight_count = plan.lock().rng.gen_range(1..=5);
1747                            for _ in 0..highlight_count {
1748                                let start_row = plan.lock().rng.gen_range(0..100);
1749                                let start_column = plan.lock().rng.gen_range(0..100);
1750                                let start = PointUtf16::new(start_row, start_column);
1751                                let end_row = plan.lock().rng.gen_range(0..100);
1752                                let end_column = plan.lock().rng.gen_range(0..100);
1753                                let end = PointUtf16::new(end_row, end_column);
1754                                let range = if start > end { end..start } else { start..end };
1755                                highlights.push(lsp::DocumentHighlight {
1756                                    range: range_to_lsp(range.clone()),
1757                                    kind: Some(lsp::DocumentHighlightKind::READ),
1758                                });
1759                            }
1760                            highlights.sort_unstable_by_key(|highlight| {
1761                                (highlight.range.start, highlight.range.end)
1762                            });
1763                            async move { Ok(Some(highlights)) }
1764                        }
1765                    });
1766                }
1767            })),
1768            ..Default::default()
1769        }))
1770        .await;
1771    client.language_registry.add(Arc::new(language));
1772
1773    while let Some(batch_id) = operation_rx.next().await {
1774        let Some((operation, skipped)) = plan.lock().next_client_operation(&client, batch_id, &cx) else { break };
1775        match apply_client_operation(&client, operation, &mut cx).await {
1776            Ok(()) => {}
1777            Err(TestError::Inapplicable) => skipped.store(true, SeqCst),
1778            Err(TestError::Other(error)) => {
1779                log::error!("{} error: {}", client.username, error);
1780            }
1781        }
1782        cx.background().simulate_random_delay().await;
1783    }
1784    log::info!("{}: done", client.username);
1785}
1786
1787fn buffer_for_full_path(
1788    client: &TestClient,
1789    project: &ModelHandle<Project>,
1790    full_path: &PathBuf,
1791    cx: &TestAppContext,
1792) -> Option<ModelHandle<language::Buffer>> {
1793    client
1794        .buffers_for_project(project)
1795        .iter()
1796        .find(|buffer| {
1797            buffer.read_with(cx, |buffer, cx| {
1798                buffer.file().unwrap().full_path(cx) == *full_path
1799            })
1800        })
1801        .cloned()
1802}
1803
1804fn project_for_root_name(
1805    client: &TestClient,
1806    root_name: &str,
1807    cx: &TestAppContext,
1808) -> Option<ModelHandle<Project>> {
1809    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1810        return Some(client.local_projects()[ix].clone());
1811    }
1812    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1813        return Some(client.remote_projects()[ix].clone());
1814    }
1815    None
1816}
1817
1818fn project_ix_for_root_name(
1819    projects: &[ModelHandle<Project>],
1820    root_name: &str,
1821    cx: &TestAppContext,
1822) -> Option<usize> {
1823    projects.iter().position(|project| {
1824        project.read_with(cx, |project, cx| {
1825            let worktree = project.visible_worktrees(cx).next().unwrap();
1826            worktree.read(cx).root_name() == root_name
1827        })
1828    })
1829}
1830
1831fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1832    project.read_with(cx, |project, cx| {
1833        project
1834            .visible_worktrees(cx)
1835            .next()
1836            .unwrap()
1837            .read(cx)
1838            .root_name()
1839            .to_string()
1840    })
1841}
1842
1843fn project_path_for_full_path(
1844    project: &ModelHandle<Project>,
1845    full_path: &Path,
1846    cx: &TestAppContext,
1847) -> Option<ProjectPath> {
1848    let mut components = full_path.components();
1849    let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1850    let path = components.as_path().into();
1851    let worktree_id = project.read_with(cx, |project, cx| {
1852        project.worktrees(cx).find_map(|worktree| {
1853            let worktree = worktree.read(cx);
1854            if worktree.root_name() == root_name {
1855                Some(worktree.id())
1856            } else {
1857                None
1858            }
1859        })
1860    })?;
1861    Some(ProjectPath { worktree_id, path })
1862}
1863
1864async fn ensure_project_shared(
1865    project: &ModelHandle<Project>,
1866    client: &TestClient,
1867    cx: &mut TestAppContext,
1868) {
1869    let first_root_name = root_name_for_project(project, cx);
1870    let active_call = cx.read(ActiveCall::global);
1871    if active_call.read_with(cx, |call, _| call.room().is_some())
1872        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
1873    {
1874        match active_call
1875            .update(cx, |call, cx| call.share_project(project.clone(), cx))
1876            .await
1877        {
1878            Ok(project_id) => {
1879                log::info!(
1880                    "{}: shared project {} with id {}",
1881                    client.username,
1882                    first_root_name,
1883                    project_id
1884                );
1885            }
1886            Err(error) => {
1887                log::error!(
1888                    "{}: error sharing project {}: {:?}",
1889                    client.username,
1890                    first_root_name,
1891                    error
1892                );
1893            }
1894        }
1895    }
1896}
1897
1898fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1899    client
1900        .local_projects()
1901        .iter()
1902        .chain(client.remote_projects().iter())
1903        .choose(rng)
1904        .cloned()
1905}
1906
1907fn gen_file_name(rng: &mut StdRng) -> String {
1908    let mut name = String::new();
1909    for _ in 0..10 {
1910        let letter = rng.gen_range('a'..='z');
1911        name.push(letter);
1912    }
1913    name
1914}
1915
1916fn path_env_var(name: &str) -> Option<PathBuf> {
1917    let value = env::var(name).ok()?;
1918    let mut path = PathBuf::from(value);
1919    if path.is_relative() {
1920        let mut abs_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
1921        abs_path.pop();
1922        abs_path.pop();
1923        abs_path.push(path);
1924        path = abs_path
1925    }
1926    Some(path)
1927}