1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use editor::Bias;
  11use fs::{FakeFs, Fs as _};
  12use futures::StreamExt as _;
  13use gpui::{executor::Deterministic, ModelHandle, Task, TestAppContext};
  14use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  15use lsp::FakeLanguageServer;
  16use parking_lot::Mutex;
  17use project::{search::SearchQuery, Project, ProjectPath};
  18use rand::{
  19    distributions::{Alphanumeric, DistString},
  20    prelude::*,
  21};
  22use serde::{Deserialize, Serialize};
  23use settings::Settings;
  24use std::{
  25    env,
  26    ops::Range,
  27    path::{Path, PathBuf},
  28    rc::Rc,
  29    sync::{
  30        atomic::{AtomicBool, Ordering::SeqCst},
  31        Arc,
  32    },
  33};
  34use util::ResultExt;
  35
  36lazy_static::lazy_static! {
  37    static ref PLAN_LOAD_PATH: Option<PathBuf> = path_env_var("LOAD_PLAN");
  38    static ref PLAN_SAVE_PATH: Option<PathBuf> = path_env_var("SAVE_PLAN");
  39    static ref LOADED_PLAN_JSON: Mutex<Option<Vec<u8>>> = Default::default();
  40    static ref PLAN: Mutex<Option<Arc<Mutex<TestPlan>>>> = Default::default();
  41}
  42
  43#[gpui::test(iterations = 100, on_failure = "on_failure")]
  44async fn test_random_collaboration(
  45    cx: &mut TestAppContext,
  46    deterministic: Arc<Deterministic>,
  47    rng: StdRng,
  48) {
  49    deterministic.forbid_parking();
  50
  51    let max_peers = env::var("MAX_PEERS")
  52        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  53        .unwrap_or(3);
  54    let max_operations = env::var("OPERATIONS")
  55        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  56        .unwrap_or(10);
  57
  58    let mut server = TestServer::start(&deterministic).await;
  59    let db = server.app_state.db.clone();
  60
  61    let mut users = Vec::new();
  62    for ix in 0..max_peers {
  63        let username = format!("user-{}", ix + 1);
  64        let user_id = db
  65            .create_user(
  66                &format!("{username}@example.com"),
  67                false,
  68                NewUserParams {
  69                    github_login: username.clone(),
  70                    github_user_id: (ix + 1) as i32,
  71                    invite_count: 0,
  72                },
  73            )
  74            .await
  75            .unwrap()
  76            .user_id;
  77        users.push(UserTestPlan {
  78            user_id,
  79            username,
  80            online: false,
  81            next_root_id: 0,
  82            operation_ix: 0,
  83        });
  84    }
  85
  86    for (ix, user_a) in users.iter().enumerate() {
  87        for user_b in &users[ix + 1..] {
  88            server
  89                .app_state
  90                .db
  91                .send_contact_request(user_a.user_id, user_b.user_id)
  92                .await
  93                .unwrap();
  94            server
  95                .app_state
  96                .db
  97                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  98                .await
  99                .unwrap();
 100        }
 101    }
 102
 103    let plan = Arc::new(Mutex::new(TestPlan::new(rng, users, max_operations)));
 104
 105    if let Some(path) = &*PLAN_LOAD_PATH {
 106        let json = LOADED_PLAN_JSON
 107            .lock()
 108            .get_or_insert_with(|| {
 109                eprintln!("loaded test plan from path {:?}", path);
 110                std::fs::read(path).unwrap()
 111            })
 112            .clone();
 113        plan.lock().deserialize(json);
 114    }
 115
 116    PLAN.lock().replace(plan.clone());
 117
 118    let mut clients = Vec::new();
 119    let mut client_tasks = Vec::new();
 120    let mut operation_channels = Vec::new();
 121
 122    loop {
 123        let Some((next_operation, applied)) = plan.lock().next_server_operation(&clients) else { break };
 124        let did_apply = apply_server_operation(
 125            deterministic.clone(),
 126            &mut server,
 127            &mut clients,
 128            &mut client_tasks,
 129            &mut operation_channels,
 130            plan.clone(),
 131            next_operation,
 132            cx,
 133        )
 134        .await;
 135        if did_apply {
 136            applied.store(true, SeqCst);
 137        }
 138    }
 139
 140    drop(operation_channels);
 141    deterministic.start_waiting();
 142    futures::future::join_all(client_tasks).await;
 143    deterministic.finish_waiting();
 144    deterministic.run_until_parked();
 145
 146    check_consistency_between_clients(&clients);
 147
 148    for (client, mut cx) in clients {
 149        cx.update(|cx| {
 150            cx.clear_globals();
 151            cx.set_global(Settings::test(cx));
 152            drop(client);
 153        });
 154    }
 155
 156    deterministic.run_until_parked();
 157}
 158
 159fn on_failure() {
 160    if let Some(plan) = PLAN.lock().clone() {
 161        if let Some(path) = &*PLAN_SAVE_PATH {
 162            eprintln!("saved test plan to path {:?}", path);
 163            std::fs::write(path, plan.lock().serialize()).unwrap();
 164        }
 165    }
 166}
 167
 168async fn apply_server_operation(
 169    deterministic: Arc<Deterministic>,
 170    server: &mut TestServer,
 171    clients: &mut Vec<(Rc<TestClient>, TestAppContext)>,
 172    client_tasks: &mut Vec<Task<()>>,
 173    operation_channels: &mut Vec<futures::channel::mpsc::UnboundedSender<usize>>,
 174    plan: Arc<Mutex<TestPlan>>,
 175    operation: Operation,
 176    cx: &mut TestAppContext,
 177) -> bool {
 178    match operation {
 179        Operation::AddConnection { user_id } => {
 180            let username;
 181            {
 182                let mut plan = plan.lock();
 183                let mut user = plan.user(user_id);
 184                if user.online {
 185                    return false;
 186                }
 187                user.online = true;
 188                username = user.username.clone();
 189            };
 190            log::info!("Adding new connection for {}", username);
 191            let next_entity_id = (user_id.0 * 10_000) as usize;
 192            let mut client_cx = TestAppContext::new(
 193                cx.foreground_platform(),
 194                cx.platform(),
 195                deterministic.build_foreground(user_id.0 as usize),
 196                deterministic.build_background(),
 197                cx.font_cache(),
 198                cx.leak_detector(),
 199                next_entity_id,
 200                cx.function_name.clone(),
 201            );
 202
 203            let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 204            let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 205            operation_channels.push(operation_tx);
 206            clients.push((client.clone(), client_cx.clone()));
 207            client_tasks.push(client_cx.foreground().spawn(simulate_client(
 208                client,
 209                operation_rx,
 210                plan.clone(),
 211                client_cx,
 212            )));
 213
 214            log::info!("Added connection for {}", username);
 215        }
 216
 217        Operation::RemoveConnection {
 218            user_id: removed_user_id,
 219        } => {
 220            log::info!("Simulating full disconnection of user {}", removed_user_id);
 221            let client_ix = clients
 222                .iter()
 223                .position(|(client, cx)| client.current_user_id(cx) == removed_user_id);
 224            let Some(client_ix) = client_ix else { return false };
 225            let user_connection_ids = server
 226                .connection_pool
 227                .lock()
 228                .user_connection_ids(removed_user_id)
 229                .collect::<Vec<_>>();
 230            assert_eq!(user_connection_ids.len(), 1);
 231            let removed_peer_id = user_connection_ids[0].into();
 232            let (client, mut client_cx) = clients.remove(client_ix);
 233            let client_task = client_tasks.remove(client_ix);
 234            operation_channels.remove(client_ix);
 235            server.forbid_connections();
 236            server.disconnect_client(removed_peer_id);
 237            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 238            deterministic.start_waiting();
 239            log::info!("Waiting for user {} to exit...", removed_user_id);
 240            client_task.await;
 241            deterministic.finish_waiting();
 242            server.allow_connections();
 243
 244            for project in client.remote_projects().iter() {
 245                project.read_with(&client_cx, |project, _| {
 246                    assert!(
 247                        project.is_read_only(),
 248                        "project {:?} should be read only",
 249                        project.remote_id()
 250                    )
 251                });
 252            }
 253
 254            for (client, cx) in clients {
 255                let contacts = server
 256                    .app_state
 257                    .db
 258                    .get_contacts(client.current_user_id(cx))
 259                    .await
 260                    .unwrap();
 261                let pool = server.connection_pool.lock();
 262                for contact in contacts {
 263                    if let db::Contact::Accepted { user_id, busy, .. } = contact {
 264                        if user_id == removed_user_id {
 265                            assert!(!pool.is_user_online(user_id));
 266                            assert!(!busy);
 267                        }
 268                    }
 269                }
 270            }
 271
 272            log::info!("{} removed", client.username);
 273            plan.lock().user(removed_user_id).online = false;
 274            client_cx.update(|cx| {
 275                cx.clear_globals();
 276                drop(client);
 277            });
 278        }
 279
 280        Operation::BounceConnection { user_id } => {
 281            log::info!("Simulating temporary disconnection of user {}", user_id);
 282            let user_connection_ids = server
 283                .connection_pool
 284                .lock()
 285                .user_connection_ids(user_id)
 286                .collect::<Vec<_>>();
 287            if user_connection_ids.is_empty() {
 288                return false;
 289            }
 290            assert_eq!(user_connection_ids.len(), 1);
 291            let peer_id = user_connection_ids[0].into();
 292            server.disconnect_client(peer_id);
 293            deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 294        }
 295
 296        Operation::RestartServer => {
 297            log::info!("Simulating server restart");
 298            server.reset().await;
 299            deterministic.advance_clock(RECEIVE_TIMEOUT);
 300            server.start().await.unwrap();
 301            deterministic.advance_clock(CLEANUP_TIMEOUT);
 302            let environment = &server.app_state.config.zed_environment;
 303            let stale_room_ids = server
 304                .app_state
 305                .db
 306                .stale_room_ids(environment, server.id())
 307                .await
 308                .unwrap();
 309            assert_eq!(stale_room_ids, vec![]);
 310        }
 311
 312        Operation::MutateClients {
 313            user_ids,
 314            batch_id,
 315            quiesce,
 316        } => {
 317            let mut applied = false;
 318            for user_id in user_ids {
 319                let client_ix = clients
 320                    .iter()
 321                    .position(|(client, cx)| client.current_user_id(cx) == user_id);
 322                let Some(client_ix) = client_ix else { continue };
 323                applied = true;
 324                if let Err(err) = operation_channels[client_ix].unbounded_send(batch_id) {
 325                    log::error!("error signaling user {user_id}: {err}");
 326                }
 327            }
 328
 329            if quiesce && applied {
 330                deterministic.run_until_parked();
 331                check_consistency_between_clients(&clients);
 332            }
 333
 334            return applied;
 335        }
 336    }
 337    true
 338}
 339
 340async fn apply_client_operation(
 341    client: &TestClient,
 342    operation: ClientOperation,
 343    cx: &mut TestAppContext,
 344) -> Result<(), TestError> {
 345    match operation {
 346        ClientOperation::AcceptIncomingCall => {
 347            let active_call = cx.read(ActiveCall::global);
 348            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 349                Err(TestError::Inapplicable)?;
 350            }
 351
 352            log::info!("{}: accepting incoming call", client.username);
 353            active_call
 354                .update(cx, |call, cx| call.accept_incoming(cx))
 355                .await?;
 356        }
 357
 358        ClientOperation::RejectIncomingCall => {
 359            let active_call = cx.read(ActiveCall::global);
 360            if active_call.read_with(cx, |call, _| call.incoming().borrow().is_none()) {
 361                Err(TestError::Inapplicable)?;
 362            }
 363
 364            log::info!("{}: declining incoming call", client.username);
 365            active_call.update(cx, |call, _| call.decline_incoming())?;
 366        }
 367
 368        ClientOperation::LeaveCall => {
 369            let active_call = cx.read(ActiveCall::global);
 370            if active_call.read_with(cx, |call, _| call.room().is_none()) {
 371                Err(TestError::Inapplicable)?;
 372            }
 373
 374            log::info!("{}: hanging up", client.username);
 375            active_call.update(cx, |call, cx| call.hang_up(cx)).await?;
 376        }
 377
 378        ClientOperation::InviteContactToCall { user_id } => {
 379            let active_call = cx.read(ActiveCall::global);
 380
 381            log::info!("{}: inviting {}", client.username, user_id,);
 382            active_call
 383                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 384                .await
 385                .log_err();
 386        }
 387
 388        ClientOperation::OpenLocalProject { first_root_name } => {
 389            log::info!(
 390                "{}: opening local project at {:?}",
 391                client.username,
 392                first_root_name
 393            );
 394
 395            let root_path = Path::new("/").join(&first_root_name);
 396            client.fs.create_dir(&root_path).await.unwrap();
 397            client
 398                .fs
 399                .create_file(&root_path.join("main.rs"), Default::default())
 400                .await
 401                .unwrap();
 402            let project = client.build_local_project(root_path, cx).await.0;
 403            ensure_project_shared(&project, client, cx).await;
 404            client.local_projects_mut().push(project.clone());
 405        }
 406
 407        ClientOperation::AddWorktreeToProject {
 408            project_root_name,
 409            new_root_path,
 410        } => {
 411            let project = project_for_root_name(client, &project_root_name, cx)
 412                .ok_or(TestError::Inapplicable)?;
 413
 414            log::info!(
 415                "{}: finding/creating local worktree at {:?} to project with root path {}",
 416                client.username,
 417                new_root_path,
 418                project_root_name
 419            );
 420
 421            ensure_project_shared(&project, client, cx).await;
 422            if !client.fs.paths().contains(&new_root_path) {
 423                client.fs.create_dir(&new_root_path).await.unwrap();
 424            }
 425            project
 426                .update(cx, |project, cx| {
 427                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 428                })
 429                .await
 430                .unwrap();
 431        }
 432
 433        ClientOperation::CloseRemoteProject { project_root_name } => {
 434            let project = project_for_root_name(client, &project_root_name, cx)
 435                .ok_or(TestError::Inapplicable)?;
 436
 437            log::info!(
 438                "{}: closing remote project with root path {}",
 439                client.username,
 440                project_root_name,
 441            );
 442
 443            let ix = client
 444                .remote_projects()
 445                .iter()
 446                .position(|p| p == &project)
 447                .unwrap();
 448            cx.update(|_| {
 449                client.remote_projects_mut().remove(ix);
 450                client.buffers().retain(|project, _| project != project);
 451                drop(project);
 452            });
 453        }
 454
 455        ClientOperation::OpenRemoteProject {
 456            host_id,
 457            first_root_name,
 458        } => {
 459            let active_call = cx.read(ActiveCall::global);
 460            let project = active_call
 461                .update(cx, |call, cx| {
 462                    let room = call.room().cloned()?;
 463                    let participant = room
 464                        .read(cx)
 465                        .remote_participants()
 466                        .get(&host_id.to_proto())?;
 467                    let project_id = participant
 468                        .projects
 469                        .iter()
 470                        .find(|project| project.worktree_root_names[0] == first_root_name)?
 471                        .id;
 472                    Some(room.update(cx, |room, cx| {
 473                        room.join_project(
 474                            project_id,
 475                            client.language_registry.clone(),
 476                            FakeFs::new(cx.background().clone()),
 477                            cx,
 478                        )
 479                    }))
 480                })
 481                .ok_or(TestError::Inapplicable)?;
 482
 483            log::info!(
 484                "{}: joining remote project of user {}, root name {}",
 485                client.username,
 486                host_id,
 487                first_root_name,
 488            );
 489
 490            let project = project.await?;
 491            client.remote_projects_mut().push(project.clone());
 492        }
 493
 494        ClientOperation::CreateWorktreeEntry {
 495            project_root_name,
 496            is_local,
 497            full_path,
 498            is_dir,
 499        } => {
 500            let project = project_for_root_name(client, &project_root_name, cx)
 501                .ok_or(TestError::Inapplicable)?;
 502            let project_path = project_path_for_full_path(&project, &full_path, cx)
 503                .ok_or(TestError::Inapplicable)?;
 504
 505            log::info!(
 506                "{}: creating {} at path {:?} in {} project {}",
 507                client.username,
 508                if is_dir { "dir" } else { "file" },
 509                full_path,
 510                if is_local { "local" } else { "remote" },
 511                project_root_name,
 512            );
 513
 514            ensure_project_shared(&project, client, cx).await;
 515            project
 516                .update(cx, |p, cx| p.create_entry(project_path, is_dir, cx))
 517                .unwrap()
 518                .await?;
 519        }
 520
 521        ClientOperation::OpenBuffer {
 522            project_root_name,
 523            is_local,
 524            full_path,
 525        } => {
 526            let project = project_for_root_name(client, &project_root_name, cx)
 527                .ok_or(TestError::Inapplicable)?;
 528            let project_path = project_path_for_full_path(&project, &full_path, cx)
 529                .ok_or(TestError::Inapplicable)?;
 530
 531            log::info!(
 532                "{}: opening buffer {:?} in {} project {}",
 533                client.username,
 534                full_path,
 535                if is_local { "local" } else { "remote" },
 536                project_root_name,
 537            );
 538
 539            ensure_project_shared(&project, client, cx).await;
 540            let buffer = project
 541                .update(cx, |project, cx| project.open_buffer(project_path, cx))
 542                .await?;
 543            client.buffers_for_project(&project).insert(buffer);
 544        }
 545
 546        ClientOperation::EditBuffer {
 547            project_root_name,
 548            is_local,
 549            full_path,
 550            edits,
 551        } => {
 552            let project = project_for_root_name(client, &project_root_name, cx)
 553                .ok_or(TestError::Inapplicable)?;
 554            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 555                .ok_or(TestError::Inapplicable)?;
 556
 557            log::info!(
 558                "{}: editing buffer {:?} in {} project {} with {:?}",
 559                client.username,
 560                full_path,
 561                if is_local { "local" } else { "remote" },
 562                project_root_name,
 563                edits
 564            );
 565
 566            ensure_project_shared(&project, client, cx).await;
 567            buffer.update(cx, |buffer, cx| {
 568                let snapshot = buffer.snapshot();
 569                buffer.edit(
 570                    edits.into_iter().map(|(range, text)| {
 571                        let start = snapshot.clip_offset(range.start, Bias::Left);
 572                        let end = snapshot.clip_offset(range.end, Bias::Right);
 573                        (start..end, text)
 574                    }),
 575                    None,
 576                    cx,
 577                );
 578            });
 579        }
 580
 581        ClientOperation::CloseBuffer {
 582            project_root_name,
 583            is_local,
 584            full_path,
 585        } => {
 586            let project = project_for_root_name(client, &project_root_name, cx)
 587                .ok_or(TestError::Inapplicable)?;
 588            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 589                .ok_or(TestError::Inapplicable)?;
 590
 591            log::info!(
 592                "{}: closing buffer {:?} in {} project {}",
 593                client.username,
 594                full_path,
 595                if is_local { "local" } else { "remote" },
 596                project_root_name
 597            );
 598
 599            ensure_project_shared(&project, client, cx).await;
 600            cx.update(|_| {
 601                client.buffers_for_project(&project).remove(&buffer);
 602                drop(buffer);
 603            });
 604        }
 605
 606        ClientOperation::SaveBuffer {
 607            project_root_name,
 608            is_local,
 609            full_path,
 610            detach,
 611        } => {
 612            let project = project_for_root_name(client, &project_root_name, cx)
 613                .ok_or(TestError::Inapplicable)?;
 614            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 615                .ok_or(TestError::Inapplicable)?;
 616
 617            log::info!(
 618                "{}: saving buffer {:?} in {} project {}, {}",
 619                client.username,
 620                full_path,
 621                if is_local { "local" } else { "remote" },
 622                project_root_name,
 623                if detach { "detaching" } else { "awaiting" }
 624            );
 625
 626            ensure_project_shared(&project, client, cx).await;
 627            let requested_version = buffer.read_with(cx, |buffer, _| buffer.version());
 628            let save = project.update(cx, |project, cx| project.save_buffer(buffer, cx));
 629            let save = cx.background().spawn(async move {
 630                let (saved_version, _, _) = save
 631                    .await
 632                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 633                assert!(saved_version.observed_all(&requested_version));
 634                anyhow::Ok(())
 635            });
 636            if detach {
 637                cx.update(|cx| save.detach_and_log_err(cx));
 638            } else {
 639                save.await?;
 640            }
 641        }
 642
 643        ClientOperation::RequestLspDataInBuffer {
 644            project_root_name,
 645            is_local,
 646            full_path,
 647            offset,
 648            kind,
 649            detach,
 650        } => {
 651            let project = project_for_root_name(client, &project_root_name, cx)
 652                .ok_or(TestError::Inapplicable)?;
 653            let buffer = buffer_for_full_path(client, &project, &full_path, cx)
 654                .ok_or(TestError::Inapplicable)?;
 655
 656            log::info!(
 657                "{}: request LSP {:?} for buffer {:?} in {} project {}, {}",
 658                client.username,
 659                kind,
 660                full_path,
 661                if is_local { "local" } else { "remote" },
 662                project_root_name,
 663                if detach { "detaching" } else { "awaiting" }
 664            );
 665
 666            use futures::{FutureExt as _, TryFutureExt as _};
 667            let offset = buffer.read_with(cx, |b, _| b.clip_offset(offset, Bias::Left));
 668            let request = cx.foreground().spawn(project.update(cx, |project, cx| {
 669                match kind {
 670                    LspRequestKind::Rename => project
 671                        .prepare_rename(buffer, offset, cx)
 672                        .map_ok(|_| ())
 673                        .boxed(),
 674                    LspRequestKind::Completion => project
 675                        .completions(&buffer, offset, cx)
 676                        .map_ok(|_| ())
 677                        .boxed(),
 678                    LspRequestKind::CodeAction => project
 679                        .code_actions(&buffer, offset..offset, cx)
 680                        .map_ok(|_| ())
 681                        .boxed(),
 682                    LspRequestKind::Definition => project
 683                        .definition(&buffer, offset, cx)
 684                        .map_ok(|_| ())
 685                        .boxed(),
 686                    LspRequestKind::Highlights => project
 687                        .document_highlights(&buffer, offset, cx)
 688                        .map_ok(|_| ())
 689                        .boxed(),
 690                }
 691            }));
 692            if detach {
 693                request.detach();
 694            } else {
 695                request.await?;
 696            }
 697        }
 698
 699        ClientOperation::SearchProject {
 700            project_root_name,
 701            is_local,
 702            query,
 703            detach,
 704        } => {
 705            let project = project_for_root_name(client, &project_root_name, cx)
 706                .ok_or(TestError::Inapplicable)?;
 707
 708            log::info!(
 709                "{}: search {} project {} for {:?}, {}",
 710                client.username,
 711                if is_local { "local" } else { "remote" },
 712                project_root_name,
 713                query,
 714                if detach { "detaching" } else { "awaiting" }
 715            );
 716
 717            let search = project.update(cx, |project, cx| {
 718                project.search(SearchQuery::text(query, false, false), cx)
 719            });
 720            drop(project);
 721            let search = cx.background().spawn(async move {
 722                search
 723                    .await
 724                    .map_err(|err| anyhow!("search request failed: {:?}", err))
 725            });
 726            if detach {
 727                cx.update(|cx| search.detach_and_log_err(cx));
 728            } else {
 729                search.await?;
 730            }
 731        }
 732
 733        ClientOperation::WriteFsEntry {
 734            path,
 735            is_dir,
 736            content,
 737        } => {
 738            if !client
 739                .fs
 740                .directories()
 741                .contains(&path.parent().unwrap().to_owned())
 742            {
 743                return Err(TestError::Inapplicable);
 744            }
 745
 746            if is_dir {
 747                log::info!("{}: creating dir at {:?}", client.username, path);
 748                client.fs.create_dir(&path).await.unwrap();
 749            } else {
 750                let exists = client.fs.metadata(&path).await?.is_some();
 751                let verb = if exists { "updating" } else { "creating" };
 752                log::info!("{}: {} file at {:?}", verb, client.username, path);
 753
 754                client
 755                    .fs
 756                    .save(&path, &content.as_str().into(), fs::LineEnding::Unix)
 757                    .await
 758                    .unwrap();
 759            }
 760        }
 761
 762        ClientOperation::WriteGitIndex {
 763            repo_path,
 764            contents,
 765        } => {
 766            if !client.fs.directories().contains(&repo_path) {
 767                return Err(TestError::Inapplicable);
 768            }
 769
 770            log::info!(
 771                "{}: writing git index for repo {:?}: {:?}",
 772                client.username,
 773                repo_path,
 774                contents
 775            );
 776
 777            let dot_git_dir = repo_path.join(".git");
 778            let contents = contents
 779                .iter()
 780                .map(|(path, contents)| (path.as_path(), contents.clone()))
 781                .collect::<Vec<_>>();
 782            if client.fs.metadata(&dot_git_dir).await?.is_none() {
 783                client.fs.create_dir(&dot_git_dir).await?;
 784            }
 785            client.fs.set_index_for_repo(&dot_git_dir, &contents).await;
 786        }
 787    }
 788    Ok(())
 789}
 790
 791fn check_consistency_between_clients(clients: &[(Rc<TestClient>, TestAppContext)]) {
 792    for (client, client_cx) in clients {
 793        for guest_project in client.remote_projects().iter() {
 794            guest_project.read_with(client_cx, |guest_project, cx| {
 795                let host_project = clients.iter().find_map(|(client, cx)| {
 796                    let project = client
 797                        .local_projects()
 798                        .iter()
 799                        .find(|host_project| {
 800                            host_project.read_with(cx, |host_project, _| {
 801                                host_project.remote_id() == guest_project.remote_id()
 802                            })
 803                        })?
 804                        .clone();
 805                    Some((project, cx))
 806                });
 807
 808                if !guest_project.is_read_only() {
 809                    if let Some((host_project, host_cx)) = host_project {
 810                        let host_worktree_snapshots =
 811                            host_project.read_with(host_cx, |host_project, cx| {
 812                                host_project
 813                                    .worktrees(cx)
 814                                    .map(|worktree| {
 815                                        let worktree = worktree.read(cx);
 816                                        (worktree.id(), worktree.snapshot())
 817                                    })
 818                                    .collect::<BTreeMap<_, _>>()
 819                            });
 820                        let guest_worktree_snapshots = guest_project
 821                            .worktrees(cx)
 822                            .map(|worktree| {
 823                                let worktree = worktree.read(cx);
 824                                (worktree.id(), worktree.snapshot())
 825                            })
 826                            .collect::<BTreeMap<_, _>>();
 827
 828                        assert_eq!(
 829                            guest_worktree_snapshots.values().map(|w| w.abs_path()).collect::<Vec<_>>(),
 830                            host_worktree_snapshots.values().map(|w| w.abs_path()).collect::<Vec<_>>(),
 831                            "{} has different worktrees than the host for project {:?}",
 832                            client.username, guest_project.remote_id(),
 833                        );
 834
 835                        for (id, host_snapshot) in &host_worktree_snapshots {
 836                            let guest_snapshot = &guest_worktree_snapshots[id];
 837                            assert_eq!(
 838                                guest_snapshot.root_name(),
 839                                host_snapshot.root_name(),
 840                                "{} has different root name than the host for worktree {}, project {:?}",
 841                                client.username,
 842                                id,
 843                                guest_project.remote_id(),
 844                            );
 845                            assert_eq!(
 846                                guest_snapshot.abs_path(),
 847                                host_snapshot.abs_path(),
 848                                "{} has different abs path than the host for worktree {}, project: {:?}",
 849                                client.username,
 850                                id,
 851                                guest_project.remote_id(),
 852                            );
 853                            assert_eq!(
 854                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 855                                host_snapshot.entries(false).collect::<Vec<_>>(),
 856                                "{} has different snapshot than the host for worktree {:?} and project {:?}",
 857                                client.username,
 858                                host_snapshot.abs_path(),
 859                                guest_project.remote_id(),
 860                            );
 861                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id(),
 862                                "{} has different scan id than the host for worktree {:?} and project {:?}",
 863                                client.username,
 864                                host_snapshot.abs_path(),
 865                                guest_project.remote_id(),
 866                            );
 867                        }
 868                    }
 869                }
 870
 871                guest_project.check_invariants(cx);
 872            });
 873        }
 874
 875        let buffers = client.buffers().clone();
 876        for (guest_project, guest_buffers) in &buffers {
 877            let project_id = if guest_project.read_with(client_cx, |project, _| {
 878                project.is_local() || project.is_read_only()
 879            }) {
 880                continue;
 881            } else {
 882                guest_project
 883                    .read_with(client_cx, |project, _| project.remote_id())
 884                    .unwrap()
 885            };
 886            let guest_user_id = client.user_id().unwrap();
 887
 888            let host_project = clients.iter().find_map(|(client, cx)| {
 889                let project = client
 890                    .local_projects()
 891                    .iter()
 892                    .find(|host_project| {
 893                        host_project.read_with(cx, |host_project, _| {
 894                            host_project.remote_id() == Some(project_id)
 895                        })
 896                    })?
 897                    .clone();
 898                Some((client.user_id().unwrap(), project, cx))
 899            });
 900
 901            let (host_user_id, host_project, host_cx) =
 902                if let Some((host_user_id, host_project, host_cx)) = host_project {
 903                    (host_user_id, host_project, host_cx)
 904                } else {
 905                    continue;
 906                };
 907
 908            for guest_buffer in guest_buffers {
 909                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 910                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 911                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 912                        panic!(
 913                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 914                            client.username,
 915                            client.peer_id(),
 916                            buffer_id
 917                        )
 918                    })
 919                });
 920                let path = host_buffer
 921                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 922
 923                assert_eq!(
 924                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 925                    0,
 926                    "{}, buffer {}, path {:?} has deferred operations",
 927                    client.username,
 928                    buffer_id,
 929                    path,
 930                );
 931                assert_eq!(
 932                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 933                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 934                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 935                    client.username,
 936                    buffer_id,
 937                    path
 938                );
 939
 940                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 941                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 942                match (host_file, guest_file) {
 943                    (Some(host_file), Some(guest_file)) => {
 944                        assert_eq!(guest_file.path(), host_file.path());
 945                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 946                        assert_eq!(
 947                            guest_file.mtime(),
 948                            host_file.mtime(),
 949                            "guest {} mtime does not match host {} for path {:?} in project {}",
 950                            guest_user_id,
 951                            host_user_id,
 952                            guest_file.path(),
 953                            project_id,
 954                        );
 955                    }
 956                    (None, None) => {}
 957                    (None, _) => panic!("host's file is None, guest's isn't"),
 958                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 959                }
 960
 961                let host_diff_base =
 962                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 963                let guest_diff_base = guest_buffer
 964                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 965                assert_eq!(guest_diff_base, host_diff_base);
 966
 967                let host_saved_version =
 968                    host_buffer.read_with(host_cx, |b, _| b.saved_version().clone());
 969                let guest_saved_version =
 970                    guest_buffer.read_with(client_cx, |b, _| b.saved_version().clone());
 971                assert_eq!(
 972                    guest_saved_version, host_saved_version,
 973                    "guest saved version does not match host's for path {path:?} in project {project_id}",
 974                );
 975
 976                let host_saved_version_fingerprint =
 977                    host_buffer.read_with(host_cx, |b, _| b.saved_version_fingerprint());
 978                let guest_saved_version_fingerprint =
 979                    guest_buffer.read_with(client_cx, |b, _| b.saved_version_fingerprint());
 980                assert_eq!(
 981                    guest_saved_version_fingerprint, host_saved_version_fingerprint,
 982                    "guest's saved fingerprint does not match host's for path {path:?} in project {project_id}",
 983                );
 984
 985                let host_saved_mtime = host_buffer.read_with(host_cx, |b, _| b.saved_mtime());
 986                let guest_saved_mtime = guest_buffer.read_with(client_cx, |b, _| b.saved_mtime());
 987                assert_eq!(
 988                    guest_saved_mtime, host_saved_mtime,
 989                    "guest's saved mtime does not match host's for path {path:?} in project {project_id}",
 990                );
 991
 992                let host_is_dirty = host_buffer.read_with(host_cx, |b, _| b.is_dirty());
 993                let guest_is_dirty = guest_buffer.read_with(client_cx, |b, _| b.is_dirty());
 994                assert_eq!(guest_is_dirty, host_is_dirty,
 995                    "guest's dirty status does not match host's for path {path:?} in project {project_id}",
 996                );
 997
 998                let host_has_conflict = host_buffer.read_with(host_cx, |b, _| b.has_conflict());
 999                let guest_has_conflict = guest_buffer.read_with(client_cx, |b, _| b.has_conflict());
1000                assert_eq!(guest_has_conflict, host_has_conflict,
1001                    "guest's conflict status does not match host's for path {path:?} in project {project_id}",
1002                );
1003            }
1004        }
1005    }
1006}
1007
1008struct TestPlan {
1009    rng: StdRng,
1010    replay: bool,
1011    stored_operations: Vec<(StoredOperation, Arc<AtomicBool>)>,
1012    max_operations: usize,
1013    operation_ix: usize,
1014    users: Vec<UserTestPlan>,
1015    next_batch_id: usize,
1016    allow_server_restarts: bool,
1017    allow_client_reconnection: bool,
1018    allow_client_disconnection: bool,
1019}
1020
1021struct UserTestPlan {
1022    user_id: UserId,
1023    username: String,
1024    next_root_id: usize,
1025    operation_ix: usize,
1026    online: bool,
1027}
1028
1029#[derive(Clone, Debug, Serialize, Deserialize)]
1030#[serde(untagged)]
1031enum StoredOperation {
1032    Server(Operation),
1033    Client {
1034        user_id: UserId,
1035        batch_id: usize,
1036        operation: ClientOperation,
1037    },
1038}
1039
1040#[derive(Clone, Debug, Serialize, Deserialize)]
1041enum Operation {
1042    AddConnection {
1043        user_id: UserId,
1044    },
1045    RemoveConnection {
1046        user_id: UserId,
1047    },
1048    BounceConnection {
1049        user_id: UserId,
1050    },
1051    RestartServer,
1052    MutateClients {
1053        batch_id: usize,
1054        #[serde(skip_serializing)]
1055        #[serde(skip_deserializing)]
1056        user_ids: Vec<UserId>,
1057        quiesce: bool,
1058    },
1059}
1060
1061#[derive(Clone, Debug, Serialize, Deserialize)]
1062enum ClientOperation {
1063    AcceptIncomingCall,
1064    RejectIncomingCall,
1065    LeaveCall,
1066    InviteContactToCall {
1067        user_id: UserId,
1068    },
1069    OpenLocalProject {
1070        first_root_name: String,
1071    },
1072    OpenRemoteProject {
1073        host_id: UserId,
1074        first_root_name: String,
1075    },
1076    AddWorktreeToProject {
1077        project_root_name: String,
1078        new_root_path: PathBuf,
1079    },
1080    CloseRemoteProject {
1081        project_root_name: String,
1082    },
1083    OpenBuffer {
1084        project_root_name: String,
1085        is_local: bool,
1086        full_path: PathBuf,
1087    },
1088    SearchProject {
1089        project_root_name: String,
1090        is_local: bool,
1091        query: String,
1092        detach: bool,
1093    },
1094    EditBuffer {
1095        project_root_name: String,
1096        is_local: bool,
1097        full_path: PathBuf,
1098        edits: Vec<(Range<usize>, Arc<str>)>,
1099    },
1100    CloseBuffer {
1101        project_root_name: String,
1102        is_local: bool,
1103        full_path: PathBuf,
1104    },
1105    SaveBuffer {
1106        project_root_name: String,
1107        is_local: bool,
1108        full_path: PathBuf,
1109        detach: bool,
1110    },
1111    RequestLspDataInBuffer {
1112        project_root_name: String,
1113        is_local: bool,
1114        full_path: PathBuf,
1115        offset: usize,
1116        kind: LspRequestKind,
1117        detach: bool,
1118    },
1119    CreateWorktreeEntry {
1120        project_root_name: String,
1121        is_local: bool,
1122        full_path: PathBuf,
1123        is_dir: bool,
1124    },
1125    WriteFsEntry {
1126        path: PathBuf,
1127        is_dir: bool,
1128        content: String,
1129    },
1130    WriteGitIndex {
1131        repo_path: PathBuf,
1132        contents: Vec<(PathBuf, String)>,
1133    },
1134}
1135
1136#[derive(Clone, Debug, Serialize, Deserialize)]
1137enum LspRequestKind {
1138    Rename,
1139    Completion,
1140    CodeAction,
1141    Definition,
1142    Highlights,
1143}
1144
1145enum TestError {
1146    Inapplicable,
1147    Other(anyhow::Error),
1148}
1149
1150impl From<anyhow::Error> for TestError {
1151    fn from(value: anyhow::Error) -> Self {
1152        Self::Other(value)
1153    }
1154}
1155
1156impl TestPlan {
1157    fn new(mut rng: StdRng, users: Vec<UserTestPlan>, max_operations: usize) -> Self {
1158        Self {
1159            replay: false,
1160            allow_server_restarts: rng.gen_bool(0.7),
1161            allow_client_reconnection: rng.gen_bool(0.7),
1162            allow_client_disconnection: rng.gen_bool(0.1),
1163            stored_operations: Vec::new(),
1164            operation_ix: 0,
1165            next_batch_id: 0,
1166            max_operations,
1167            users,
1168            rng,
1169        }
1170    }
1171
1172    fn deserialize(&mut self, json: Vec<u8>) {
1173        let stored_operations: Vec<StoredOperation> = serde_json::from_slice(&json).unwrap();
1174        self.replay = true;
1175        self.stored_operations = stored_operations
1176            .iter()
1177            .cloned()
1178            .enumerate()
1179            .map(|(i, mut operation)| {
1180                if let StoredOperation::Server(Operation::MutateClients {
1181                    batch_id: current_batch_id,
1182                    user_ids,
1183                    ..
1184                }) = &mut operation
1185                {
1186                    assert!(user_ids.is_empty());
1187                    user_ids.extend(stored_operations[i + 1..].iter().filter_map(|operation| {
1188                        if let StoredOperation::Client {
1189                            user_id, batch_id, ..
1190                        } = operation
1191                        {
1192                            if batch_id == current_batch_id {
1193                                return Some(user_id);
1194                            }
1195                        }
1196                        None
1197                    }));
1198                    user_ids.sort_unstable();
1199                }
1200                (operation, Arc::new(AtomicBool::new(false)))
1201            })
1202            .collect()
1203    }
1204
1205    fn serialize(&mut self) -> Vec<u8> {
1206        // Format each operation as one line
1207        let mut json = Vec::new();
1208        json.push(b'[');
1209        for (operation, applied) in &self.stored_operations {
1210            if !applied.load(SeqCst) {
1211                continue;
1212            }
1213            if json.len() > 1 {
1214                json.push(b',');
1215            }
1216            json.extend_from_slice(b"\n  ");
1217            serde_json::to_writer(&mut json, operation).unwrap();
1218        }
1219        json.extend_from_slice(b"\n]\n");
1220        json
1221    }
1222
1223    fn next_server_operation(
1224        &mut self,
1225        clients: &[(Rc<TestClient>, TestAppContext)],
1226    ) -> Option<(Operation, Arc<AtomicBool>)> {
1227        if self.replay {
1228            while let Some(stored_operation) = self.stored_operations.get(self.operation_ix) {
1229                self.operation_ix += 1;
1230                if let (StoredOperation::Server(operation), applied) = stored_operation {
1231                    return Some((operation.clone(), applied.clone()));
1232                }
1233            }
1234            None
1235        } else {
1236            let operation = self.generate_server_operation(clients)?;
1237            let applied = Arc::new(AtomicBool::new(false));
1238            self.stored_operations
1239                .push((StoredOperation::Server(operation.clone()), applied.clone()));
1240            Some((operation, applied))
1241        }
1242    }
1243
1244    fn next_client_operation(
1245        &mut self,
1246        client: &TestClient,
1247        current_batch_id: usize,
1248        cx: &TestAppContext,
1249    ) -> Option<(ClientOperation, Arc<AtomicBool>)> {
1250        let current_user_id = client.current_user_id(cx);
1251        let user_ix = self
1252            .users
1253            .iter()
1254            .position(|user| user.user_id == current_user_id)
1255            .unwrap();
1256        let user_plan = &mut self.users[user_ix];
1257
1258        if self.replay {
1259            while let Some(stored_operation) = self.stored_operations.get(user_plan.operation_ix) {
1260                user_plan.operation_ix += 1;
1261                if let (
1262                    StoredOperation::Client {
1263                        user_id, operation, ..
1264                    },
1265                    applied,
1266                ) = stored_operation
1267                {
1268                    if user_id == &current_user_id {
1269                        return Some((operation.clone(), applied.clone()));
1270                    }
1271                }
1272            }
1273            None
1274        } else {
1275            let operation = self.generate_client_operation(current_user_id, client, cx)?;
1276            let applied = Arc::new(AtomicBool::new(false));
1277            self.stored_operations.push((
1278                StoredOperation::Client {
1279                    user_id: current_user_id,
1280                    batch_id: current_batch_id,
1281                    operation: operation.clone(),
1282                },
1283                applied.clone(),
1284            ));
1285            Some((operation, applied))
1286        }
1287    }
1288
1289    fn generate_server_operation(
1290        &mut self,
1291        clients: &[(Rc<TestClient>, TestAppContext)],
1292    ) -> Option<Operation> {
1293        if self.operation_ix == self.max_operations {
1294            return None;
1295        }
1296
1297        Some(loop {
1298            break match self.rng.gen_range(0..100) {
1299                0..=29 if clients.len() < self.users.len() => {
1300                    let user = self
1301                        .users
1302                        .iter()
1303                        .filter(|u| !u.online)
1304                        .choose(&mut self.rng)
1305                        .unwrap();
1306                    self.operation_ix += 1;
1307                    Operation::AddConnection {
1308                        user_id: user.user_id,
1309                    }
1310                }
1311                30..=34 if clients.len() > 1 && self.allow_client_disconnection => {
1312                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1313                    let user_id = client.current_user_id(cx);
1314                    self.operation_ix += 1;
1315                    Operation::RemoveConnection { user_id }
1316                }
1317                35..=39 if clients.len() > 1 && self.allow_client_reconnection => {
1318                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
1319                    let user_id = client.current_user_id(cx);
1320                    self.operation_ix += 1;
1321                    Operation::BounceConnection { user_id }
1322                }
1323                40..=44 if self.allow_server_restarts && clients.len() > 1 => {
1324                    self.operation_ix += 1;
1325                    Operation::RestartServer
1326                }
1327                _ if !clients.is_empty() => {
1328                    let count = self
1329                        .rng
1330                        .gen_range(1..10)
1331                        .min(self.max_operations - self.operation_ix);
1332                    let batch_id = util::post_inc(&mut self.next_batch_id);
1333                    let mut user_ids = (0..count)
1334                        .map(|_| {
1335                            let ix = self.rng.gen_range(0..clients.len());
1336                            let (client, cx) = &clients[ix];
1337                            client.current_user_id(cx)
1338                        })
1339                        .collect::<Vec<_>>();
1340                    user_ids.sort_unstable();
1341                    Operation::MutateClients {
1342                        user_ids,
1343                        batch_id,
1344                        quiesce: self.rng.gen_bool(0.7),
1345                    }
1346                }
1347                _ => continue,
1348            };
1349        })
1350    }
1351
1352    fn generate_client_operation(
1353        &mut self,
1354        user_id: UserId,
1355        client: &TestClient,
1356        cx: &TestAppContext,
1357    ) -> Option<ClientOperation> {
1358        if self.operation_ix == self.max_operations {
1359            return None;
1360        }
1361
1362        self.operation_ix += 1;
1363        let call = cx.read(ActiveCall::global);
1364        Some(loop {
1365            match self.rng.gen_range(0..100_u32) {
1366                // Mutate the call
1367                0..=29 => {
1368                    // Respond to an incoming call
1369                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
1370                        break if self.rng.gen_bool(0.7) {
1371                            ClientOperation::AcceptIncomingCall
1372                        } else {
1373                            ClientOperation::RejectIncomingCall
1374                        };
1375                    }
1376
1377                    match self.rng.gen_range(0..100_u32) {
1378                        // Invite a contact to the current call
1379                        0..=70 => {
1380                            let available_contacts =
1381                                client.user_store.read_with(cx, |user_store, _| {
1382                                    user_store
1383                                        .contacts()
1384                                        .iter()
1385                                        .filter(|contact| contact.online && !contact.busy)
1386                                        .cloned()
1387                                        .collect::<Vec<_>>()
1388                                });
1389                            if !available_contacts.is_empty() {
1390                                let contact = available_contacts.choose(&mut self.rng).unwrap();
1391                                break ClientOperation::InviteContactToCall {
1392                                    user_id: UserId(contact.user.id as i32),
1393                                };
1394                            }
1395                        }
1396
1397                        // Leave the current call
1398                        71.. => {
1399                            if self.allow_client_disconnection
1400                                && call.read_with(cx, |call, _| call.room().is_some())
1401                            {
1402                                break ClientOperation::LeaveCall;
1403                            }
1404                        }
1405                    }
1406                }
1407
1408                // Mutate projects
1409                30..=59 => match self.rng.gen_range(0..100_u32) {
1410                    // Open a new project
1411                    0..=70 => {
1412                        // Open a remote project
1413                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
1414                            let existing_remote_project_ids = cx.read(|cx| {
1415                                client
1416                                    .remote_projects()
1417                                    .iter()
1418                                    .map(|p| p.read(cx).remote_id().unwrap())
1419                                    .collect::<Vec<_>>()
1420                            });
1421                            let new_remote_projects = room.read_with(cx, |room, _| {
1422                                room.remote_participants()
1423                                    .values()
1424                                    .flat_map(|participant| {
1425                                        participant.projects.iter().filter_map(|project| {
1426                                            if existing_remote_project_ids.contains(&project.id) {
1427                                                None
1428                                            } else {
1429                                                Some((
1430                                                    UserId::from_proto(participant.user.id),
1431                                                    project.worktree_root_names[0].clone(),
1432                                                ))
1433                                            }
1434                                        })
1435                                    })
1436                                    .collect::<Vec<_>>()
1437                            });
1438                            if !new_remote_projects.is_empty() {
1439                                let (host_id, first_root_name) =
1440                                    new_remote_projects.choose(&mut self.rng).unwrap().clone();
1441                                break ClientOperation::OpenRemoteProject {
1442                                    host_id,
1443                                    first_root_name,
1444                                };
1445                            }
1446                        }
1447                        // Open a local project
1448                        else {
1449                            let first_root_name = self.next_root_dir_name(user_id);
1450                            break ClientOperation::OpenLocalProject { first_root_name };
1451                        }
1452                    }
1453
1454                    // Close a remote project
1455                    71..=80 => {
1456                        if !client.remote_projects().is_empty() {
1457                            let project = client
1458                                .remote_projects()
1459                                .choose(&mut self.rng)
1460                                .unwrap()
1461                                .clone();
1462                            let first_root_name = root_name_for_project(&project, cx);
1463                            break ClientOperation::CloseRemoteProject {
1464                                project_root_name: first_root_name,
1465                            };
1466                        }
1467                    }
1468
1469                    // Mutate project worktrees
1470                    81.. => match self.rng.gen_range(0..100_u32) {
1471                        // Add a worktree to a local project
1472                        0..=50 => {
1473                            let Some(project) = client
1474                                .local_projects()
1475                                .choose(&mut self.rng)
1476                                .cloned() else { continue };
1477                            let project_root_name = root_name_for_project(&project, cx);
1478                            let mut paths = client.fs.paths();
1479                            paths.remove(0);
1480                            let new_root_path = if paths.is_empty() || self.rng.gen() {
1481                                Path::new("/").join(&self.next_root_dir_name(user_id))
1482                            } else {
1483                                paths.choose(&mut self.rng).unwrap().clone()
1484                            };
1485                            break ClientOperation::AddWorktreeToProject {
1486                                project_root_name,
1487                                new_root_path,
1488                            };
1489                        }
1490
1491                        // Add an entry to a worktree
1492                        _ => {
1493                            let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1494                            let project_root_name = root_name_for_project(&project, cx);
1495                            let is_local = project.read_with(cx, |project, _| project.is_local());
1496                            let worktree = project.read_with(cx, |project, cx| {
1497                                project
1498                                    .worktrees(cx)
1499                                    .filter(|worktree| {
1500                                        let worktree = worktree.read(cx);
1501                                        worktree.is_visible()
1502                                            && worktree.entries(false).any(|e| e.is_file())
1503                                            && worktree.root_entry().map_or(false, |e| e.is_dir())
1504                                    })
1505                                    .choose(&mut self.rng)
1506                            });
1507                            let Some(worktree) = worktree else { continue };
1508                            let is_dir = self.rng.gen::<bool>();
1509                            let mut full_path =
1510                                worktree.read_with(cx, |w, _| PathBuf::from(w.root_name()));
1511                            full_path.push(gen_file_name(&mut self.rng));
1512                            if !is_dir {
1513                                full_path.set_extension("rs");
1514                            }
1515                            break ClientOperation::CreateWorktreeEntry {
1516                                project_root_name,
1517                                is_local,
1518                                full_path,
1519                                is_dir,
1520                            };
1521                        }
1522                    },
1523                },
1524
1525                // Query and mutate buffers
1526                60..=90 => {
1527                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
1528                    let project_root_name = root_name_for_project(&project, cx);
1529                    let is_local = project.read_with(cx, |project, _| project.is_local());
1530
1531                    match self.rng.gen_range(0..100_u32) {
1532                        // Manipulate an existing buffer
1533                        0..=70 => {
1534                            let Some(buffer) = client
1535                                .buffers_for_project(&project)
1536                                .iter()
1537                                .choose(&mut self.rng)
1538                                .cloned() else { continue };
1539
1540                            let full_path = buffer
1541                                .read_with(cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
1542
1543                            match self.rng.gen_range(0..100_u32) {
1544                                // Close the buffer
1545                                0..=15 => {
1546                                    break ClientOperation::CloseBuffer {
1547                                        project_root_name,
1548                                        is_local,
1549                                        full_path,
1550                                    };
1551                                }
1552                                // Save the buffer
1553                                16..=29 if buffer.read_with(cx, |b, _| b.is_dirty()) => {
1554                                    let detach = self.rng.gen_bool(0.3);
1555                                    break ClientOperation::SaveBuffer {
1556                                        project_root_name,
1557                                        is_local,
1558                                        full_path,
1559                                        detach,
1560                                    };
1561                                }
1562                                // Edit the buffer
1563                                30..=69 => {
1564                                    let edits = buffer.read_with(cx, |buffer, _| {
1565                                        buffer.get_random_edits(&mut self.rng, 3)
1566                                    });
1567                                    break ClientOperation::EditBuffer {
1568                                        project_root_name,
1569                                        is_local,
1570                                        full_path,
1571                                        edits,
1572                                    };
1573                                }
1574                                // Make an LSP request
1575                                _ => {
1576                                    let offset = buffer.read_with(cx, |buffer, _| {
1577                                        buffer.clip_offset(
1578                                            self.rng.gen_range(0..=buffer.len()),
1579                                            language::Bias::Left,
1580                                        )
1581                                    });
1582                                    let detach = self.rng.gen();
1583                                    break ClientOperation::RequestLspDataInBuffer {
1584                                        project_root_name,
1585                                        full_path,
1586                                        offset,
1587                                        is_local,
1588                                        kind: match self.rng.gen_range(0..5_u32) {
1589                                            0 => LspRequestKind::Rename,
1590                                            1 => LspRequestKind::Highlights,
1591                                            2 => LspRequestKind::Definition,
1592                                            3 => LspRequestKind::CodeAction,
1593                                            4.. => LspRequestKind::Completion,
1594                                        },
1595                                        detach,
1596                                    };
1597                                }
1598                            }
1599                        }
1600
1601                        71..=80 => {
1602                            let query = self.rng.gen_range('a'..='z').to_string();
1603                            let detach = self.rng.gen_bool(0.3);
1604                            break ClientOperation::SearchProject {
1605                                project_root_name,
1606                                is_local,
1607                                query,
1608                                detach,
1609                            };
1610                        }
1611
1612                        // Open a buffer
1613                        81.. => {
1614                            let worktree = project.read_with(cx, |project, cx| {
1615                                project
1616                                    .worktrees(cx)
1617                                    .filter(|worktree| {
1618                                        let worktree = worktree.read(cx);
1619                                        worktree.is_visible()
1620                                            && worktree.entries(false).any(|e| e.is_file())
1621                                    })
1622                                    .choose(&mut self.rng)
1623                            });
1624                            let Some(worktree) = worktree else { continue };
1625                            let full_path = worktree.read_with(cx, |worktree, _| {
1626                                let entry = worktree
1627                                    .entries(false)
1628                                    .filter(|e| e.is_file())
1629                                    .choose(&mut self.rng)
1630                                    .unwrap();
1631                                if entry.path.as_ref() == Path::new("") {
1632                                    Path::new(worktree.root_name()).into()
1633                                } else {
1634                                    Path::new(worktree.root_name()).join(&entry.path)
1635                                }
1636                            });
1637                            break ClientOperation::OpenBuffer {
1638                                project_root_name,
1639                                is_local,
1640                                full_path,
1641                            };
1642                        }
1643                    }
1644                }
1645
1646                // Update a git index
1647                91..=95 => {
1648                    let repo_path = client
1649                        .fs
1650                        .directories()
1651                        .choose(&mut self.rng)
1652                        .unwrap()
1653                        .clone();
1654
1655                    let mut file_paths = client
1656                        .fs
1657                        .files()
1658                        .into_iter()
1659                        .filter(|path| path.starts_with(&repo_path))
1660                        .collect::<Vec<_>>();
1661                    let count = self.rng.gen_range(0..=file_paths.len());
1662                    file_paths.shuffle(&mut self.rng);
1663                    file_paths.truncate(count);
1664
1665                    let mut contents = Vec::new();
1666                    for abs_child_file_path in &file_paths {
1667                        let child_file_path = abs_child_file_path
1668                            .strip_prefix(&repo_path)
1669                            .unwrap()
1670                            .to_path_buf();
1671                        let new_base = Alphanumeric.sample_string(&mut self.rng, 16);
1672                        contents.push((child_file_path, new_base));
1673                    }
1674
1675                    break ClientOperation::WriteGitIndex {
1676                        repo_path,
1677                        contents,
1678                    };
1679                }
1680
1681                // Create or update a file or directory
1682                96.. => {
1683                    let is_dir = self.rng.gen::<bool>();
1684                    let content;
1685                    let mut path;
1686                    let dir_paths = client.fs.directories();
1687
1688                    if is_dir {
1689                        content = String::new();
1690                        path = dir_paths.choose(&mut self.rng).unwrap().clone();
1691                        path.push(gen_file_name(&mut self.rng));
1692                    } else {
1693                        content = Alphanumeric.sample_string(&mut self.rng, 16);
1694
1695                        // Create a new file or overwrite an existing file
1696                        let file_paths = client.fs.files();
1697                        if file_paths.is_empty() || self.rng.gen_bool(0.5) {
1698                            path = dir_paths.choose(&mut self.rng).unwrap().clone();
1699                            path.push(gen_file_name(&mut self.rng));
1700                            path.set_extension("rs");
1701                        } else {
1702                            path = file_paths.choose(&mut self.rng).unwrap().clone()
1703                        };
1704                    }
1705                    break ClientOperation::WriteFsEntry {
1706                        path,
1707                        is_dir,
1708                        content,
1709                    };
1710                }
1711            }
1712        })
1713    }
1714
1715    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
1716        let user_ix = self
1717            .users
1718            .iter()
1719            .position(|user| user.user_id == user_id)
1720            .unwrap();
1721        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
1722        format!("dir-{user_id}-{root_id}")
1723    }
1724
1725    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
1726        let ix = self
1727            .users
1728            .iter()
1729            .position(|user| user.user_id == user_id)
1730            .unwrap();
1731        &mut self.users[ix]
1732    }
1733}
1734
1735async fn simulate_client(
1736    client: Rc<TestClient>,
1737    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<usize>,
1738    plan: Arc<Mutex<TestPlan>>,
1739    mut cx: TestAppContext,
1740) {
1741    // Setup language server
1742    let mut language = Language::new(
1743        LanguageConfig {
1744            name: "Rust".into(),
1745            path_suffixes: vec!["rs".to_string()],
1746            ..Default::default()
1747        },
1748        None,
1749    );
1750    let _fake_language_servers = language
1751        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
1752            name: "the-fake-language-server",
1753            capabilities: lsp::LanguageServer::full_capabilities(),
1754            initializer: Some(Box::new({
1755                let fs = client.fs.clone();
1756                move |fake_server: &mut FakeLanguageServer| {
1757                    fake_server.handle_request::<lsp::request::Completion, _, _>(
1758                        |_, _| async move {
1759                            Ok(Some(lsp::CompletionResponse::Array(vec![
1760                                lsp::CompletionItem {
1761                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
1762                                        range: lsp::Range::new(
1763                                            lsp::Position::new(0, 0),
1764                                            lsp::Position::new(0, 0),
1765                                        ),
1766                                        new_text: "the-new-text".to_string(),
1767                                    })),
1768                                    ..Default::default()
1769                                },
1770                            ])))
1771                        },
1772                    );
1773
1774                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
1775                        |_, _| async move {
1776                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
1777                                lsp::CodeAction {
1778                                    title: "the-code-action".to_string(),
1779                                    ..Default::default()
1780                                },
1781                            )]))
1782                        },
1783                    );
1784
1785                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
1786                        |params, _| async move {
1787                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
1788                                params.position,
1789                                params.position,
1790                            ))))
1791                        },
1792                    );
1793
1794                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
1795                        let fs = fs.clone();
1796                        move |_, cx| {
1797                            let background = cx.background();
1798                            let mut rng = background.rng();
1799                            let count = rng.gen_range::<usize, _>(1..3);
1800                            let files = fs.files();
1801                            let files = (0..count)
1802                                .map(|_| files.choose(&mut *rng).unwrap().clone())
1803                                .collect::<Vec<_>>();
1804                            async move {
1805                                log::info!("LSP: Returning definitions in files {:?}", &files);
1806                                Ok(Some(lsp::GotoDefinitionResponse::Array(
1807                                    files
1808                                        .into_iter()
1809                                        .map(|file| lsp::Location {
1810                                            uri: lsp::Url::from_file_path(file).unwrap(),
1811                                            range: Default::default(),
1812                                        })
1813                                        .collect(),
1814                                )))
1815                            }
1816                        }
1817                    });
1818
1819                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>(
1820                        move |_, cx| {
1821                            let mut highlights = Vec::new();
1822                            let background = cx.background();
1823                            let mut rng = background.rng();
1824
1825                            let highlight_count = rng.gen_range(1..=5);
1826                            for _ in 0..highlight_count {
1827                                let start_row = rng.gen_range(0..100);
1828                                let start_column = rng.gen_range(0..100);
1829                                let end_row = rng.gen_range(0..100);
1830                                let end_column = rng.gen_range(0..100);
1831                                let start = PointUtf16::new(start_row, start_column);
1832                                let end = PointUtf16::new(end_row, end_column);
1833                                let range = if start > end { end..start } else { start..end };
1834                                highlights.push(lsp::DocumentHighlight {
1835                                    range: range_to_lsp(range.clone()),
1836                                    kind: Some(lsp::DocumentHighlightKind::READ),
1837                                });
1838                            }
1839                            highlights.sort_unstable_by_key(|highlight| {
1840                                (highlight.range.start, highlight.range.end)
1841                            });
1842                            async move { Ok(Some(highlights)) }
1843                        },
1844                    );
1845                }
1846            })),
1847            ..Default::default()
1848        }))
1849        .await;
1850    client.language_registry.add(Arc::new(language));
1851
1852    while let Some(batch_id) = operation_rx.next().await {
1853        let Some((operation, applied)) = plan.lock().next_client_operation(&client, batch_id, &cx) else { break };
1854        applied.store(true, SeqCst);
1855        match apply_client_operation(&client, operation, &mut cx).await {
1856            Ok(()) => {}
1857            Err(TestError::Inapplicable) => {
1858                applied.store(false, SeqCst);
1859                log::info!("skipped operation");
1860            }
1861            Err(TestError::Other(error)) => {
1862                log::error!("{} error: {}", client.username, error);
1863            }
1864        }
1865        cx.background().simulate_random_delay().await;
1866    }
1867    log::info!("{}: done", client.username);
1868}
1869
1870fn buffer_for_full_path(
1871    client: &TestClient,
1872    project: &ModelHandle<Project>,
1873    full_path: &PathBuf,
1874    cx: &TestAppContext,
1875) -> Option<ModelHandle<language::Buffer>> {
1876    client
1877        .buffers_for_project(project)
1878        .iter()
1879        .find(|buffer| {
1880            buffer.read_with(cx, |buffer, cx| {
1881                buffer.file().unwrap().full_path(cx) == *full_path
1882            })
1883        })
1884        .cloned()
1885}
1886
1887fn project_for_root_name(
1888    client: &TestClient,
1889    root_name: &str,
1890    cx: &TestAppContext,
1891) -> Option<ModelHandle<Project>> {
1892    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1893        return Some(client.local_projects()[ix].clone());
1894    }
1895    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1896        return Some(client.remote_projects()[ix].clone());
1897    }
1898    None
1899}
1900
1901fn project_ix_for_root_name(
1902    projects: &[ModelHandle<Project>],
1903    root_name: &str,
1904    cx: &TestAppContext,
1905) -> Option<usize> {
1906    projects.iter().position(|project| {
1907        project.read_with(cx, |project, cx| {
1908            let worktree = project.visible_worktrees(cx).next().unwrap();
1909            worktree.read(cx).root_name() == root_name
1910        })
1911    })
1912}
1913
1914fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1915    project.read_with(cx, |project, cx| {
1916        project
1917            .visible_worktrees(cx)
1918            .next()
1919            .unwrap()
1920            .read(cx)
1921            .root_name()
1922            .to_string()
1923    })
1924}
1925
1926fn project_path_for_full_path(
1927    project: &ModelHandle<Project>,
1928    full_path: &Path,
1929    cx: &TestAppContext,
1930) -> Option<ProjectPath> {
1931    let mut components = full_path.components();
1932    let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1933    let path = components.as_path().into();
1934    let worktree_id = project.read_with(cx, |project, cx| {
1935        project.worktrees(cx).find_map(|worktree| {
1936            let worktree = worktree.read(cx);
1937            if worktree.root_name() == root_name {
1938                Some(worktree.id())
1939            } else {
1940                None
1941            }
1942        })
1943    })?;
1944    Some(ProjectPath { worktree_id, path })
1945}
1946
1947async fn ensure_project_shared(
1948    project: &ModelHandle<Project>,
1949    client: &TestClient,
1950    cx: &mut TestAppContext,
1951) {
1952    let first_root_name = root_name_for_project(project, cx);
1953    let active_call = cx.read(ActiveCall::global);
1954    if active_call.read_with(cx, |call, _| call.room().is_some())
1955        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
1956    {
1957        match active_call
1958            .update(cx, |call, cx| call.share_project(project.clone(), cx))
1959            .await
1960        {
1961            Ok(project_id) => {
1962                log::info!(
1963                    "{}: shared project {} with id {}",
1964                    client.username,
1965                    first_root_name,
1966                    project_id
1967                );
1968            }
1969            Err(error) => {
1970                log::error!(
1971                    "{}: error sharing project {}: {:?}",
1972                    client.username,
1973                    first_root_name,
1974                    error
1975                );
1976            }
1977        }
1978    }
1979}
1980
1981fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1982    client
1983        .local_projects()
1984        .iter()
1985        .chain(client.remote_projects().iter())
1986        .choose(rng)
1987        .cloned()
1988}
1989
1990fn gen_file_name(rng: &mut StdRng) -> String {
1991    let mut name = String::new();
1992    for _ in 0..10 {
1993        let letter = rng.gen_range('a'..='z');
1994        name.push(letter);
1995    }
1996    name
1997}
1998
1999fn path_env_var(name: &str) -> Option<PathBuf> {
2000    let value = env::var(name).ok()?;
2001    let mut path = PathBuf::from(value);
2002    if path.is_relative() {
2003        let mut abs_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
2004        abs_path.pop();
2005        abs_path.pop();
2006        abs_path.push(path);
2007        path = abs_path
2008    }
2009    Some(path)
2010}