randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16, Rope};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::{
  18    distributions::{Alphanumeric, DistString},
  19    prelude::*,
  20};
  21use settings::Settings;
  22use std::{
  23    env,
  24    ffi::OsStr,
  25    path::{Path, PathBuf},
  26    sync::Arc,
  27};
  28
  29#[gpui::test(iterations = 100)]
  30async fn test_random_collaboration(
  31    cx: &mut TestAppContext,
  32    deterministic: Arc<Deterministic>,
  33    rng: StdRng,
  34) {
  35    deterministic.forbid_parking();
  36    let rng = Arc::new(Mutex::new(rng));
  37
  38    let max_peers = env::var("MAX_PEERS")
  39        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  40        .unwrap_or(5);
  41
  42    let max_operations = env::var("OPERATIONS")
  43        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  44        .unwrap_or(10);
  45
  46    let mut server = TestServer::start(&deterministic).await;
  47    let db = server.app_state.db.clone();
  48
  49    let mut available_users = Vec::new();
  50    for ix in 0..max_peers {
  51        let username = format!("user-{}", ix + 1);
  52        let user_id = db
  53            .create_user(
  54                &format!("{username}@example.com"),
  55                false,
  56                NewUserParams {
  57                    github_login: username.clone(),
  58                    github_user_id: (ix + 1) as i32,
  59                    invite_count: 0,
  60                },
  61            )
  62            .await
  63            .unwrap()
  64            .user_id;
  65        available_users.push((user_id, username));
  66    }
  67
  68    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  69        for (user_id_b, _) in &available_users[ix + 1..] {
  70            server
  71                .app_state
  72                .db
  73                .send_contact_request(*user_id_a, *user_id_b)
  74                .await
  75                .unwrap();
  76            server
  77                .app_state
  78                .db
  79                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  80                .await
  81                .unwrap();
  82        }
  83    }
  84
  85    let mut clients = Vec::new();
  86    let mut user_ids = Vec::new();
  87    let mut op_start_signals = Vec::new();
  88    let mut next_entity_id = 100000;
  89    let allow_server_restarts = rng.lock().gen_bool(0.7);
  90    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  91    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  92
  93    let mut operations = 0;
  94    while operations < max_operations {
  95        let distribution = rng.lock().gen_range(0..100);
  96        match distribution {
  97            0..=19 if !available_users.is_empty() => {
  98                let client_ix = rng.lock().gen_range(0..available_users.len());
  99                let (_, username) = available_users.remove(client_ix);
 100                log::info!("Adding new connection for {}", username);
 101                next_entity_id += 100000;
 102                let mut client_cx = TestAppContext::new(
 103                    cx.foreground_platform(),
 104                    cx.platform(),
 105                    deterministic.build_foreground(next_entity_id),
 106                    deterministic.build_background(),
 107                    cx.font_cache(),
 108                    cx.leak_detector(),
 109                    next_entity_id,
 110                    cx.function_name.clone(),
 111                );
 112
 113                client_cx.update(|cx| cx.set_global(Settings::test(cx)));
 114
 115                let op_start_signal = futures::channel::mpsc::unbounded();
 116                let client = server.create_client(&mut client_cx, &username).await;
 117                user_ids.push(client.current_user_id(&client_cx));
 118                op_start_signals.push(op_start_signal.0);
 119                clients.push(client_cx.foreground().spawn(simulate_client(
 120                    client,
 121                    op_start_signal.1,
 122                    allow_client_disconnection,
 123                    rng.clone(),
 124                    client_cx,
 125                )));
 126
 127                log::info!("Added connection for {}", username);
 128                operations += 1;
 129            }
 130
 131            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 132                let client_ix = rng.lock().gen_range(1..clients.len());
 133                log::info!(
 134                    "Simulating full disconnection of user {}",
 135                    user_ids[client_ix]
 136                );
 137                let removed_user_id = user_ids.remove(client_ix);
 138                let user_connection_ids = server
 139                    .connection_pool
 140                    .lock()
 141                    .user_connection_ids(removed_user_id)
 142                    .collect::<Vec<_>>();
 143                assert_eq!(user_connection_ids.len(), 1);
 144                let removed_peer_id = user_connection_ids[0].into();
 145                let client = clients.remove(client_ix);
 146                op_start_signals.remove(client_ix);
 147                server.forbid_connections();
 148                server.disconnect_client(removed_peer_id);
 149                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 150                deterministic.start_waiting();
 151                log::info!("Waiting for user {} to exit...", removed_user_id);
 152                let (client, mut client_cx) = client.await;
 153                deterministic.finish_waiting();
 154                server.allow_connections();
 155
 156                for project in &client.remote_projects {
 157                    project.read_with(&client_cx, |project, _| {
 158                        assert!(
 159                            project.is_read_only(),
 160                            "project {:?} should be read only",
 161                            project.remote_id()
 162                        )
 163                    });
 164                }
 165                for user_id in &user_ids {
 166                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 167                    let pool = server.connection_pool.lock();
 168                    for contact in contacts {
 169                        if let db::Contact::Accepted { user_id, busy, .. } = contact {
 170                            if user_id == removed_user_id {
 171                                assert!(!pool.is_user_online(user_id));
 172                                assert!(!busy);
 173                            }
 174                        }
 175                    }
 176                }
 177
 178                log::info!("{} removed", client.username);
 179                available_users.push((removed_user_id, client.username.clone()));
 180                client_cx.update(|cx| {
 181                    cx.clear_globals();
 182                    cx.set_global(Settings::test(cx));
 183                    drop(client);
 184                });
 185
 186                operations += 1;
 187            }
 188
 189            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 190                let client_ix = rng.lock().gen_range(1..clients.len());
 191                let user_id = user_ids[client_ix];
 192                log::info!("Simulating temporary disconnection of user {}", user_id);
 193                let user_connection_ids = server
 194                    .connection_pool
 195                    .lock()
 196                    .user_connection_ids(user_id)
 197                    .collect::<Vec<_>>();
 198                assert_eq!(user_connection_ids.len(), 1);
 199                let peer_id = user_connection_ids[0].into();
 200                server.disconnect_client(peer_id);
 201                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 202                operations += 1;
 203            }
 204
 205            30..=34 if allow_server_restarts => {
 206                log::info!("Simulating server restart");
 207                server.reset().await;
 208                deterministic.advance_clock(RECEIVE_TIMEOUT);
 209                server.start().await.unwrap();
 210                deterministic.advance_clock(CLEANUP_TIMEOUT);
 211                let environment = &server.app_state.config.zed_environment;
 212                let stale_room_ids = server
 213                    .app_state
 214                    .db
 215                    .stale_room_ids(environment, server.id())
 216                    .await
 217                    .unwrap();
 218                assert_eq!(stale_room_ids, vec![]);
 219            }
 220
 221            _ if !op_start_signals.is_empty() => {
 222                while operations < max_operations && rng.lock().gen_bool(0.7) {
 223                    op_start_signals
 224                        .choose(&mut *rng.lock())
 225                        .unwrap()
 226                        .unbounded_send(())
 227                        .unwrap();
 228                    operations += 1;
 229                }
 230
 231                if rng.lock().gen_bool(0.8) {
 232                    deterministic.run_until_parked();
 233                }
 234            }
 235            _ => {}
 236        }
 237    }
 238
 239    drop(op_start_signals);
 240    deterministic.start_waiting();
 241    let clients = futures::future::join_all(clients).await;
 242    deterministic.finish_waiting();
 243    deterministic.run_until_parked();
 244
 245    for (client, client_cx) in &clients {
 246        for guest_project in &client.remote_projects {
 247            guest_project.read_with(client_cx, |guest_project, cx| {
 248                let host_project = clients.iter().find_map(|(client, cx)| {
 249                    let project = client.local_projects.iter().find(|host_project| {
 250                        host_project.read_with(cx, |host_project, _| {
 251                            host_project.remote_id() == guest_project.remote_id()
 252                        })
 253                    })?;
 254                    Some((project, cx))
 255                });
 256
 257                if !guest_project.is_read_only() {
 258                    if let Some((host_project, host_cx)) = host_project {
 259                        let host_worktree_snapshots =
 260                            host_project.read_with(host_cx, |host_project, cx| {
 261                                host_project
 262                                    .worktrees(cx)
 263                                    .map(|worktree| {
 264                                        let worktree = worktree.read(cx);
 265                                        (worktree.id(), worktree.snapshot())
 266                                    })
 267                                    .collect::<BTreeMap<_, _>>()
 268                            });
 269                        let guest_worktree_snapshots = guest_project
 270                            .worktrees(cx)
 271                            .map(|worktree| {
 272                                let worktree = worktree.read(cx);
 273                                (worktree.id(), worktree.snapshot())
 274                            })
 275                            .collect::<BTreeMap<_, _>>();
 276
 277                        assert_eq!(
 278                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 279                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 280                            "{} has different worktrees than the host",
 281                            client.username
 282                        );
 283
 284                        for (id, host_snapshot) in &host_worktree_snapshots {
 285                            let guest_snapshot = &guest_worktree_snapshots[id];
 286                            assert_eq!(
 287                                guest_snapshot.root_name(),
 288                                host_snapshot.root_name(),
 289                                "{} has different root name than the host for worktree {}",
 290                                client.username,
 291                                id
 292                            );
 293                            assert_eq!(
 294                                guest_snapshot.abs_path(),
 295                                host_snapshot.abs_path(),
 296                                "{} has different abs path than the host for worktree {}",
 297                                client.username,
 298                                id
 299                            );
 300                            assert_eq!(
 301                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 302                                host_snapshot.entries(false).collect::<Vec<_>>(),
 303                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 304                                client.username,
 305                                id,
 306                                host_snapshot.abs_path(),
 307                                host_project.read_with(host_cx, |project, _| project.remote_id())
 308                            );
 309                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 310                        }
 311                    }
 312                }
 313
 314                guest_project.check_invariants(cx);
 315            });
 316        }
 317
 318        for (guest_project, guest_buffers) in &client.buffers {
 319            let project_id = if guest_project.read_with(client_cx, |project, _| {
 320                project.is_local() || project.is_read_only()
 321            }) {
 322                continue;
 323            } else {
 324                guest_project
 325                    .read_with(client_cx, |project, _| project.remote_id())
 326                    .unwrap()
 327            };
 328            let guest_user_id = client.user_id().unwrap();
 329
 330            let host_project = clients.iter().find_map(|(client, cx)| {
 331                let project = client.local_projects.iter().find(|host_project| {
 332                    host_project.read_with(cx, |host_project, _| {
 333                        host_project.remote_id() == Some(project_id)
 334                    })
 335                })?;
 336                Some((client.user_id().unwrap(), project, cx))
 337            });
 338
 339            let (host_user_id, host_project, host_cx) =
 340                if let Some((host_user_id, host_project, host_cx)) = host_project {
 341                    (host_user_id, host_project, host_cx)
 342                } else {
 343                    continue;
 344                };
 345
 346            for guest_buffer in guest_buffers {
 347                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 348                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 349                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 350                        panic!(
 351                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 352                            client.username,
 353                            client.peer_id(),
 354                            buffer_id
 355                        )
 356                    })
 357                });
 358                let path = host_buffer
 359                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 360
 361                assert_eq!(
 362                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 363                    0,
 364                    "{}, buffer {}, path {:?} has deferred operations",
 365                    client.username,
 366                    buffer_id,
 367                    path,
 368                );
 369                assert_eq!(
 370                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 371                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 372                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 373                    client.username,
 374                    buffer_id,
 375                    path
 376                );
 377
 378                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 379                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 380                match (host_file, guest_file) {
 381                    (Some(host_file), Some(guest_file)) => {
 382                        assert_eq!(guest_file.path(), host_file.path());
 383                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 384                        assert_eq!(
 385                            guest_file.mtime(),
 386                            host_file.mtime(),
 387                            "guest {} mtime does not match host {} for path {:?} in project {}",
 388                            guest_user_id,
 389                            host_user_id,
 390                            guest_file.path(),
 391                            project_id,
 392                        );
 393                    }
 394                    (None, None) => {}
 395                    (None, _) => panic!("host's file is None, guest's isn't"),
 396                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 397                }
 398
 399                let host_diff_base =
 400                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 401                let guest_diff_base = guest_buffer
 402                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 403                assert_eq!(guest_diff_base, host_diff_base);
 404
 405                let host_saved_version =
 406                    host_buffer.read_with(host_cx, |b, _| b.saved_version().clone());
 407                let guest_saved_version =
 408                    guest_buffer.read_with(client_cx, |b, _| b.saved_version().clone());
 409                assert_eq!(guest_saved_version, host_saved_version);
 410
 411                let host_saved_version_fingerprint =
 412                    host_buffer.read_with(host_cx, |b, _| b.saved_version_fingerprint());
 413                let guest_saved_version_fingerprint =
 414                    guest_buffer.read_with(client_cx, |b, _| b.saved_version_fingerprint());
 415                assert_eq!(
 416                    guest_saved_version_fingerprint,
 417                    host_saved_version_fingerprint
 418                );
 419
 420                let host_saved_mtime = host_buffer.read_with(host_cx, |b, _| b.saved_mtime());
 421                let guest_saved_mtime = guest_buffer.read_with(client_cx, |b, _| b.saved_mtime());
 422                assert_eq!(guest_saved_mtime, host_saved_mtime);
 423
 424                let host_is_dirty = host_buffer.read_with(host_cx, |b, _| b.is_dirty());
 425                let guest_is_dirty = guest_buffer.read_with(client_cx, |b, _| b.is_dirty());
 426                assert_eq!(guest_is_dirty, host_is_dirty);
 427
 428                let host_has_conflict = host_buffer.read_with(host_cx, |b, _| b.has_conflict());
 429                let guest_has_conflict = guest_buffer.read_with(client_cx, |b, _| b.has_conflict());
 430                assert_eq!(guest_has_conflict, host_has_conflict);
 431            }
 432        }
 433    }
 434
 435    for (client, mut cx) in clients {
 436        cx.update(|cx| {
 437            cx.clear_globals();
 438            cx.set_global(Settings::test(cx));
 439            drop(client);
 440        });
 441    }
 442}
 443
 444async fn simulate_client(
 445    mut client: TestClient,
 446    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 447    can_hang_up: bool,
 448    rng: Arc<Mutex<StdRng>>,
 449    mut cx: TestAppContext,
 450) -> (TestClient, TestAppContext) {
 451    // Setup language server
 452    let mut language = Language::new(
 453        LanguageConfig {
 454            name: "Rust".into(),
 455            path_suffixes: vec!["rs".to_string()],
 456            ..Default::default()
 457        },
 458        None,
 459    );
 460    let _fake_language_servers = language
 461        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 462            name: "the-fake-language-server",
 463            capabilities: lsp::LanguageServer::full_capabilities(),
 464            initializer: Some(Box::new({
 465                let rng = rng.clone();
 466                let fs = client.fs.clone();
 467                move |fake_server: &mut FakeLanguageServer| {
 468                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 469                        |_, _| async move {
 470                            Ok(Some(lsp::CompletionResponse::Array(vec![
 471                                lsp::CompletionItem {
 472                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 473                                        range: lsp::Range::new(
 474                                            lsp::Position::new(0, 0),
 475                                            lsp::Position::new(0, 0),
 476                                        ),
 477                                        new_text: "the-new-text".to_string(),
 478                                    })),
 479                                    ..Default::default()
 480                                },
 481                            ])))
 482                        },
 483                    );
 484
 485                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 486                        |_, _| async move {
 487                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 488                                lsp::CodeAction {
 489                                    title: "the-code-action".to_string(),
 490                                    ..Default::default()
 491                                },
 492                            )]))
 493                        },
 494                    );
 495
 496                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 497                        |params, _| async move {
 498                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 499                                params.position,
 500                                params.position,
 501                            ))))
 502                        },
 503                    );
 504
 505                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 506                        let fs = fs.clone();
 507                        let rng = rng.clone();
 508                        move |_, _| {
 509                            let fs = fs.clone();
 510                            let rng = rng.clone();
 511                            async move {
 512                                let files = fs.files().await;
 513                                let mut rng = rng.lock();
 514                                let count = rng.gen_range::<usize, _>(1..3);
 515                                let files = (0..count)
 516                                    .map(|_| files.choose(&mut *rng).unwrap())
 517                                    .collect::<Vec<_>>();
 518                                log::info!("LSP: Returning definitions in files {:?}", &files);
 519                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 520                                    files
 521                                        .into_iter()
 522                                        .map(|file| lsp::Location {
 523                                            uri: lsp::Url::from_file_path(file).unwrap(),
 524                                            range: Default::default(),
 525                                        })
 526                                        .collect(),
 527                                )))
 528                            }
 529                        }
 530                    });
 531
 532                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 533                        let rng = rng.clone();
 534                        move |_, _| {
 535                            let mut highlights = Vec::new();
 536                            let highlight_count = rng.lock().gen_range(1..=5);
 537                            for _ in 0..highlight_count {
 538                                let start_row = rng.lock().gen_range(0..100);
 539                                let start_column = rng.lock().gen_range(0..100);
 540                                let start = PointUtf16::new(start_row, start_column);
 541                                let end_row = rng.lock().gen_range(0..100);
 542                                let end_column = rng.lock().gen_range(0..100);
 543                                let end = PointUtf16::new(end_row, end_column);
 544                                let range = if start > end { end..start } else { start..end };
 545                                highlights.push(lsp::DocumentHighlight {
 546                                    range: range_to_lsp(range.clone()),
 547                                    kind: Some(lsp::DocumentHighlightKind::READ),
 548                                });
 549                            }
 550                            highlights.sort_unstable_by_key(|highlight| {
 551                                (highlight.range.start, highlight.range.end)
 552                            });
 553                            async move { Ok(Some(highlights)) }
 554                        }
 555                    });
 556                }
 557            })),
 558            ..Default::default()
 559        }))
 560        .await;
 561    client.language_registry.add(Arc::new(language));
 562
 563    while op_start_signal.next().await.is_some() {
 564        if let Err(error) =
 565            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 566        {
 567            log::error!("{} error: {:?}", client.username, error);
 568        }
 569
 570        cx.background().simulate_random_delay().await;
 571    }
 572    log::info!("{}: done", client.username);
 573
 574    (client, cx)
 575}
 576
 577async fn randomly_mutate_client(
 578    client: &mut TestClient,
 579    can_hang_up: bool,
 580    rng: Arc<Mutex<StdRng>>,
 581    cx: &mut TestAppContext,
 582) -> Result<()> {
 583    let choice = rng.lock().gen_range(0..100);
 584    match choice {
 585        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 586        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 587        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 588            randomly_mutate_worktrees(client, &rng, cx).await?;
 589        }
 590        60..=74 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 591            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 592        }
 593        75..=84 => randomly_mutate_git(client, &rng).await,
 594        _ => randomly_mutate_fs(client, &rng).await,
 595    }
 596
 597    Ok(())
 598}
 599
 600async fn randomly_mutate_active_call(
 601    client: &mut TestClient,
 602    can_hang_up: bool,
 603    rng: &Mutex<StdRng>,
 604    cx: &mut TestAppContext,
 605) -> Result<()> {
 606    let active_call = cx.read(ActiveCall::global);
 607    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 608        if rng.lock().gen_bool(0.7) {
 609            log::info!("{}: accepting incoming call", client.username);
 610            active_call
 611                .update(cx, |call, cx| call.accept_incoming(cx))
 612                .await?;
 613        } else {
 614            log::info!("{}: declining incoming call", client.username);
 615            active_call.update(cx, |call, _| call.decline_incoming())?;
 616        }
 617    } else {
 618        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 619            user_store
 620                .contacts()
 621                .iter()
 622                .filter(|contact| contact.online && !contact.busy)
 623                .cloned()
 624                .collect::<Vec<_>>()
 625        });
 626
 627        let distribution = rng.lock().gen_range(0..100);
 628        match distribution {
 629            0..=29 if !available_contacts.is_empty() => {
 630                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 631                log::info!(
 632                    "{}: inviting {}",
 633                    client.username,
 634                    contact.user.github_login
 635                );
 636                active_call
 637                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 638                    .await?;
 639            }
 640            30..=39
 641                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 642            {
 643                log::info!("{}: hanging up", client.username);
 644                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 645            }
 646            _ => {}
 647        }
 648    }
 649
 650    Ok(())
 651}
 652
 653async fn randomly_mutate_git(client: &mut TestClient, rng: &Mutex<StdRng>) {
 654    let directories = client.fs.directories().await;
 655    let mut dir_path = directories.choose(&mut *rng.lock()).unwrap().clone();
 656    if dir_path.file_name() == Some(OsStr::new(".git")) {
 657        dir_path.pop();
 658    }
 659    let mut git_dir_path = dir_path.clone();
 660    git_dir_path.push(".git");
 661
 662    if !directories.contains(&git_dir_path) {
 663        log::info!(
 664            "{}: creating git directory at {:?}",
 665            client.username,
 666            git_dir_path
 667        );
 668        client.fs.create_dir(&git_dir_path).await.unwrap();
 669    }
 670
 671    let mut child_file_paths = child_file_paths(client, &dir_path).await;
 672    let count = rng.lock().gen_range(0..=child_file_paths.len());
 673    child_file_paths.shuffle(&mut *rng.lock());
 674    child_file_paths.truncate(count);
 675
 676    let mut new_index = Vec::new();
 677    for abs_child_file_path in &child_file_paths {
 678        let child_file_path = abs_child_file_path.strip_prefix(&dir_path).unwrap();
 679        let new_base = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 680        new_index.push((child_file_path, new_base));
 681    }
 682    log::info!(
 683        "{}: updating Git index at {:?}: {:#?}",
 684        client.username,
 685        git_dir_path,
 686        new_index
 687    );
 688    client
 689        .fs
 690        .set_index_for_repo(&git_dir_path, &new_index)
 691        .await;
 692}
 693
 694async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 695    let parent_dir_path = client
 696        .fs
 697        .directories()
 698        .await
 699        .choose(&mut *rng.lock())
 700        .unwrap()
 701        .clone();
 702
 703    let is_dir = rng.lock().gen::<bool>();
 704    if is_dir {
 705        let mut dir_path = parent_dir_path.clone();
 706        dir_path.push(gen_file_name(rng));
 707        log::info!("{}: creating local dir at {:?}", client.username, dir_path);
 708        client.fs.create_dir(&dir_path).await.unwrap();
 709    } else {
 710        let child_file_paths = child_file_paths(client, &parent_dir_path).await;
 711        let create_new_file = child_file_paths.is_empty() || rng.lock().gen();
 712        let text = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 713        if create_new_file {
 714            let mut file_path = parent_dir_path.clone();
 715            file_path.push(gen_file_name(rng));
 716            file_path.set_extension("rs");
 717            log::info!(
 718                "{}: creating local file at {:?}",
 719                client.username,
 720                file_path
 721            );
 722            client
 723                .fs
 724                .create_file(&file_path, Default::default())
 725                .await
 726                .unwrap();
 727            log::info!(
 728                "{}: setting local file {:?} text to {:?}",
 729                client.username,
 730                file_path,
 731                text
 732            );
 733            client
 734                .fs
 735                .save(&file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 736                .await
 737                .unwrap();
 738        } else {
 739            let file_path = child_file_paths.choose(&mut *rng.lock()).unwrap();
 740            log::info!(
 741                "{}: setting local file {:?} text to {:?}",
 742                client.username,
 743                file_path,
 744                text
 745            );
 746            client
 747                .fs
 748                .save(file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 749                .await
 750                .unwrap();
 751        }
 752    }
 753}
 754
 755async fn randomly_mutate_projects(
 756    client: &mut TestClient,
 757    rng: &Mutex<StdRng>,
 758    cx: &mut TestAppContext,
 759) -> Result<()> {
 760    let active_call = cx.read(ActiveCall::global);
 761    let remote_projects =
 762        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 763            room.read_with(cx, |room, _| {
 764                room.remote_participants()
 765                    .values()
 766                    .flat_map(|participant| participant.projects.clone())
 767                    .collect::<Vec<_>>()
 768            })
 769        } else {
 770            Default::default()
 771        };
 772
 773    let project = if remote_projects.is_empty() || rng.lock().gen() {
 774        if client.local_projects.is_empty() || rng.lock().gen() {
 775            let paths = client.fs.paths().await;
 776            let local_project = if paths.is_empty() || rng.lock().gen() {
 777                let root_path = client.create_new_root_dir();
 778                client.fs.create_dir(&root_path).await.unwrap();
 779                client
 780                    .fs
 781                    .create_file(&root_path.join("main.rs"), Default::default())
 782                    .await
 783                    .unwrap();
 784                log::info!(
 785                    "{}: opening local project at {:?}",
 786                    client.username,
 787                    root_path
 788                );
 789                client.build_local_project(root_path, cx).await.0
 790            } else {
 791                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 792                log::info!(
 793                    "{}: opening local project at {:?}",
 794                    client.username,
 795                    root_path
 796                );
 797                client.build_local_project(root_path, cx).await.0
 798            };
 799            client.local_projects.push(local_project.clone());
 800            local_project
 801        } else {
 802            client
 803                .local_projects
 804                .choose(&mut *rng.lock())
 805                .unwrap()
 806                .clone()
 807        }
 808    } else {
 809        if client.remote_projects.is_empty() || rng.lock().gen() {
 810            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 811            let remote_project = if let Some(project) =
 812                client.remote_projects.iter().find(|project| {
 813                    project.read_with(cx, |project, _| {
 814                        project.remote_id() == Some(remote_project_id)
 815                    })
 816                }) {
 817                project.clone()
 818            } else {
 819                log::info!(
 820                    "{}: opening remote project {}",
 821                    client.username,
 822                    remote_project_id
 823                );
 824                let call = cx.read(ActiveCall::global);
 825                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 826                let remote_project = room
 827                    .update(cx, |room, cx| {
 828                        room.join_project(
 829                            remote_project_id,
 830                            client.language_registry.clone(),
 831                            FakeFs::new(cx.background().clone()),
 832                            cx,
 833                        )
 834                    })
 835                    .await?;
 836                client.remote_projects.push(remote_project.clone());
 837                remote_project
 838            };
 839
 840            remote_project
 841        } else {
 842            client
 843                .remote_projects
 844                .choose(&mut *rng.lock())
 845                .unwrap()
 846                .clone()
 847        }
 848    };
 849
 850    if active_call.read_with(cx, |call, _| call.room().is_some())
 851        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 852    {
 853        match active_call
 854            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 855            .await
 856        {
 857            Ok(project_id) => {
 858                log::info!("{}: shared project with id {}", client.username, project_id);
 859            }
 860            Err(error) => {
 861                log::error!("{}: error sharing project, {:?}", client.username, error);
 862            }
 863        }
 864    }
 865
 866    let choice = rng.lock().gen_range(0..100);
 867    match choice {
 868        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 869            let paths = client.fs.paths().await;
 870            let path = paths.choose(&mut *rng.lock()).unwrap();
 871            log::info!(
 872                "{}: finding/creating local worktree for path {:?}",
 873                client.username,
 874                path
 875            );
 876            project
 877                .update(cx, |project, cx| {
 878                    project.find_or_create_local_worktree(&path, true, cx)
 879                })
 880                .await
 881                .unwrap();
 882        }
 883        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 884            log::info!(
 885                "{}: dropping remote project {}",
 886                client.username,
 887                project.read_with(cx, |project, _| project.remote_id().unwrap())
 888            );
 889
 890            cx.update(|_| {
 891                client
 892                    .remote_projects
 893                    .retain(|remote_project| *remote_project != project);
 894                client.buffers.remove(&project);
 895                drop(project);
 896            });
 897        }
 898        _ => {}
 899    }
 900
 901    Ok(())
 902}
 903
 904async fn randomly_mutate_worktrees(
 905    client: &mut TestClient,
 906    rng: &Mutex<StdRng>,
 907    cx: &mut TestAppContext,
 908) -> Result<()> {
 909    let project = choose_random_project(client, rng).unwrap();
 910    let Some(worktree) = project.read_with(cx, |project, cx| {
 911        project
 912            .worktrees(cx)
 913            .filter(|worktree| {
 914                let worktree = worktree.read(cx);
 915                worktree.is_visible()
 916                    && worktree.entries(false).any(|e| e.is_file())
 917                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 918            })
 919            .choose(&mut *rng.lock())
 920    }) else {
 921        return Ok(())
 922    };
 923
 924    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 925        (worktree.id(), worktree.root_name().to_string())
 926    });
 927
 928    let is_dir = rng.lock().gen::<bool>();
 929    let mut new_path = PathBuf::new();
 930    new_path.push(gen_file_name(rng));
 931    if !is_dir {
 932        new_path.set_extension("rs");
 933    }
 934    log::info!(
 935        "{}: creating {:?} in worktree {} ({})",
 936        client.username,
 937        new_path,
 938        worktree_id,
 939        worktree_root_name,
 940    );
 941    project
 942        .update(cx, |project, cx| {
 943            project.create_entry((worktree_id, new_path), is_dir, cx)
 944        })
 945        .unwrap()
 946        .await?;
 947    Ok(())
 948}
 949
 950async fn randomly_query_and_mutate_buffers(
 951    client: &mut TestClient,
 952    rng: &Mutex<StdRng>,
 953    cx: &mut TestAppContext,
 954) -> Result<()> {
 955    let project = choose_random_project(client, rng).unwrap();
 956    let buffers = client.buffers.entry(project.clone()).or_default();
 957    let buffer = if buffers.is_empty() || rng.lock().gen() {
 958        let Some(worktree) = project.read_with(cx, |project, cx| {
 959            project
 960                .worktrees(cx)
 961                .filter(|worktree| {
 962                    let worktree = worktree.read(cx);
 963                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 964                })
 965                .choose(&mut *rng.lock())
 966        }) else {
 967            return Ok(());
 968        };
 969
 970        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 971            let entry = worktree
 972                .entries(false)
 973                .filter(|e| e.is_file())
 974                .choose(&mut *rng.lock())
 975                .unwrap();
 976            (
 977                worktree.root_name().to_string(),
 978                (worktree.id(), entry.path.clone()),
 979            )
 980        });
 981        log::info!(
 982            "{}: opening path {:?} in worktree {} ({})",
 983            client.username,
 984            project_path.1,
 985            project_path.0,
 986            worktree_root_name,
 987        );
 988        let buffer = project
 989            .update(cx, |project, cx| {
 990                project.open_buffer(project_path.clone(), cx)
 991            })
 992            .await?;
 993        log::info!(
 994            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 995            client.username,
 996            project_path.1,
 997            project_path.0,
 998            worktree_root_name,
 999            buffer.read_with(cx, |buffer, _| buffer.remote_id())
1000        );
1001        buffers.insert(buffer.clone());
1002        buffer
1003    } else {
1004        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
1005    };
1006
1007    let choice = rng.lock().gen_range(0..100);
1008    match choice {
1009        0..=9 => {
1010            cx.update(|cx| {
1011                log::info!(
1012                    "{}: dropping buffer {:?}",
1013                    client.username,
1014                    buffer.read(cx).file().unwrap().full_path(cx)
1015                );
1016                buffers.remove(&buffer);
1017                drop(buffer);
1018            });
1019        }
1020        10..=19 => {
1021            let completions = project.update(cx, |project, cx| {
1022                log::info!(
1023                    "{}: requesting completions for buffer {} ({:?})",
1024                    client.username,
1025                    buffer.read(cx).remote_id(),
1026                    buffer.read(cx).file().unwrap().full_path(cx)
1027                );
1028                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1029                project.completions(&buffer, offset, cx)
1030            });
1031            let completions = cx.background().spawn(async move {
1032                completions
1033                    .await
1034                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
1035            });
1036            if rng.lock().gen_bool(0.3) {
1037                log::info!("{}: detaching completions request", client.username);
1038                cx.update(|cx| completions.detach_and_log_err(cx));
1039            } else {
1040                completions.await?;
1041            }
1042        }
1043        20..=29 => {
1044            let code_actions = project.update(cx, |project, cx| {
1045                log::info!(
1046                    "{}: requesting code actions for buffer {} ({:?})",
1047                    client.username,
1048                    buffer.read(cx).remote_id(),
1049                    buffer.read(cx).file().unwrap().full_path(cx)
1050                );
1051                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
1052                project.code_actions(&buffer, range, cx)
1053            });
1054            let code_actions = cx.background().spawn(async move {
1055                code_actions
1056                    .await
1057                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
1058            });
1059            if rng.lock().gen_bool(0.3) {
1060                log::info!("{}: detaching code actions request", client.username);
1061                cx.update(|cx| code_actions.detach_and_log_err(cx));
1062            } else {
1063                code_actions.await?;
1064            }
1065        }
1066        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1067            let requested_version = buffer.update(cx, |buffer, cx| {
1068                log::info!(
1069                    "{}: saving buffer {} ({:?})",
1070                    client.username,
1071                    buffer.remote_id(),
1072                    buffer.file().unwrap().full_path(cx)
1073                );
1074                buffer.version()
1075            });
1076            let save = project.update(cx, |project, cx| project.save_buffer(buffer, cx));
1077            let save = cx.background().spawn(async move {
1078                let (saved_version, _, _) = save
1079                    .await
1080                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1081                assert!(saved_version.observed_all(&requested_version));
1082                Ok::<_, anyhow::Error>(())
1083            });
1084            if rng.lock().gen_bool(0.3) {
1085                log::info!("{}: detaching save request", client.username);
1086                cx.update(|cx| save.detach_and_log_err(cx));
1087            } else {
1088                save.await?;
1089            }
1090        }
1091        40..=44 => {
1092            let prepare_rename = project.update(cx, |project, cx| {
1093                log::info!(
1094                    "{}: preparing rename for buffer {} ({:?})",
1095                    client.username,
1096                    buffer.read(cx).remote_id(),
1097                    buffer.read(cx).file().unwrap().full_path(cx)
1098                );
1099                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1100                project.prepare_rename(buffer, offset, cx)
1101            });
1102            let prepare_rename = cx.background().spawn(async move {
1103                prepare_rename
1104                    .await
1105                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1106            });
1107            if rng.lock().gen_bool(0.3) {
1108                log::info!("{}: detaching prepare rename request", client.username);
1109                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1110            } else {
1111                prepare_rename.await?;
1112            }
1113        }
1114        45..=49 => {
1115            let definitions = project.update(cx, |project, cx| {
1116                log::info!(
1117                    "{}: requesting definitions for buffer {} ({:?})",
1118                    client.username,
1119                    buffer.read(cx).remote_id(),
1120                    buffer.read(cx).file().unwrap().full_path(cx)
1121                );
1122                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1123                project.definition(&buffer, offset, cx)
1124            });
1125            let definitions = cx.background().spawn(async move {
1126                definitions
1127                    .await
1128                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1129            });
1130            if rng.lock().gen_bool(0.3) {
1131                log::info!("{}: detaching definitions request", client.username);
1132                cx.update(|cx| definitions.detach_and_log_err(cx));
1133            } else {
1134                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1135            }
1136        }
1137        50..=54 => {
1138            let highlights = project.update(cx, |project, cx| {
1139                log::info!(
1140                    "{}: requesting highlights for buffer {} ({:?})",
1141                    client.username,
1142                    buffer.read(cx).remote_id(),
1143                    buffer.read(cx).file().unwrap().full_path(cx)
1144                );
1145                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1146                project.document_highlights(&buffer, offset, cx)
1147            });
1148            let highlights = cx.background().spawn(async move {
1149                highlights
1150                    .await
1151                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1152            });
1153            if rng.lock().gen_bool(0.3) {
1154                log::info!("{}: detaching highlights request", client.username);
1155                cx.update(|cx| highlights.detach_and_log_err(cx));
1156            } else {
1157                highlights.await?;
1158            }
1159        }
1160        55..=59 => {
1161            let search = project.update(cx, |project, cx| {
1162                let query = rng.lock().gen_range('a'..='z');
1163                log::info!("{}: project-wide search {:?}", client.username, query);
1164                project.search(SearchQuery::text(query, false, false), cx)
1165            });
1166            let search = cx.background().spawn(async move {
1167                search
1168                    .await
1169                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1170            });
1171            if rng.lock().gen_bool(0.3) {
1172                log::info!("{}: detaching search request", client.username);
1173                cx.update(|cx| search.detach_and_log_err(cx));
1174            } else {
1175                buffers.extend(search.await?.into_keys());
1176            }
1177        }
1178        _ => {
1179            buffer.update(cx, |buffer, cx| {
1180                log::info!(
1181                    "{}: updating buffer {} ({:?})",
1182                    client.username,
1183                    buffer.remote_id(),
1184                    buffer.file().unwrap().full_path(cx)
1185                );
1186                if rng.lock().gen_bool(0.7) {
1187                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1188                } else {
1189                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1190                }
1191            });
1192        }
1193    }
1194
1195    Ok(())
1196}
1197
1198fn choose_random_project(
1199    client: &mut TestClient,
1200    rng: &Mutex<StdRng>,
1201) -> Option<ModelHandle<Project>> {
1202    client
1203        .local_projects
1204        .iter()
1205        .chain(&client.remote_projects)
1206        .choose(&mut *rng.lock())
1207        .cloned()
1208}
1209
1210fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1211    let mut name = String::new();
1212    for _ in 0..10 {
1213        let letter = rng.lock().gen_range('a'..='z');
1214        name.push(letter);
1215    }
1216    name
1217}
1218
1219async fn child_file_paths(client: &TestClient, dir_path: &Path) -> Vec<PathBuf> {
1220    let mut child_paths = client.fs.read_dir(dir_path).await.unwrap();
1221    let mut child_file_paths = Vec::new();
1222    while let Some(child_path) = child_paths.next().await {
1223        let child_path = child_path.unwrap();
1224        if client.fs.is_file(&child_path).await {
1225            child_file_paths.push(child_path);
1226        }
1227    }
1228    child_file_paths
1229}