randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::prelude::*;
  18use std::{env, path::PathBuf, sync::Arc};
  19
  20#[gpui::test(iterations = 1, seed = 4742)]
  21async fn test_random_collaboration(
  22    cx: &mut TestAppContext,
  23    deterministic: Arc<Deterministic>,
  24    rng: StdRng,
  25) {
  26    deterministic.forbid_parking();
  27    let rng = Arc::new(Mutex::new(rng));
  28
  29    let max_peers = env::var("MAX_PEERS")
  30        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  31        .unwrap_or(5);
  32
  33    let max_operations = env::var("OPERATIONS")
  34        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  35        .unwrap_or(10);
  36
  37    let mut server = TestServer::start(&deterministic).await;
  38    let db = server.app_state.db.clone();
  39
  40    let mut available_users = Vec::new();
  41    for ix in 0..max_peers {
  42        let username = format!("user-{}", ix + 1);
  43        let user_id = db
  44            .create_user(
  45                &format!("{username}@example.com"),
  46                false,
  47                NewUserParams {
  48                    github_login: username.clone(),
  49                    github_user_id: (ix + 1) as i32,
  50                    invite_count: 0,
  51                },
  52            )
  53            .await
  54            .unwrap()
  55            .user_id;
  56        available_users.push((user_id, username));
  57    }
  58
  59    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  60        for (user_id_b, _) in &available_users[ix + 1..] {
  61            server
  62                .app_state
  63                .db
  64                .send_contact_request(*user_id_a, *user_id_b)
  65                .await
  66                .unwrap();
  67            server
  68                .app_state
  69                .db
  70                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  71                .await
  72                .unwrap();
  73        }
  74    }
  75
  76    let mut clients = Vec::new();
  77    let mut user_ids = Vec::new();
  78    let mut op_start_signals = Vec::new();
  79    let mut next_entity_id = 100000;
  80    let allow_server_restarts = rng.lock().gen_bool(0.7);
  81    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  82    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  83
  84    let mut operations = 0;
  85    while operations < max_operations {
  86        let distribution = rng.lock().gen_range(0..100);
  87        match distribution {
  88            0..=19 if !available_users.is_empty() => {
  89                let client_ix = rng.lock().gen_range(0..available_users.len());
  90                let (_, username) = available_users.remove(client_ix);
  91                log::info!("Adding new connection for {}", username);
  92                next_entity_id += 100000;
  93                let mut client_cx = TestAppContext::new(
  94                    cx.foreground_platform(),
  95                    cx.platform(),
  96                    deterministic.build_foreground(next_entity_id),
  97                    deterministic.build_background(),
  98                    cx.font_cache(),
  99                    cx.leak_detector(),
 100                    next_entity_id,
 101                    cx.function_name.clone(),
 102                );
 103
 104                let op_start_signal = futures::channel::mpsc::unbounded();
 105                let client = server.create_client(&mut client_cx, &username).await;
 106                user_ids.push(client.current_user_id(&client_cx));
 107                op_start_signals.push(op_start_signal.0);
 108                clients.push(client_cx.foreground().spawn(simulate_client(
 109                    client,
 110                    op_start_signal.1,
 111                    allow_client_disconnection,
 112                    rng.clone(),
 113                    client_cx,
 114                )));
 115
 116                log::info!("Added connection for {}", username);
 117                operations += 1;
 118            }
 119
 120            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 121                let client_ix = rng.lock().gen_range(1..clients.len());
 122                log::info!(
 123                    "Simulating full disconnection of user {}",
 124                    user_ids[client_ix]
 125                );
 126                let removed_user_id = user_ids.remove(client_ix);
 127                let user_connection_ids = server
 128                    .connection_pool
 129                    .lock()
 130                    .user_connection_ids(removed_user_id)
 131                    .collect::<Vec<_>>();
 132                assert_eq!(user_connection_ids.len(), 1);
 133                let removed_peer_id = user_connection_ids[0].into();
 134                let client = clients.remove(client_ix);
 135                op_start_signals.remove(client_ix);
 136                server.forbid_connections();
 137                server.disconnect_client(removed_peer_id);
 138                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 139                deterministic.start_waiting();
 140                log::info!("Waiting for user {} to exit...", removed_user_id);
 141                let (client, mut client_cx) = client.await;
 142                deterministic.finish_waiting();
 143                server.allow_connections();
 144
 145                for project in &client.remote_projects {
 146                    project.read_with(&client_cx, |project, _| {
 147                        assert!(
 148                            project.is_read_only(),
 149                            "project {:?} should be read only",
 150                            project.remote_id()
 151                        )
 152                    });
 153                }
 154                for user_id in &user_ids {
 155                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 156                    let pool = server.connection_pool.lock();
 157                    for contact in contacts {
 158                        if let db::Contact::Accepted { user_id, .. } = contact {
 159                            if pool.is_user_online(user_id) {
 160                                assert_ne!(
 161                                    user_id, removed_user_id,
 162                                    "removed client is still a contact of another peer"
 163                                );
 164                            }
 165                        }
 166                    }
 167                }
 168
 169                log::info!("{} removed", client.username);
 170                available_users.push((removed_user_id, client.username.clone()));
 171                client_cx.update(|cx| {
 172                    cx.clear_globals();
 173                    drop(client);
 174                });
 175
 176                operations += 1;
 177            }
 178
 179            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 180                let client_ix = rng.lock().gen_range(1..clients.len());
 181                let user_id = user_ids[client_ix];
 182                log::info!("Simulating temporary disconnection of user {}", user_id);
 183                let user_connection_ids = server
 184                    .connection_pool
 185                    .lock()
 186                    .user_connection_ids(user_id)
 187                    .collect::<Vec<_>>();
 188                assert_eq!(user_connection_ids.len(), 1);
 189                let peer_id = user_connection_ids[0].into();
 190                server.disconnect_client(peer_id);
 191                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 192                operations += 1;
 193            }
 194
 195            30..=34 if allow_server_restarts => {
 196                log::info!("Simulating server restart");
 197                server.reset().await;
 198                deterministic.advance_clock(RECEIVE_TIMEOUT);
 199                server.start().await.unwrap();
 200                deterministic.advance_clock(CLEANUP_TIMEOUT);
 201                let environment = &server.app_state.config.zed_environment;
 202                let stale_room_ids = server
 203                    .app_state
 204                    .db
 205                    .stale_room_ids(environment, server.id())
 206                    .await
 207                    .unwrap();
 208                assert_eq!(stale_room_ids, vec![]);
 209            }
 210
 211            _ if !op_start_signals.is_empty() => {
 212                while operations < max_operations && rng.lock().gen_bool(0.7) {
 213                    op_start_signals
 214                        .choose(&mut *rng.lock())
 215                        .unwrap()
 216                        .unbounded_send(())
 217                        .unwrap();
 218                    operations += 1;
 219                }
 220
 221                if rng.lock().gen_bool(0.8) {
 222                    deterministic.run_until_parked();
 223                }
 224            }
 225            _ => {}
 226        }
 227    }
 228
 229    drop(op_start_signals);
 230    deterministic.start_waiting();
 231    let clients = futures::future::join_all(clients).await;
 232    deterministic.finish_waiting();
 233    deterministic.run_until_parked();
 234
 235    for (client, client_cx) in &clients {
 236        for guest_project in &client.remote_projects {
 237            guest_project.read_with(client_cx, |guest_project, cx| {
 238                let host_project = clients.iter().find_map(|(client, cx)| {
 239                    let project = client.local_projects.iter().find(|host_project| {
 240                        host_project.read_with(cx, |host_project, _| {
 241                            host_project.remote_id() == guest_project.remote_id()
 242                        })
 243                    })?;
 244                    Some((project, cx))
 245                });
 246
 247                if !guest_project.is_read_only() {
 248                    if let Some((host_project, host_cx)) = host_project {
 249                        let host_worktree_snapshots =
 250                            host_project.read_with(host_cx, |host_project, cx| {
 251                                host_project
 252                                    .worktrees(cx)
 253                                    .map(|worktree| {
 254                                        let worktree = worktree.read(cx);
 255                                        (worktree.id(), worktree.snapshot())
 256                                    })
 257                                    .collect::<BTreeMap<_, _>>()
 258                            });
 259                        let guest_worktree_snapshots = guest_project
 260                            .worktrees(cx)
 261                            .map(|worktree| {
 262                                let worktree = worktree.read(cx);
 263                                (worktree.id(), worktree.snapshot())
 264                            })
 265                            .collect::<BTreeMap<_, _>>();
 266
 267                        assert_eq!(
 268                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 269                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 270                            "{} has different worktrees than the host",
 271                            client.username
 272                        );
 273
 274                        for (id, host_snapshot) in &host_worktree_snapshots {
 275                            let guest_snapshot = &guest_worktree_snapshots[id];
 276                            assert_eq!(
 277                                guest_snapshot.root_name(),
 278                                host_snapshot.root_name(),
 279                                "{} has different root name than the host for worktree {}",
 280                                client.username,
 281                                id
 282                            );
 283                            assert_eq!(
 284                                guest_snapshot.abs_path(),
 285                                host_snapshot.abs_path(),
 286                                "{} has different abs path than the host for worktree {}",
 287                                client.username,
 288                                id
 289                            );
 290                            assert_eq!(
 291                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 292                                host_snapshot.entries(false).collect::<Vec<_>>(),
 293                                "{} has different snapshot than the host for worktree {} ({:?})",
 294                                client.username,
 295                                id,
 296                                host_snapshot.abs_path()
 297                            );
 298                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 299                        }
 300                    }
 301                }
 302
 303                guest_project.check_invariants(cx);
 304            });
 305        }
 306
 307        for (guest_project, guest_buffers) in &client.buffers {
 308            let project_id = if guest_project.read_with(client_cx, |project, _| {
 309                project.is_local() || project.is_read_only()
 310            }) {
 311                continue;
 312            } else {
 313                guest_project
 314                    .read_with(client_cx, |project, _| project.remote_id())
 315                    .unwrap()
 316            };
 317
 318            let host_project = clients.iter().find_map(|(client, cx)| {
 319                let project = client.local_projects.iter().find(|host_project| {
 320                    host_project.read_with(cx, |host_project, _| {
 321                        host_project.remote_id() == Some(project_id)
 322                    })
 323                })?;
 324                Some((project, cx))
 325            });
 326
 327            let (host_project, host_cx) = if let Some((host_project, host_cx)) = host_project {
 328                (host_project, host_cx)
 329            } else {
 330                continue;
 331            };
 332
 333            for guest_buffer in guest_buffers {
 334                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 335                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 336                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 337                        panic!(
 338                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 339                            client.username,
 340                            client.peer_id(),
 341                            buffer_id
 342                        )
 343                    })
 344                });
 345                let path = host_buffer
 346                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 347
 348                assert_eq!(
 349                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 350                    0,
 351                    "{}, buffer {}, path {:?} has deferred operations",
 352                    client.username,
 353                    buffer_id,
 354                    path,
 355                );
 356                assert_eq!(
 357                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 358                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 359                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 360                    client.username,
 361                    buffer_id,
 362                    path
 363                );
 364
 365                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 366                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 367                match (host_file, guest_file) {
 368                    (Some(host_file), Some(guest_file)) => {
 369                        assert_eq!(host_file.mtime(), guest_file.mtime());
 370                        assert_eq!(host_file.path(), guest_file.path());
 371                        assert_eq!(host_file.is_deleted(), guest_file.is_deleted());
 372                    }
 373                    (None, None) => {}
 374                    (None, _) => panic!("host's file is None, guest's isn't "),
 375                    (_, None) => panic!("guest's file is None, hosts's isn't "),
 376                }
 377            }
 378        }
 379    }
 380
 381    for (client, mut cx) in clients {
 382        cx.update(|cx| {
 383            cx.clear_globals();
 384            drop(client);
 385        });
 386    }
 387}
 388
 389async fn simulate_client(
 390    mut client: TestClient,
 391    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 392    can_hang_up: bool,
 393    rng: Arc<Mutex<StdRng>>,
 394    mut cx: TestAppContext,
 395) -> (TestClient, TestAppContext) {
 396    // Setup language server
 397    let mut language = Language::new(
 398        LanguageConfig {
 399            name: "Rust".into(),
 400            path_suffixes: vec!["rs".to_string()],
 401            ..Default::default()
 402        },
 403        None,
 404    );
 405    let _fake_language_servers = language
 406        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 407            name: "the-fake-language-server",
 408            capabilities: lsp::LanguageServer::full_capabilities(),
 409            initializer: Some(Box::new({
 410                let rng = rng.clone();
 411                let fs = client.fs.clone();
 412                move |fake_server: &mut FakeLanguageServer| {
 413                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 414                        |_, _| async move {
 415                            Ok(Some(lsp::CompletionResponse::Array(vec![
 416                                lsp::CompletionItem {
 417                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 418                                        range: lsp::Range::new(
 419                                            lsp::Position::new(0, 0),
 420                                            lsp::Position::new(0, 0),
 421                                        ),
 422                                        new_text: "the-new-text".to_string(),
 423                                    })),
 424                                    ..Default::default()
 425                                },
 426                            ])))
 427                        },
 428                    );
 429
 430                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 431                        |_, _| async move {
 432                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 433                                lsp::CodeAction {
 434                                    title: "the-code-action".to_string(),
 435                                    ..Default::default()
 436                                },
 437                            )]))
 438                        },
 439                    );
 440
 441                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 442                        |params, _| async move {
 443                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 444                                params.position,
 445                                params.position,
 446                            ))))
 447                        },
 448                    );
 449
 450                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 451                        let fs = fs.clone();
 452                        let rng = rng.clone();
 453                        move |_, _| {
 454                            let fs = fs.clone();
 455                            let rng = rng.clone();
 456                            async move {
 457                                let files = fs.files().await;
 458                                let mut rng = rng.lock();
 459                                let count = rng.gen_range::<usize, _>(1..3);
 460                                let files = (0..count)
 461                                    .map(|_| files.choose(&mut *rng).unwrap())
 462                                    .collect::<Vec<_>>();
 463                                log::info!("LSP: Returning definitions in files {:?}", &files);
 464                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 465                                    files
 466                                        .into_iter()
 467                                        .map(|file| lsp::Location {
 468                                            uri: lsp::Url::from_file_path(file).unwrap(),
 469                                            range: Default::default(),
 470                                        })
 471                                        .collect(),
 472                                )))
 473                            }
 474                        }
 475                    });
 476
 477                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 478                        let rng = rng.clone();
 479                        move |_, _| {
 480                            let mut highlights = Vec::new();
 481                            let highlight_count = rng.lock().gen_range(1..=5);
 482                            for _ in 0..highlight_count {
 483                                let start_row = rng.lock().gen_range(0..100);
 484                                let start_column = rng.lock().gen_range(0..100);
 485                                let start = PointUtf16::new(start_row, start_column);
 486                                let end_row = rng.lock().gen_range(0..100);
 487                                let end_column = rng.lock().gen_range(0..100);
 488                                let end = PointUtf16::new(end_row, end_column);
 489                                let range = if start > end { end..start } else { start..end };
 490                                highlights.push(lsp::DocumentHighlight {
 491                                    range: range_to_lsp(range.clone()),
 492                                    kind: Some(lsp::DocumentHighlightKind::READ),
 493                                });
 494                            }
 495                            highlights.sort_unstable_by_key(|highlight| {
 496                                (highlight.range.start, highlight.range.end)
 497                            });
 498                            async move { Ok(Some(highlights)) }
 499                        }
 500                    });
 501                }
 502            })),
 503            ..Default::default()
 504        }))
 505        .await;
 506    client.language_registry.add(Arc::new(language));
 507
 508    while op_start_signal.next().await.is_some() {
 509        if let Err(error) =
 510            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 511        {
 512            log::error!("{} error: {:?}", client.username, error);
 513        }
 514
 515        cx.background().simulate_random_delay().await;
 516    }
 517    log::info!("{}: done", client.username);
 518
 519    (client, cx)
 520}
 521
 522async fn randomly_mutate_client(
 523    client: &mut TestClient,
 524    can_hang_up: bool,
 525    rng: Arc<Mutex<StdRng>>,
 526    cx: &mut TestAppContext,
 527) -> Result<()> {
 528    let choice = rng.lock().gen_range(0..100);
 529    match choice {
 530        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 531        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 532        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 533            randomly_mutate_worktrees(client, &rng, cx).await?;
 534        }
 535        60..=84 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 536            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 537        }
 538        _ => randomly_mutate_fs(client, &rng).await,
 539    }
 540
 541    Ok(())
 542}
 543
 544async fn randomly_mutate_active_call(
 545    client: &mut TestClient,
 546    can_hang_up: bool,
 547    rng: &Mutex<StdRng>,
 548    cx: &mut TestAppContext,
 549) -> Result<()> {
 550    let active_call = cx.read(ActiveCall::global);
 551    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 552        if rng.lock().gen_bool(0.7) {
 553            log::info!("{}: accepting incoming call", client.username);
 554            active_call
 555                .update(cx, |call, cx| call.accept_incoming(cx))
 556                .await?;
 557        } else {
 558            log::info!("{}: declining incoming call", client.username);
 559            active_call.update(cx, |call, _| call.decline_incoming())?;
 560        }
 561    } else {
 562        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 563            user_store
 564                .contacts()
 565                .iter()
 566                .filter(|contact| contact.online && !contact.busy)
 567                .cloned()
 568                .collect::<Vec<_>>()
 569        });
 570
 571        let distribution = rng.lock().gen_range(0..100);
 572        match distribution {
 573            0..=29 if !available_contacts.is_empty() => {
 574                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 575                log::info!(
 576                    "{}: inviting {}",
 577                    client.username,
 578                    contact.user.github_login
 579                );
 580                active_call
 581                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 582                    .await?;
 583            }
 584            30..=39
 585                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 586            {
 587                log::info!("{}: hanging up", client.username);
 588                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 589            }
 590            _ => {}
 591        }
 592    }
 593
 594    Ok(())
 595}
 596
 597async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 598    let is_dir = rng.lock().gen::<bool>();
 599    let mut new_path = client
 600        .fs
 601        .directories()
 602        .await
 603        .choose(&mut *rng.lock())
 604        .unwrap()
 605        .clone();
 606    new_path.push(gen_file_name(rng));
 607    if is_dir {
 608        log::info!("{}: creating local dir at {:?}", client.username, new_path);
 609        client.fs.create_dir(&new_path).await.unwrap();
 610    } else {
 611        new_path.set_extension("rs");
 612        log::info!("{}: creating local file at {:?}", client.username, new_path);
 613        client
 614            .fs
 615            .create_file(&new_path, Default::default())
 616            .await
 617            .unwrap();
 618    }
 619}
 620
 621async fn randomly_mutate_projects(
 622    client: &mut TestClient,
 623    rng: &Mutex<StdRng>,
 624    cx: &mut TestAppContext,
 625) -> Result<()> {
 626    let active_call = cx.read(ActiveCall::global);
 627    let remote_projects =
 628        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 629            room.read_with(cx, |room, _| {
 630                room.remote_participants()
 631                    .values()
 632                    .flat_map(|participant| participant.projects.clone())
 633                    .collect::<Vec<_>>()
 634            })
 635        } else {
 636            Default::default()
 637        };
 638
 639    let project = if remote_projects.is_empty() || rng.lock().gen() {
 640        if client.local_projects.is_empty() || rng.lock().gen() {
 641            let paths = client.fs.paths().await;
 642            let local_project = if paths.is_empty() || rng.lock().gen() {
 643                let root_path = client.create_new_root_dir();
 644                client.fs.create_dir(&root_path).await.unwrap();
 645                client
 646                    .fs
 647                    .create_file(&root_path.join("main.rs"), Default::default())
 648                    .await
 649                    .unwrap();
 650                log::info!(
 651                    "{}: opening local project at {:?}",
 652                    client.username,
 653                    root_path
 654                );
 655                client.build_local_project(root_path, cx).await.0
 656            } else {
 657                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 658                log::info!(
 659                    "{}: opening local project at {:?}",
 660                    client.username,
 661                    root_path
 662                );
 663                client.build_local_project(root_path, cx).await.0
 664            };
 665            client.local_projects.push(local_project.clone());
 666            local_project
 667        } else {
 668            client
 669                .local_projects
 670                .choose(&mut *rng.lock())
 671                .unwrap()
 672                .clone()
 673        }
 674    } else {
 675        if client.remote_projects.is_empty() || rng.lock().gen() {
 676            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 677            let remote_project = if let Some(project) =
 678                client.remote_projects.iter().find(|project| {
 679                    project.read_with(cx, |project, _| {
 680                        project.remote_id() == Some(remote_project_id)
 681                    })
 682                }) {
 683                project.clone()
 684            } else {
 685                log::info!(
 686                    "{}: opening remote project {}",
 687                    client.username,
 688                    remote_project_id
 689                );
 690                let call = cx.read(ActiveCall::global);
 691                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 692                let remote_project = room
 693                    .update(cx, |room, cx| {
 694                        room.join_project(
 695                            remote_project_id,
 696                            client.language_registry.clone(),
 697                            FakeFs::new(cx.background().clone()),
 698                            cx,
 699                        )
 700                    })
 701                    .await?;
 702                client.remote_projects.push(remote_project.clone());
 703                remote_project
 704            };
 705
 706            remote_project
 707        } else {
 708            client
 709                .remote_projects
 710                .choose(&mut *rng.lock())
 711                .unwrap()
 712                .clone()
 713        }
 714    };
 715
 716    if active_call.read_with(cx, |call, _| call.room().is_some())
 717        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 718    {
 719        match active_call
 720            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 721            .await
 722        {
 723            Ok(project_id) => {
 724                log::info!("{}: shared project with id {}", client.username, project_id);
 725            }
 726            Err(error) => {
 727                log::error!("{}: error sharing project, {:?}", client.username, error);
 728            }
 729        }
 730    }
 731
 732    let choice = rng.lock().gen_range(0..100);
 733    match choice {
 734        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 735            let paths = client.fs.paths().await;
 736            let path = paths.choose(&mut *rng.lock()).unwrap();
 737            log::info!(
 738                "{}: finding/creating local worktree for path {:?}",
 739                client.username,
 740                path
 741            );
 742            project
 743                .update(cx, |project, cx| {
 744                    project.find_or_create_local_worktree(&path, true, cx)
 745                })
 746                .await
 747                .unwrap();
 748        }
 749        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 750            log::info!(
 751                "{}: dropping remote project {}",
 752                client.username,
 753                project.read_with(cx, |project, _| project.remote_id().unwrap())
 754            );
 755
 756            cx.update(|_| {
 757                client
 758                    .remote_projects
 759                    .retain(|remote_project| *remote_project != project);
 760                client.buffers.remove(&project);
 761                drop(project);
 762            });
 763        }
 764        _ => {}
 765    }
 766
 767    Ok(())
 768}
 769
 770async fn randomly_mutate_worktrees(
 771    client: &mut TestClient,
 772    rng: &Mutex<StdRng>,
 773    cx: &mut TestAppContext,
 774) -> Result<()> {
 775    let project = choose_random_project(client, rng).unwrap();
 776    let Some(worktree) = project.read_with(cx, |project, cx| {
 777        project
 778            .worktrees(cx)
 779            .filter(|worktree| {
 780                let worktree = worktree.read(cx);
 781                worktree.is_visible()
 782                    && worktree.entries(false).any(|e| e.is_file())
 783                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 784            })
 785            .choose(&mut *rng.lock())
 786    }) else {
 787        return Ok(())
 788    };
 789
 790    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 791        (worktree.id(), worktree.root_name().to_string())
 792    });
 793
 794    let is_dir = rng.lock().gen::<bool>();
 795    let mut new_path = PathBuf::new();
 796    new_path.push(gen_file_name(rng));
 797    if !is_dir {
 798        new_path.set_extension("rs");
 799    }
 800    log::info!(
 801        "{}: creating {:?} in worktree {} ({})",
 802        client.username,
 803        new_path,
 804        worktree_id,
 805        worktree_root_name,
 806    );
 807    project
 808        .update(cx, |project, cx| {
 809            project.create_entry((worktree_id, new_path), is_dir, cx)
 810        })
 811        .unwrap()
 812        .await?;
 813    Ok(())
 814}
 815
 816async fn randomly_query_and_mutate_buffers(
 817    client: &mut TestClient,
 818    rng: &Mutex<StdRng>,
 819    cx: &mut TestAppContext,
 820) -> Result<()> {
 821    let project = choose_random_project(client, rng).unwrap();
 822    let buffers = client.buffers.entry(project.clone()).or_default();
 823    let buffer = if buffers.is_empty() || rng.lock().gen() {
 824        let Some(worktree) = project.read_with(cx, |project, cx| {
 825            project
 826                .worktrees(cx)
 827                .filter(|worktree| {
 828                    let worktree = worktree.read(cx);
 829                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 830                })
 831                .choose(&mut *rng.lock())
 832        }) else {
 833            return Ok(());
 834        };
 835
 836        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 837            let entry = worktree
 838                .entries(false)
 839                .filter(|e| e.is_file())
 840                .choose(&mut *rng.lock())
 841                .unwrap();
 842            (
 843                worktree.root_name().to_string(),
 844                (worktree.id(), entry.path.clone()),
 845            )
 846        });
 847        log::info!(
 848            "{}: opening path {:?} in worktree {} ({})",
 849            client.username,
 850            project_path.1,
 851            project_path.0,
 852            worktree_root_name,
 853        );
 854        let buffer = project
 855            .update(cx, |project, cx| {
 856                project.open_buffer(project_path.clone(), cx)
 857            })
 858            .await?;
 859        log::info!(
 860            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 861            client.username,
 862            project_path.1,
 863            project_path.0,
 864            worktree_root_name,
 865            buffer.read_with(cx, |buffer, _| buffer.remote_id())
 866        );
 867        buffers.insert(buffer.clone());
 868        buffer
 869    } else {
 870        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
 871    };
 872
 873    let choice = rng.lock().gen_range(0..100);
 874    match choice {
 875        0..=9 => {
 876            cx.update(|cx| {
 877                log::info!(
 878                    "{}: dropping buffer {:?}",
 879                    client.username,
 880                    buffer.read(cx).file().unwrap().full_path(cx)
 881                );
 882                buffers.remove(&buffer);
 883                drop(buffer);
 884            });
 885        }
 886        10..=19 => {
 887            let completions = project.update(cx, |project, cx| {
 888                log::info!(
 889                    "{}: requesting completions for buffer {} ({:?})",
 890                    client.username,
 891                    buffer.read(cx).remote_id(),
 892                    buffer.read(cx).file().unwrap().full_path(cx)
 893                );
 894                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 895                project.completions(&buffer, offset, cx)
 896            });
 897            let completions = cx.background().spawn(async move {
 898                completions
 899                    .await
 900                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
 901            });
 902            if rng.lock().gen_bool(0.3) {
 903                log::info!("{}: detaching completions request", client.username);
 904                cx.update(|cx| completions.detach_and_log_err(cx));
 905            } else {
 906                completions.await?;
 907            }
 908        }
 909        20..=29 => {
 910            let code_actions = project.update(cx, |project, cx| {
 911                log::info!(
 912                    "{}: requesting code actions for buffer {} ({:?})",
 913                    client.username,
 914                    buffer.read(cx).remote_id(),
 915                    buffer.read(cx).file().unwrap().full_path(cx)
 916                );
 917                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
 918                project.code_actions(&buffer, range, cx)
 919            });
 920            let code_actions = cx.background().spawn(async move {
 921                code_actions
 922                    .await
 923                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
 924            });
 925            if rng.lock().gen_bool(0.3) {
 926                log::info!("{}: detaching code actions request", client.username);
 927                cx.update(|cx| code_actions.detach_and_log_err(cx));
 928            } else {
 929                code_actions.await?;
 930            }
 931        }
 932        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
 933            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
 934                log::info!(
 935                    "{}: saving buffer {} ({:?})",
 936                    client.username,
 937                    buffer.remote_id(),
 938                    buffer.file().unwrap().full_path(cx)
 939                );
 940                (buffer.version(), buffer.save(cx))
 941            });
 942            let save = cx.background().spawn(async move {
 943                let (saved_version, _, _) = save
 944                    .await
 945                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 946                assert!(saved_version.observed_all(&requested_version));
 947                Ok::<_, anyhow::Error>(())
 948            });
 949            if rng.lock().gen_bool(0.3) {
 950                log::info!("{}: detaching save request", client.username);
 951                cx.update(|cx| save.detach_and_log_err(cx));
 952            } else {
 953                save.await?;
 954            }
 955        }
 956        40..=44 => {
 957            let prepare_rename = project.update(cx, |project, cx| {
 958                log::info!(
 959                    "{}: preparing rename for buffer {} ({:?})",
 960                    client.username,
 961                    buffer.read(cx).remote_id(),
 962                    buffer.read(cx).file().unwrap().full_path(cx)
 963                );
 964                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 965                project.prepare_rename(buffer, offset, cx)
 966            });
 967            let prepare_rename = cx.background().spawn(async move {
 968                prepare_rename
 969                    .await
 970                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
 971            });
 972            if rng.lock().gen_bool(0.3) {
 973                log::info!("{}: detaching prepare rename request", client.username);
 974                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
 975            } else {
 976                prepare_rename.await?;
 977            }
 978        }
 979        45..=49 => {
 980            let definitions = project.update(cx, |project, cx| {
 981                log::info!(
 982                    "{}: requesting definitions for buffer {} ({:?})",
 983                    client.username,
 984                    buffer.read(cx).remote_id(),
 985                    buffer.read(cx).file().unwrap().full_path(cx)
 986                );
 987                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 988                project.definition(&buffer, offset, cx)
 989            });
 990            let definitions = cx.background().spawn(async move {
 991                definitions
 992                    .await
 993                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
 994            });
 995            if rng.lock().gen_bool(0.3) {
 996                log::info!("{}: detaching definitions request", client.username);
 997                cx.update(|cx| definitions.detach_and_log_err(cx));
 998            } else {
 999                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1000            }
1001        }
1002        50..=54 => {
1003            let highlights = project.update(cx, |project, cx| {
1004                log::info!(
1005                    "{}: requesting highlights for buffer {} ({:?})",
1006                    client.username,
1007                    buffer.read(cx).remote_id(),
1008                    buffer.read(cx).file().unwrap().full_path(cx)
1009                );
1010                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1011                project.document_highlights(&buffer, offset, cx)
1012            });
1013            let highlights = cx.background().spawn(async move {
1014                highlights
1015                    .await
1016                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1017            });
1018            if rng.lock().gen_bool(0.3) {
1019                log::info!("{}: detaching highlights request", client.username);
1020                cx.update(|cx| highlights.detach_and_log_err(cx));
1021            } else {
1022                highlights.await?;
1023            }
1024        }
1025        55..=59 => {
1026            let search = project.update(cx, |project, cx| {
1027                let query = rng.lock().gen_range('a'..='z');
1028                log::info!("{}: project-wide search {:?}", client.username, query);
1029                project.search(SearchQuery::text(query, false, false), cx)
1030            });
1031            let search = cx.background().spawn(async move {
1032                search
1033                    .await
1034                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1035            });
1036            if rng.lock().gen_bool(0.3) {
1037                log::info!("{}: detaching search request", client.username);
1038                cx.update(|cx| search.detach_and_log_err(cx));
1039            } else {
1040                buffers.extend(search.await?.into_keys());
1041            }
1042        }
1043        _ => {
1044            buffer.update(cx, |buffer, cx| {
1045                log::info!(
1046                    "{}: updating buffer {} ({:?})",
1047                    client.username,
1048                    buffer.remote_id(),
1049                    buffer.file().unwrap().full_path(cx)
1050                );
1051                if rng.lock().gen_bool(0.7) {
1052                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1053                } else {
1054                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1055                }
1056            });
1057        }
1058    }
1059
1060    Ok(())
1061}
1062
1063fn choose_random_project(
1064    client: &mut TestClient,
1065    rng: &Mutex<StdRng>,
1066) -> Option<ModelHandle<Project>> {
1067    client
1068        .local_projects
1069        .iter()
1070        .chain(&client.remote_projects)
1071        .choose(&mut *rng.lock())
1072        .cloned()
1073}
1074
1075fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1076    let mut name = String::new();
1077    for _ in 0..10 {
1078        let letter = rng.lock().gen_range('a'..='z');
1079        name.push(letter);
1080    }
1081    name
1082}