randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::prelude::*;
  18use std::{env, path::PathBuf, sync::Arc};
  19
  20#[gpui::test(iterations = 100)]
  21async fn test_random_collaboration(
  22    cx: &mut TestAppContext,
  23    deterministic: Arc<Deterministic>,
  24    rng: StdRng,
  25) {
  26    deterministic.forbid_parking();
  27    let rng = Arc::new(Mutex::new(rng));
  28
  29    let max_peers = env::var("MAX_PEERS")
  30        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  31        .unwrap_or(5);
  32
  33    let max_operations = env::var("OPERATIONS")
  34        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  35        .unwrap_or(10);
  36
  37    let mut server = TestServer::start(&deterministic).await;
  38    let db = server.app_state.db.clone();
  39
  40    let mut available_users = Vec::new();
  41    for ix in 0..max_peers {
  42        let username = format!("user-{}", ix + 1);
  43        let user_id = db
  44            .create_user(
  45                &format!("{username}@example.com"),
  46                false,
  47                NewUserParams {
  48                    github_login: username.clone(),
  49                    github_user_id: (ix + 1) as i32,
  50                    invite_count: 0,
  51                },
  52            )
  53            .await
  54            .unwrap()
  55            .user_id;
  56        available_users.push((user_id, username));
  57    }
  58
  59    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  60        for (user_id_b, _) in &available_users[ix + 1..] {
  61            server
  62                .app_state
  63                .db
  64                .send_contact_request(*user_id_a, *user_id_b)
  65                .await
  66                .unwrap();
  67            server
  68                .app_state
  69                .db
  70                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  71                .await
  72                .unwrap();
  73        }
  74    }
  75
  76    let mut clients = Vec::new();
  77    let mut user_ids = Vec::new();
  78    let mut op_start_signals = Vec::new();
  79    let mut next_entity_id = 100000;
  80    let allow_server_restarts = rng.lock().gen_bool(0.7);
  81    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  82    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  83
  84    let mut operations = 0;
  85    while operations < max_operations {
  86        let distribution = rng.lock().gen_range(0..100);
  87        match distribution {
  88            0..=19 if !available_users.is_empty() => {
  89                let client_ix = rng.lock().gen_range(0..available_users.len());
  90                let (_, username) = available_users.remove(client_ix);
  91                log::info!("Adding new connection for {}", username);
  92                next_entity_id += 100000;
  93                let mut client_cx = TestAppContext::new(
  94                    cx.foreground_platform(),
  95                    cx.platform(),
  96                    deterministic.build_foreground(next_entity_id),
  97                    deterministic.build_background(),
  98                    cx.font_cache(),
  99                    cx.leak_detector(),
 100                    next_entity_id,
 101                    cx.function_name.clone(),
 102                );
 103
 104                let op_start_signal = futures::channel::mpsc::unbounded();
 105                let client = server.create_client(&mut client_cx, &username).await;
 106                user_ids.push(client.current_user_id(&client_cx));
 107                op_start_signals.push(op_start_signal.0);
 108                clients.push(client_cx.foreground().spawn(simulate_client(
 109                    client,
 110                    op_start_signal.1,
 111                    allow_client_disconnection,
 112                    rng.clone(),
 113                    client_cx,
 114                )));
 115
 116                log::info!("Added connection for {}", username);
 117                operations += 1;
 118            }
 119
 120            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 121                let client_ix = rng.lock().gen_range(1..clients.len());
 122                log::info!(
 123                    "Simulating full disconnection of user {}",
 124                    user_ids[client_ix]
 125                );
 126                let removed_user_id = user_ids.remove(client_ix);
 127                let user_connection_ids = server
 128                    .connection_pool
 129                    .lock()
 130                    .user_connection_ids(removed_user_id)
 131                    .collect::<Vec<_>>();
 132                assert_eq!(user_connection_ids.len(), 1);
 133                let removed_peer_id = user_connection_ids[0].into();
 134                let client = clients.remove(client_ix);
 135                op_start_signals.remove(client_ix);
 136                server.forbid_connections();
 137                server.disconnect_client(removed_peer_id);
 138                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 139                deterministic.start_waiting();
 140                log::info!("Waiting for user {} to exit...", removed_user_id);
 141                let (client, mut client_cx) = client.await;
 142                deterministic.finish_waiting();
 143                server.allow_connections();
 144
 145                for project in &client.remote_projects {
 146                    project.read_with(&client_cx, |project, _| {
 147                        assert!(
 148                            project.is_read_only(),
 149                            "project {:?} should be read only",
 150                            project.remote_id()
 151                        )
 152                    });
 153                }
 154                for user_id in &user_ids {
 155                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 156                    let pool = server.connection_pool.lock();
 157                    for contact in contacts {
 158                        if let db::Contact::Accepted { user_id, .. } = contact {
 159                            if pool.is_user_online(user_id) {
 160                                assert_ne!(
 161                                    user_id, removed_user_id,
 162                                    "removed client is still a contact of another peer"
 163                                );
 164                            }
 165                        }
 166                    }
 167                }
 168
 169                log::info!("{} removed", client.username);
 170                available_users.push((removed_user_id, client.username.clone()));
 171                client_cx.update(|cx| {
 172                    cx.clear_globals();
 173                    drop(client);
 174                });
 175
 176                operations += 1;
 177            }
 178
 179            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 180                let client_ix = rng.lock().gen_range(1..clients.len());
 181                let user_id = user_ids[client_ix];
 182                log::info!("Simulating temporary disconnection of user {}", user_id);
 183                let user_connection_ids = server
 184                    .connection_pool
 185                    .lock()
 186                    .user_connection_ids(user_id)
 187                    .collect::<Vec<_>>();
 188                assert_eq!(user_connection_ids.len(), 1);
 189                let peer_id = user_connection_ids[0].into();
 190                server.disconnect_client(peer_id);
 191                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 192                operations += 1;
 193            }
 194
 195            30..=34 if allow_server_restarts => {
 196                log::info!("Simulating server restart");
 197                server.reset().await;
 198                deterministic.advance_clock(RECEIVE_TIMEOUT);
 199                server.start().await.unwrap();
 200                deterministic.advance_clock(CLEANUP_TIMEOUT);
 201                let environment = &server.app_state.config.zed_environment;
 202                let stale_room_ids = server
 203                    .app_state
 204                    .db
 205                    .stale_room_ids(environment, server.id())
 206                    .await
 207                    .unwrap();
 208                assert_eq!(stale_room_ids, vec![]);
 209            }
 210
 211            _ if !op_start_signals.is_empty() => {
 212                while operations < max_operations && rng.lock().gen_bool(0.7) {
 213                    op_start_signals
 214                        .choose(&mut *rng.lock())
 215                        .unwrap()
 216                        .unbounded_send(())
 217                        .unwrap();
 218                    operations += 1;
 219                }
 220
 221                if rng.lock().gen_bool(0.8) {
 222                    deterministic.run_until_parked();
 223                }
 224            }
 225            _ => {}
 226        }
 227    }
 228
 229    drop(op_start_signals);
 230    deterministic.start_waiting();
 231    let clients = futures::future::join_all(clients).await;
 232    deterministic.finish_waiting();
 233    deterministic.run_until_parked();
 234
 235    for (client, client_cx) in &clients {
 236        for guest_project in &client.remote_projects {
 237            guest_project.read_with(client_cx, |guest_project, cx| {
 238                let host_project = clients.iter().find_map(|(client, cx)| {
 239                    let project = client.local_projects.iter().find(|host_project| {
 240                        host_project.read_with(cx, |host_project, _| {
 241                            host_project.remote_id() == guest_project.remote_id()
 242                        })
 243                    })?;
 244                    Some((project, cx))
 245                });
 246
 247                if !guest_project.is_read_only() {
 248                    if let Some((host_project, host_cx)) = host_project {
 249                        let host_worktree_snapshots =
 250                            host_project.read_with(host_cx, |host_project, cx| {
 251                                host_project
 252                                    .worktrees(cx)
 253                                    .map(|worktree| {
 254                                        let worktree = worktree.read(cx);
 255                                        (worktree.id(), worktree.snapshot())
 256                                    })
 257                                    .collect::<BTreeMap<_, _>>()
 258                            });
 259                        let guest_worktree_snapshots = guest_project
 260                            .worktrees(cx)
 261                            .map(|worktree| {
 262                                let worktree = worktree.read(cx);
 263                                (worktree.id(), worktree.snapshot())
 264                            })
 265                            .collect::<BTreeMap<_, _>>();
 266
 267                        assert_eq!(
 268                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 269                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 270                            "{} has different worktrees than the host",
 271                            client.username
 272                        );
 273
 274                        for (id, host_snapshot) in &host_worktree_snapshots {
 275                            let guest_snapshot = &guest_worktree_snapshots[id];
 276                            assert_eq!(
 277                                guest_snapshot.root_name(),
 278                                host_snapshot.root_name(),
 279                                "{} has different root name than the host for worktree {}",
 280                                client.username,
 281                                id
 282                            );
 283                            assert_eq!(
 284                                guest_snapshot.abs_path(),
 285                                host_snapshot.abs_path(),
 286                                "{} has different abs path than the host for worktree {}",
 287                                client.username,
 288                                id
 289                            );
 290                            assert_eq!(
 291                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 292                                host_snapshot.entries(false).collect::<Vec<_>>(),
 293                                "{} has different snapshot than the host for worktree {}",
 294                                client.username,
 295                                id
 296                            );
 297                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 298                        }
 299                    }
 300                }
 301
 302                guest_project.check_invariants(cx);
 303            });
 304        }
 305
 306        for (guest_project, guest_buffers) in &client.buffers {
 307            let project_id = if guest_project.read_with(client_cx, |project, _| {
 308                project.is_local() || project.is_read_only()
 309            }) {
 310                continue;
 311            } else {
 312                guest_project
 313                    .read_with(client_cx, |project, _| project.remote_id())
 314                    .unwrap()
 315            };
 316
 317            let host_project = clients.iter().find_map(|(client, cx)| {
 318                let project = client.local_projects.iter().find(|host_project| {
 319                    host_project.read_with(cx, |host_project, _| {
 320                        host_project.remote_id() == Some(project_id)
 321                    })
 322                })?;
 323                Some((project, cx))
 324            });
 325
 326            let (host_project, host_cx) = if let Some((host_project, host_cx)) = host_project {
 327                (host_project, host_cx)
 328            } else {
 329                continue;
 330            };
 331
 332            for guest_buffer in guest_buffers {
 333                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 334                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 335                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 336                        panic!(
 337                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 338                            client.username,
 339                            client.peer_id(),
 340                            buffer_id
 341                        )
 342                    })
 343                });
 344                let path = host_buffer
 345                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 346
 347                assert_eq!(
 348                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 349                    0,
 350                    "{}, buffer {}, path {:?} has deferred operations",
 351                    client.username,
 352                    buffer_id,
 353                    path,
 354                );
 355                assert_eq!(
 356                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 357                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 358                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 359                    client.username,
 360                    buffer_id,
 361                    path
 362                );
 363
 364                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 365                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 366                match (host_file, guest_file) {
 367                    (Some(host_file), Some(guest_file)) => {
 368                        assert_eq!(host_file.mtime(), guest_file.mtime());
 369                        assert_eq!(host_file.path(), guest_file.path());
 370                        assert_eq!(host_file.is_deleted(), guest_file.is_deleted());
 371                    }
 372                    (None, None) => {}
 373                    (None, _) => panic!("host's file is None, guest's isn't "),
 374                    (_, None) => panic!("guest's file is None, hosts's isn't "),
 375                }
 376            }
 377        }
 378    }
 379
 380    for (client, mut cx) in clients {
 381        cx.update(|cx| {
 382            cx.clear_globals();
 383            drop(client);
 384        });
 385    }
 386}
 387
 388async fn simulate_client(
 389    mut client: TestClient,
 390    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 391    can_hang_up: bool,
 392    rng: Arc<Mutex<StdRng>>,
 393    mut cx: TestAppContext,
 394) -> (TestClient, TestAppContext) {
 395    // Setup language server
 396    let mut language = Language::new(
 397        LanguageConfig {
 398            name: "Rust".into(),
 399            path_suffixes: vec!["rs".to_string()],
 400            ..Default::default()
 401        },
 402        None,
 403    );
 404    let _fake_language_servers = language
 405        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 406            name: "the-fake-language-server",
 407            capabilities: lsp::LanguageServer::full_capabilities(),
 408            initializer: Some(Box::new({
 409                let rng = rng.clone();
 410                let fs = client.fs.clone();
 411                move |fake_server: &mut FakeLanguageServer| {
 412                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 413                        |_, _| async move {
 414                            Ok(Some(lsp::CompletionResponse::Array(vec![
 415                                lsp::CompletionItem {
 416                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 417                                        range: lsp::Range::new(
 418                                            lsp::Position::new(0, 0),
 419                                            lsp::Position::new(0, 0),
 420                                        ),
 421                                        new_text: "the-new-text".to_string(),
 422                                    })),
 423                                    ..Default::default()
 424                                },
 425                            ])))
 426                        },
 427                    );
 428
 429                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 430                        |_, _| async move {
 431                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 432                                lsp::CodeAction {
 433                                    title: "the-code-action".to_string(),
 434                                    ..Default::default()
 435                                },
 436                            )]))
 437                        },
 438                    );
 439
 440                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 441                        |params, _| async move {
 442                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 443                                params.position,
 444                                params.position,
 445                            ))))
 446                        },
 447                    );
 448
 449                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 450                        let fs = fs.clone();
 451                        let rng = rng.clone();
 452                        move |_, _| {
 453                            let fs = fs.clone();
 454                            let rng = rng.clone();
 455                            async move {
 456                                let files = fs.files().await;
 457                                let mut rng = rng.lock();
 458                                let count = rng.gen_range::<usize, _>(1..3);
 459                                let files = (0..count)
 460                                    .map(|_| files.choose(&mut *rng).unwrap())
 461                                    .collect::<Vec<_>>();
 462                                log::info!("LSP: Returning definitions in files {:?}", &files);
 463                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 464                                    files
 465                                        .into_iter()
 466                                        .map(|file| lsp::Location {
 467                                            uri: lsp::Url::from_file_path(file).unwrap(),
 468                                            range: Default::default(),
 469                                        })
 470                                        .collect(),
 471                                )))
 472                            }
 473                        }
 474                    });
 475
 476                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 477                        let rng = rng.clone();
 478                        move |_, _| {
 479                            let mut highlights = Vec::new();
 480                            let highlight_count = rng.lock().gen_range(1..=5);
 481                            for _ in 0..highlight_count {
 482                                let start_row = rng.lock().gen_range(0..100);
 483                                let start_column = rng.lock().gen_range(0..100);
 484                                let start = PointUtf16::new(start_row, start_column);
 485                                let end_row = rng.lock().gen_range(0..100);
 486                                let end_column = rng.lock().gen_range(0..100);
 487                                let end = PointUtf16::new(end_row, end_column);
 488                                let range = if start > end { end..start } else { start..end };
 489                                highlights.push(lsp::DocumentHighlight {
 490                                    range: range_to_lsp(range.clone()),
 491                                    kind: Some(lsp::DocumentHighlightKind::READ),
 492                                });
 493                            }
 494                            highlights.sort_unstable_by_key(|highlight| {
 495                                (highlight.range.start, highlight.range.end)
 496                            });
 497                            async move { Ok(Some(highlights)) }
 498                        }
 499                    });
 500                }
 501            })),
 502            ..Default::default()
 503        }))
 504        .await;
 505    client.language_registry.add(Arc::new(language));
 506
 507    while op_start_signal.next().await.is_some() {
 508        if let Err(error) =
 509            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 510        {
 511            log::error!("{} error: {:?}", client.username, error);
 512        }
 513
 514        cx.background().simulate_random_delay().await;
 515    }
 516    log::info!("{}: done", client.username);
 517
 518    (client, cx)
 519}
 520
 521async fn randomly_mutate_client(
 522    client: &mut TestClient,
 523    can_hang_up: bool,
 524    rng: Arc<Mutex<StdRng>>,
 525    cx: &mut TestAppContext,
 526) -> Result<()> {
 527    let choice = rng.lock().gen_range(0..100);
 528    match choice {
 529        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 530        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 531        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 532            randomly_mutate_worktrees(client, &rng, cx).await?;
 533        }
 534        60..=84 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 535            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 536        }
 537        _ => randomly_mutate_fs(client, &rng).await,
 538    }
 539
 540    Ok(())
 541}
 542
 543async fn randomly_mutate_active_call(
 544    client: &mut TestClient,
 545    can_hang_up: bool,
 546    rng: &Mutex<StdRng>,
 547    cx: &mut TestAppContext,
 548) -> Result<()> {
 549    let active_call = cx.read(ActiveCall::global);
 550    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 551        if rng.lock().gen_bool(0.7) {
 552            log::info!("{}: accepting incoming call", client.username);
 553            active_call
 554                .update(cx, |call, cx| call.accept_incoming(cx))
 555                .await?;
 556        } else {
 557            log::info!("{}: declining incoming call", client.username);
 558            active_call.update(cx, |call, _| call.decline_incoming())?;
 559        }
 560    } else {
 561        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 562            user_store
 563                .contacts()
 564                .iter()
 565                .filter(|contact| contact.online && !contact.busy)
 566                .cloned()
 567                .collect::<Vec<_>>()
 568        });
 569
 570        let distribution = rng.lock().gen_range(0..100);
 571        match distribution {
 572            0..=29 if !available_contacts.is_empty() => {
 573                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 574                log::info!(
 575                    "{}: inviting {}",
 576                    client.username,
 577                    contact.user.github_login
 578                );
 579                active_call
 580                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 581                    .await?;
 582            }
 583            30..=39
 584                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 585            {
 586                log::info!("{}: hanging up", client.username);
 587                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 588            }
 589            _ => {}
 590        }
 591    }
 592
 593    Ok(())
 594}
 595
 596async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 597    let is_dir = rng.lock().gen::<bool>();
 598    let mut new_path = client
 599        .fs
 600        .directories()
 601        .await
 602        .choose(&mut *rng.lock())
 603        .unwrap()
 604        .clone();
 605    new_path.push(gen_file_name(rng));
 606    if is_dir {
 607        log::info!("{}: creating local dir at {:?}", client.username, new_path);
 608        client.fs.create_dir(&new_path).await.unwrap();
 609    } else {
 610        new_path.set_extension("rs");
 611        log::info!("{}: creating local file at {:?}", client.username, new_path);
 612        client
 613            .fs
 614            .create_file(&new_path, Default::default())
 615            .await
 616            .unwrap();
 617    }
 618}
 619
 620async fn randomly_mutate_projects(
 621    client: &mut TestClient,
 622    rng: &Mutex<StdRng>,
 623    cx: &mut TestAppContext,
 624) -> Result<()> {
 625    let active_call = cx.read(ActiveCall::global);
 626    let remote_projects =
 627        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 628            room.read_with(cx, |room, _| {
 629                room.remote_participants()
 630                    .values()
 631                    .flat_map(|participant| participant.projects.clone())
 632                    .collect::<Vec<_>>()
 633            })
 634        } else {
 635            Default::default()
 636        };
 637
 638    let project = if remote_projects.is_empty() || rng.lock().gen() {
 639        if client.local_projects.is_empty() || rng.lock().gen() {
 640            let paths = client.fs.paths().await;
 641            let local_project = if paths.is_empty() || rng.lock().gen() {
 642                let root_path = client.create_new_root_dir();
 643                client.fs.create_dir(&root_path).await.unwrap();
 644                client
 645                    .fs
 646                    .create_file(&root_path.join("main.rs"), Default::default())
 647                    .await
 648                    .unwrap();
 649                log::info!(
 650                    "{}: opening local project at {:?}",
 651                    client.username,
 652                    root_path
 653                );
 654                client.build_local_project(root_path, cx).await.0
 655            } else {
 656                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 657                log::info!(
 658                    "{}: opening local project at {:?}",
 659                    client.username,
 660                    root_path
 661                );
 662                client.build_local_project(root_path, cx).await.0
 663            };
 664            client.local_projects.push(local_project.clone());
 665            local_project
 666        } else {
 667            client
 668                .local_projects
 669                .choose(&mut *rng.lock())
 670                .unwrap()
 671                .clone()
 672        }
 673    } else {
 674        if client.remote_projects.is_empty() || rng.lock().gen() {
 675            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 676            let remote_project = if let Some(project) =
 677                client.remote_projects.iter().find(|project| {
 678                    project.read_with(cx, |project, _| {
 679                        project.remote_id() == Some(remote_project_id)
 680                    })
 681                }) {
 682                project.clone()
 683            } else {
 684                log::info!(
 685                    "{}: opening remote project {}",
 686                    client.username,
 687                    remote_project_id
 688                );
 689                let call = cx.read(ActiveCall::global);
 690                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 691                let remote_project = room
 692                    .update(cx, |room, cx| {
 693                        room.join_project(
 694                            remote_project_id,
 695                            client.language_registry.clone(),
 696                            FakeFs::new(cx.background().clone()),
 697                            cx,
 698                        )
 699                    })
 700                    .await?;
 701                client.remote_projects.push(remote_project.clone());
 702                remote_project
 703            };
 704
 705            remote_project
 706        } else {
 707            client
 708                .remote_projects
 709                .choose(&mut *rng.lock())
 710                .unwrap()
 711                .clone()
 712        }
 713    };
 714
 715    if active_call.read_with(cx, |call, _| call.room().is_some())
 716        && project.read_with(cx, |project, _| project.is_local())
 717    {
 718        if let Err(error) = active_call
 719            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 720            .await
 721        {
 722            log::error!("{}: error sharing project, {:?}", client.username, error);
 723        }
 724    }
 725
 726    let choice = rng.lock().gen_range(0..100);
 727    match choice {
 728        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 729            let paths = client.fs.paths().await;
 730            let path = paths.choose(&mut *rng.lock()).unwrap();
 731            log::info!(
 732                "{}: find or create local worktree for path {:?}",
 733                client.username,
 734                path
 735            );
 736            project
 737                .update(cx, |project, cx| {
 738                    project.find_or_create_local_worktree(&path, true, cx)
 739                })
 740                .await
 741                .unwrap();
 742        }
 743        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 744            log::info!(
 745                "{}: dropping remote project {}",
 746                client.username,
 747                project.read_with(cx, |project, _| project.remote_id().unwrap())
 748            );
 749
 750            cx.update(|_| {
 751                client
 752                    .remote_projects
 753                    .retain(|remote_project| *remote_project != project);
 754                client.buffers.remove(&project);
 755                drop(project);
 756            });
 757        }
 758        _ => {}
 759    }
 760
 761    Ok(())
 762}
 763
 764async fn randomly_mutate_worktrees(
 765    client: &mut TestClient,
 766    rng: &Mutex<StdRng>,
 767    cx: &mut TestAppContext,
 768) -> Result<()> {
 769    let project = choose_random_project(client, rng).unwrap();
 770    let Some(worktree) = project.read_with(cx, |project, cx| {
 771        project
 772            .worktrees(cx)
 773            .filter(|worktree| {
 774                let worktree = worktree.read(cx);
 775                worktree.is_visible()
 776                    && worktree.entries(false).any(|e| e.is_file())
 777                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 778            })
 779            .choose(&mut *rng.lock())
 780    }) else {
 781        return Ok(())
 782    };
 783
 784    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 785        (worktree.id(), worktree.root_name().to_string())
 786    });
 787
 788    let is_dir = rng.lock().gen::<bool>();
 789    let mut new_path = PathBuf::new();
 790    new_path.push(gen_file_name(rng));
 791    if !is_dir {
 792        new_path.set_extension("rs");
 793    }
 794    log::info!(
 795        "{}: creating {:?} in worktree {} ({})",
 796        client.username,
 797        new_path,
 798        worktree_id,
 799        worktree_root_name,
 800    );
 801    project
 802        .update(cx, |project, cx| {
 803            project.create_entry((worktree_id, new_path), is_dir, cx)
 804        })
 805        .unwrap()
 806        .await?;
 807    Ok(())
 808}
 809
 810async fn randomly_query_and_mutate_buffers(
 811    client: &mut TestClient,
 812    rng: &Mutex<StdRng>,
 813    cx: &mut TestAppContext,
 814) -> Result<()> {
 815    let project = choose_random_project(client, rng).unwrap();
 816    let buffers = client.buffers.entry(project.clone()).or_default();
 817    let buffer = if buffers.is_empty() || rng.lock().gen() {
 818        let Some(worktree) = project.read_with(cx, |project, cx| {
 819            project
 820                .worktrees(cx)
 821                .filter(|worktree| {
 822                    let worktree = worktree.read(cx);
 823                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 824                })
 825                .choose(&mut *rng.lock())
 826        }) else {
 827            return Ok(());
 828        };
 829
 830        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 831            let entry = worktree
 832                .entries(false)
 833                .filter(|e| e.is_file())
 834                .choose(&mut *rng.lock())
 835                .unwrap();
 836            (
 837                worktree.root_name().to_string(),
 838                (worktree.id(), entry.path.clone()),
 839            )
 840        });
 841        log::info!(
 842            "{}: opening path {:?} in worktree {} ({})",
 843            client.username,
 844            project_path.1,
 845            project_path.0,
 846            worktree_root_name,
 847        );
 848        let buffer = project
 849            .update(cx, |project, cx| {
 850                project.open_buffer(project_path.clone(), cx)
 851            })
 852            .await?;
 853        log::info!(
 854            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 855            client.username,
 856            project_path.1,
 857            project_path.0,
 858            worktree_root_name,
 859            buffer.read_with(cx, |buffer, _| buffer.remote_id())
 860        );
 861        buffers.insert(buffer.clone());
 862        buffer
 863    } else {
 864        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
 865    };
 866
 867    let choice = rng.lock().gen_range(0..100);
 868    match choice {
 869        0..=9 => {
 870            cx.update(|cx| {
 871                log::info!(
 872                    "{}: dropping buffer {:?}",
 873                    client.username,
 874                    buffer.read(cx).file().unwrap().full_path(cx)
 875                );
 876                buffers.remove(&buffer);
 877                drop(buffer);
 878            });
 879        }
 880        10..=19 => {
 881            let completions = project.update(cx, |project, cx| {
 882                log::info!(
 883                    "{}: requesting completions for buffer {} ({:?})",
 884                    client.username,
 885                    buffer.read(cx).remote_id(),
 886                    buffer.read(cx).file().unwrap().full_path(cx)
 887                );
 888                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 889                project.completions(&buffer, offset, cx)
 890            });
 891            let completions = cx.background().spawn(async move {
 892                completions
 893                    .await
 894                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
 895            });
 896            if rng.lock().gen_bool(0.3) {
 897                log::info!("{}: detaching completions request", client.username);
 898                cx.update(|cx| completions.detach_and_log_err(cx));
 899            } else {
 900                completions.await?;
 901            }
 902        }
 903        20..=29 => {
 904            let code_actions = project.update(cx, |project, cx| {
 905                log::info!(
 906                    "{}: requesting code actions for buffer {} ({:?})",
 907                    client.username,
 908                    buffer.read(cx).remote_id(),
 909                    buffer.read(cx).file().unwrap().full_path(cx)
 910                );
 911                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
 912                project.code_actions(&buffer, range, cx)
 913            });
 914            let code_actions = cx.background().spawn(async move {
 915                code_actions
 916                    .await
 917                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
 918            });
 919            if rng.lock().gen_bool(0.3) {
 920                log::info!("{}: detaching code actions request", client.username);
 921                cx.update(|cx| code_actions.detach_and_log_err(cx));
 922            } else {
 923                code_actions.await?;
 924            }
 925        }
 926        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
 927            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
 928                log::info!(
 929                    "{}: saving buffer {} ({:?})",
 930                    client.username,
 931                    buffer.remote_id(),
 932                    buffer.file().unwrap().full_path(cx)
 933                );
 934                (buffer.version(), buffer.save(cx))
 935            });
 936            let save = cx.background().spawn(async move {
 937                let (saved_version, _, _) = save
 938                    .await
 939                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
 940                assert!(saved_version.observed_all(&requested_version));
 941                Ok::<_, anyhow::Error>(())
 942            });
 943            if rng.lock().gen_bool(0.3) {
 944                log::info!("{}: detaching save request", client.username);
 945                cx.update(|cx| save.detach_and_log_err(cx));
 946            } else {
 947                save.await?;
 948            }
 949        }
 950        40..=44 => {
 951            let prepare_rename = project.update(cx, |project, cx| {
 952                log::info!(
 953                    "{}: preparing rename for buffer {} ({:?})",
 954                    client.username,
 955                    buffer.read(cx).remote_id(),
 956                    buffer.read(cx).file().unwrap().full_path(cx)
 957                );
 958                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 959                project.prepare_rename(buffer, offset, cx)
 960            });
 961            let prepare_rename = cx.background().spawn(async move {
 962                prepare_rename
 963                    .await
 964                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
 965            });
 966            if rng.lock().gen_bool(0.3) {
 967                log::info!("{}: detaching prepare rename request", client.username);
 968                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
 969            } else {
 970                prepare_rename.await?;
 971            }
 972        }
 973        45..=49 => {
 974            let definitions = project.update(cx, |project, cx| {
 975                log::info!(
 976                    "{}: requesting definitions for buffer {} ({:?})",
 977                    client.username,
 978                    buffer.read(cx).remote_id(),
 979                    buffer.read(cx).file().unwrap().full_path(cx)
 980                );
 981                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 982                project.definition(&buffer, offset, cx)
 983            });
 984            let definitions = cx.background().spawn(async move {
 985                definitions
 986                    .await
 987                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
 988            });
 989            if rng.lock().gen_bool(0.3) {
 990                log::info!("{}: detaching definitions request", client.username);
 991                cx.update(|cx| definitions.detach_and_log_err(cx));
 992            } else {
 993                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
 994            }
 995        }
 996        50..=54 => {
 997            let highlights = project.update(cx, |project, cx| {
 998                log::info!(
 999                    "{}: requesting highlights for buffer {} ({:?})",
1000                    client.username,
1001                    buffer.read(cx).remote_id(),
1002                    buffer.read(cx).file().unwrap().full_path(cx)
1003                );
1004                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1005                project.document_highlights(&buffer, offset, cx)
1006            });
1007            let highlights = cx.background().spawn(async move {
1008                highlights
1009                    .await
1010                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1011            });
1012            if rng.lock().gen_bool(0.3) {
1013                log::info!("{}: detaching highlights request", client.username);
1014                cx.update(|cx| highlights.detach_and_log_err(cx));
1015            } else {
1016                highlights.await?;
1017            }
1018        }
1019        55..=59 => {
1020            let search = project.update(cx, |project, cx| {
1021                let query = rng.lock().gen_range('a'..='z');
1022                log::info!("{}: project-wide search {:?}", client.username, query);
1023                project.search(SearchQuery::text(query, false, false), cx)
1024            });
1025            let search = cx.background().spawn(async move {
1026                search
1027                    .await
1028                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1029            });
1030            if rng.lock().gen_bool(0.3) {
1031                log::info!("{}: detaching search request", client.username);
1032                cx.update(|cx| search.detach_and_log_err(cx));
1033            } else {
1034                buffers.extend(search.await?.into_keys());
1035            }
1036        }
1037        _ => {
1038            buffer.update(cx, |buffer, cx| {
1039                log::info!(
1040                    "{}: updating buffer {} ({:?})",
1041                    client.username,
1042                    buffer.remote_id(),
1043                    buffer.file().unwrap().full_path(cx)
1044                );
1045                if rng.lock().gen_bool(0.7) {
1046                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1047                } else {
1048                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1049                }
1050            });
1051        }
1052    }
1053
1054    Ok(())
1055}
1056
1057fn choose_random_project(
1058    client: &mut TestClient,
1059    rng: &Mutex<StdRng>,
1060) -> Option<ModelHandle<Project>> {
1061    client
1062        .local_projects
1063        .iter()
1064        .chain(&client.remote_projects)
1065        .choose(&mut *rng.lock())
1066        .cloned()
1067}
1068
1069fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1070    let mut name = String::new();
1071    for _ in 0..10 {
1072        let letter = rng.lock().gen_range('a'..='z');
1073        name.push(letter);
1074    }
1075    name
1076}