randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16, Rope};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::{
  18    distributions::{Alphanumeric, DistString},
  19    prelude::*,
  20};
  21use settings::Settings;
  22use std::{
  23    env,
  24    ffi::OsStr,
  25    path::{Path, PathBuf},
  26    sync::Arc,
  27};
  28
  29#[gpui::test(iterations = 100)]
  30async fn test_random_collaboration(
  31    cx: &mut TestAppContext,
  32    deterministic: Arc<Deterministic>,
  33    rng: StdRng,
  34) {
  35    deterministic.forbid_parking();
  36    let rng = Arc::new(Mutex::new(rng));
  37
  38    let max_peers = env::var("MAX_PEERS")
  39        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  40        .unwrap_or(5);
  41
  42    let max_operations = env::var("OPERATIONS")
  43        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  44        .unwrap_or(10);
  45
  46    let mut server = TestServer::start(&deterministic).await;
  47    let db = server.app_state.db.clone();
  48
  49    let mut available_users = Vec::new();
  50    for ix in 0..max_peers {
  51        let username = format!("user-{}", ix + 1);
  52        let user_id = db
  53            .create_user(
  54                &format!("{username}@example.com"),
  55                false,
  56                NewUserParams {
  57                    github_login: username.clone(),
  58                    github_user_id: (ix + 1) as i32,
  59                    invite_count: 0,
  60                },
  61            )
  62            .await
  63            .unwrap()
  64            .user_id;
  65        available_users.push((user_id, username));
  66    }
  67
  68    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  69        for (user_id_b, _) in &available_users[ix + 1..] {
  70            server
  71                .app_state
  72                .db
  73                .send_contact_request(*user_id_a, *user_id_b)
  74                .await
  75                .unwrap();
  76            server
  77                .app_state
  78                .db
  79                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  80                .await
  81                .unwrap();
  82        }
  83    }
  84
  85    let mut clients = Vec::new();
  86    let mut user_ids = Vec::new();
  87    let mut op_start_signals = Vec::new();
  88    let mut next_entity_id = 100000;
  89    let allow_server_restarts = rng.lock().gen_bool(0.7);
  90    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  91    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  92
  93    let mut operations = 0;
  94    while operations < max_operations {
  95        let distribution = rng.lock().gen_range(0..100);
  96        match distribution {
  97            0..=19 if !available_users.is_empty() => {
  98                let client_ix = rng.lock().gen_range(0..available_users.len());
  99                let (_, username) = available_users.remove(client_ix);
 100                log::info!("Adding new connection for {}", username);
 101                next_entity_id += 100000;
 102                let mut client_cx = TestAppContext::new(
 103                    cx.foreground_platform(),
 104                    cx.platform(),
 105                    deterministic.build_foreground(next_entity_id),
 106                    deterministic.build_background(),
 107                    cx.font_cache(),
 108                    cx.leak_detector(),
 109                    next_entity_id,
 110                    cx.function_name.clone(),
 111                );
 112
 113                client_cx.update(|cx| cx.set_global(Settings::test(cx)));
 114
 115                let op_start_signal = futures::channel::mpsc::unbounded();
 116                let client = server.create_client(&mut client_cx, &username).await;
 117                user_ids.push(client.current_user_id(&client_cx));
 118                op_start_signals.push(op_start_signal.0);
 119                clients.push(client_cx.foreground().spawn(simulate_client(
 120                    client,
 121                    op_start_signal.1,
 122                    allow_client_disconnection,
 123                    rng.clone(),
 124                    client_cx,
 125                )));
 126
 127                log::info!("Added connection for {}", username);
 128                operations += 1;
 129            }
 130
 131            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 132                let client_ix = rng.lock().gen_range(1..clients.len());
 133                log::info!(
 134                    "Simulating full disconnection of user {}",
 135                    user_ids[client_ix]
 136                );
 137                let removed_user_id = user_ids.remove(client_ix);
 138                let user_connection_ids = server
 139                    .connection_pool
 140                    .lock()
 141                    .user_connection_ids(removed_user_id)
 142                    .collect::<Vec<_>>();
 143                assert_eq!(user_connection_ids.len(), 1);
 144                let removed_peer_id = user_connection_ids[0].into();
 145                let client = clients.remove(client_ix);
 146                op_start_signals.remove(client_ix);
 147                server.forbid_connections();
 148                server.disconnect_client(removed_peer_id);
 149                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 150                deterministic.start_waiting();
 151                log::info!("Waiting for user {} to exit...", removed_user_id);
 152                let (client, mut client_cx) = client.await;
 153                deterministic.finish_waiting();
 154                server.allow_connections();
 155
 156                for project in &client.remote_projects {
 157                    project.read_with(&client_cx, |project, _| {
 158                        assert!(
 159                            project.is_read_only(),
 160                            "project {:?} should be read only",
 161                            project.remote_id()
 162                        )
 163                    });
 164                }
 165                for user_id in &user_ids {
 166                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 167                    let pool = server.connection_pool.lock();
 168                    for contact in contacts {
 169                        if let db::Contact::Accepted { user_id, .. } = contact {
 170                            if pool.is_user_online(user_id) {
 171                                assert_ne!(
 172                                    user_id, removed_user_id,
 173                                    "removed client is still a contact of another peer"
 174                                );
 175                            }
 176                        }
 177                    }
 178                }
 179
 180                log::info!("{} removed", client.username);
 181                available_users.push((removed_user_id, client.username.clone()));
 182                client_cx.update(|cx| {
 183                    cx.clear_globals();
 184                    cx.set_global(Settings::test(cx));
 185                    drop(client);
 186                });
 187
 188                operations += 1;
 189            }
 190
 191            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 192                let client_ix = rng.lock().gen_range(1..clients.len());
 193                let user_id = user_ids[client_ix];
 194                log::info!("Simulating temporary disconnection of user {}", user_id);
 195                let user_connection_ids = server
 196                    .connection_pool
 197                    .lock()
 198                    .user_connection_ids(user_id)
 199                    .collect::<Vec<_>>();
 200                assert_eq!(user_connection_ids.len(), 1);
 201                let peer_id = user_connection_ids[0].into();
 202                server.disconnect_client(peer_id);
 203                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 204                operations += 1;
 205            }
 206
 207            30..=34 if allow_server_restarts => {
 208                log::info!("Simulating server restart");
 209                server.reset().await;
 210                deterministic.advance_clock(RECEIVE_TIMEOUT);
 211                server.start().await.unwrap();
 212                deterministic.advance_clock(CLEANUP_TIMEOUT);
 213                let environment = &server.app_state.config.zed_environment;
 214                let stale_room_ids = server
 215                    .app_state
 216                    .db
 217                    .stale_room_ids(environment, server.id())
 218                    .await
 219                    .unwrap();
 220                assert_eq!(stale_room_ids, vec![]);
 221            }
 222
 223            _ if !op_start_signals.is_empty() => {
 224                while operations < max_operations && rng.lock().gen_bool(0.7) {
 225                    op_start_signals
 226                        .choose(&mut *rng.lock())
 227                        .unwrap()
 228                        .unbounded_send(())
 229                        .unwrap();
 230                    operations += 1;
 231                }
 232
 233                if rng.lock().gen_bool(0.8) {
 234                    deterministic.run_until_parked();
 235                }
 236            }
 237            _ => {}
 238        }
 239    }
 240
 241    drop(op_start_signals);
 242    deterministic.start_waiting();
 243    let clients = futures::future::join_all(clients).await;
 244    deterministic.finish_waiting();
 245    deterministic.run_until_parked();
 246
 247    for (client, client_cx) in &clients {
 248        for guest_project in &client.remote_projects {
 249            guest_project.read_with(client_cx, |guest_project, cx| {
 250                let host_project = clients.iter().find_map(|(client, cx)| {
 251                    let project = client.local_projects.iter().find(|host_project| {
 252                        host_project.read_with(cx, |host_project, _| {
 253                            host_project.remote_id() == guest_project.remote_id()
 254                        })
 255                    })?;
 256                    Some((project, cx))
 257                });
 258
 259                if !guest_project.is_read_only() {
 260                    if let Some((host_project, host_cx)) = host_project {
 261                        let host_worktree_snapshots =
 262                            host_project.read_with(host_cx, |host_project, cx| {
 263                                host_project
 264                                    .worktrees(cx)
 265                                    .map(|worktree| {
 266                                        let worktree = worktree.read(cx);
 267                                        (worktree.id(), worktree.snapshot())
 268                                    })
 269                                    .collect::<BTreeMap<_, _>>()
 270                            });
 271                        let guest_worktree_snapshots = guest_project
 272                            .worktrees(cx)
 273                            .map(|worktree| {
 274                                let worktree = worktree.read(cx);
 275                                (worktree.id(), worktree.snapshot())
 276                            })
 277                            .collect::<BTreeMap<_, _>>();
 278
 279                        assert_eq!(
 280                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 281                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 282                            "{} has different worktrees than the host",
 283                            client.username
 284                        );
 285
 286                        for (id, host_snapshot) in &host_worktree_snapshots {
 287                            let guest_snapshot = &guest_worktree_snapshots[id];
 288                            assert_eq!(
 289                                guest_snapshot.root_name(),
 290                                host_snapshot.root_name(),
 291                                "{} has different root name than the host for worktree {}",
 292                                client.username,
 293                                id
 294                            );
 295                            assert_eq!(
 296                                guest_snapshot.abs_path(),
 297                                host_snapshot.abs_path(),
 298                                "{} has different abs path than the host for worktree {}",
 299                                client.username,
 300                                id
 301                            );
 302                            assert_eq!(
 303                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 304                                host_snapshot.entries(false).collect::<Vec<_>>(),
 305                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 306                                client.username,
 307                                id,
 308                                host_snapshot.abs_path(),
 309                                host_project.read_with(host_cx, |project, _| project.remote_id())
 310                            );
 311                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 312                        }
 313                    }
 314                }
 315
 316                guest_project.check_invariants(cx);
 317            });
 318        }
 319
 320        for (guest_project, guest_buffers) in &client.buffers {
 321            let project_id = if guest_project.read_with(client_cx, |project, _| {
 322                project.is_local() || project.is_read_only()
 323            }) {
 324                continue;
 325            } else {
 326                guest_project
 327                    .read_with(client_cx, |project, _| project.remote_id())
 328                    .unwrap()
 329            };
 330            let guest_user_id = client.user_id().unwrap();
 331
 332            let host_project = clients.iter().find_map(|(client, cx)| {
 333                let project = client.local_projects.iter().find(|host_project| {
 334                    host_project.read_with(cx, |host_project, _| {
 335                        host_project.remote_id() == Some(project_id)
 336                    })
 337                })?;
 338                Some((client.user_id().unwrap(), project, cx))
 339            });
 340
 341            let (host_user_id, host_project, host_cx) =
 342                if let Some((host_user_id, host_project, host_cx)) = host_project {
 343                    (host_user_id, host_project, host_cx)
 344                } else {
 345                    continue;
 346                };
 347
 348            for guest_buffer in guest_buffers {
 349                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 350                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 351                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 352                        panic!(
 353                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 354                            client.username,
 355                            client.peer_id(),
 356                            buffer_id
 357                        )
 358                    })
 359                });
 360                let path = host_buffer
 361                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 362
 363                assert_eq!(
 364                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 365                    0,
 366                    "{}, buffer {}, path {:?} has deferred operations",
 367                    client.username,
 368                    buffer_id,
 369                    path,
 370                );
 371                assert_eq!(
 372                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 373                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 374                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 375                    client.username,
 376                    buffer_id,
 377                    path
 378                );
 379
 380                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 381                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 382                match (host_file, guest_file) {
 383                    (Some(host_file), Some(guest_file)) => {
 384                        assert_eq!(guest_file.path(), host_file.path());
 385                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 386                        assert_eq!(
 387                            guest_file.mtime(),
 388                            host_file.mtime(),
 389                            "guest {} mtime does not match host {} for path {:?} in project {}",
 390                            guest_user_id,
 391                            host_user_id,
 392                            guest_file.path(),
 393                            project_id,
 394                        );
 395                    }
 396                    (None, None) => {}
 397                    (None, _) => panic!("host's file is None, guest's isn't"),
 398                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 399                }
 400
 401                let host_diff_base =
 402                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 403                let guest_diff_base = guest_buffer
 404                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 405                assert_eq!(guest_diff_base, host_diff_base);
 406
 407                let host_saved_version =
 408                    host_buffer.read_with(host_cx, |b, _| b.saved_version().clone());
 409                let guest_saved_version =
 410                    guest_buffer.read_with(client_cx, |b, _| b.saved_version().clone());
 411                assert_eq!(guest_saved_version, host_saved_version);
 412
 413                let host_saved_version_fingerprint =
 414                    host_buffer.read_with(host_cx, |b, _| b.saved_version_fingerprint());
 415                let guest_saved_version_fingerprint =
 416                    guest_buffer.read_with(client_cx, |b, _| b.saved_version_fingerprint());
 417                assert_eq!(
 418                    guest_saved_version_fingerprint,
 419                    host_saved_version_fingerprint
 420                );
 421
 422                let host_saved_mtime = host_buffer.read_with(host_cx, |b, _| b.saved_mtime());
 423                let guest_saved_mtime = guest_buffer.read_with(client_cx, |b, _| b.saved_mtime());
 424                assert_eq!(guest_saved_mtime, host_saved_mtime);
 425
 426                let host_is_dirty = host_buffer.read_with(host_cx, |b, _| b.is_dirty());
 427                let guest_is_dirty = guest_buffer.read_with(client_cx, |b, _| b.is_dirty());
 428                assert_eq!(guest_is_dirty, host_is_dirty);
 429
 430                let host_has_conflict = host_buffer.read_with(host_cx, |b, _| b.has_conflict());
 431                let guest_has_conflict = guest_buffer.read_with(client_cx, |b, _| b.has_conflict());
 432                assert_eq!(guest_has_conflict, host_has_conflict);
 433            }
 434        }
 435    }
 436
 437    for (client, mut cx) in clients {
 438        cx.update(|cx| {
 439            cx.clear_globals();
 440            cx.set_global(Settings::test(cx));
 441            drop(client);
 442        });
 443    }
 444}
 445
 446async fn simulate_client(
 447    mut client: TestClient,
 448    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 449    can_hang_up: bool,
 450    rng: Arc<Mutex<StdRng>>,
 451    mut cx: TestAppContext,
 452) -> (TestClient, TestAppContext) {
 453    // Setup language server
 454    let mut language = Language::new(
 455        LanguageConfig {
 456            name: "Rust".into(),
 457            path_suffixes: vec!["rs".to_string()],
 458            ..Default::default()
 459        },
 460        None,
 461    );
 462    let _fake_language_servers = language
 463        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 464            name: "the-fake-language-server",
 465            capabilities: lsp::LanguageServer::full_capabilities(),
 466            initializer: Some(Box::new({
 467                let rng = rng.clone();
 468                let fs = client.fs.clone();
 469                move |fake_server: &mut FakeLanguageServer| {
 470                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 471                        |_, _| async move {
 472                            Ok(Some(lsp::CompletionResponse::Array(vec![
 473                                lsp::CompletionItem {
 474                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 475                                        range: lsp::Range::new(
 476                                            lsp::Position::new(0, 0),
 477                                            lsp::Position::new(0, 0),
 478                                        ),
 479                                        new_text: "the-new-text".to_string(),
 480                                    })),
 481                                    ..Default::default()
 482                                },
 483                            ])))
 484                        },
 485                    );
 486
 487                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 488                        |_, _| async move {
 489                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 490                                lsp::CodeAction {
 491                                    title: "the-code-action".to_string(),
 492                                    ..Default::default()
 493                                },
 494                            )]))
 495                        },
 496                    );
 497
 498                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 499                        |params, _| async move {
 500                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 501                                params.position,
 502                                params.position,
 503                            ))))
 504                        },
 505                    );
 506
 507                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 508                        let fs = fs.clone();
 509                        let rng = rng.clone();
 510                        move |_, _| {
 511                            let fs = fs.clone();
 512                            let rng = rng.clone();
 513                            async move {
 514                                let files = fs.files().await;
 515                                let mut rng = rng.lock();
 516                                let count = rng.gen_range::<usize, _>(1..3);
 517                                let files = (0..count)
 518                                    .map(|_| files.choose(&mut *rng).unwrap())
 519                                    .collect::<Vec<_>>();
 520                                log::info!("LSP: Returning definitions in files {:?}", &files);
 521                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 522                                    files
 523                                        .into_iter()
 524                                        .map(|file| lsp::Location {
 525                                            uri: lsp::Url::from_file_path(file).unwrap(),
 526                                            range: Default::default(),
 527                                        })
 528                                        .collect(),
 529                                )))
 530                            }
 531                        }
 532                    });
 533
 534                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 535                        let rng = rng.clone();
 536                        move |_, _| {
 537                            let mut highlights = Vec::new();
 538                            let highlight_count = rng.lock().gen_range(1..=5);
 539                            for _ in 0..highlight_count {
 540                                let start_row = rng.lock().gen_range(0..100);
 541                                let start_column = rng.lock().gen_range(0..100);
 542                                let start = PointUtf16::new(start_row, start_column);
 543                                let end_row = rng.lock().gen_range(0..100);
 544                                let end_column = rng.lock().gen_range(0..100);
 545                                let end = PointUtf16::new(end_row, end_column);
 546                                let range = if start > end { end..start } else { start..end };
 547                                highlights.push(lsp::DocumentHighlight {
 548                                    range: range_to_lsp(range.clone()),
 549                                    kind: Some(lsp::DocumentHighlightKind::READ),
 550                                });
 551                            }
 552                            highlights.sort_unstable_by_key(|highlight| {
 553                                (highlight.range.start, highlight.range.end)
 554                            });
 555                            async move { Ok(Some(highlights)) }
 556                        }
 557                    });
 558                }
 559            })),
 560            ..Default::default()
 561        }))
 562        .await;
 563    client.language_registry.add(Arc::new(language));
 564
 565    while op_start_signal.next().await.is_some() {
 566        if let Err(error) =
 567            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 568        {
 569            log::error!("{} error: {:?}", client.username, error);
 570        }
 571
 572        cx.background().simulate_random_delay().await;
 573    }
 574    log::info!("{}: done", client.username);
 575
 576    (client, cx)
 577}
 578
 579async fn randomly_mutate_client(
 580    client: &mut TestClient,
 581    can_hang_up: bool,
 582    rng: Arc<Mutex<StdRng>>,
 583    cx: &mut TestAppContext,
 584) -> Result<()> {
 585    let choice = rng.lock().gen_range(0..100);
 586    match choice {
 587        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 588        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 589        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 590            randomly_mutate_worktrees(client, &rng, cx).await?;
 591        }
 592        60..=74 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 593            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 594        }
 595        75..=84 => randomly_mutate_git(client, &rng).await,
 596        _ => randomly_mutate_fs(client, &rng).await,
 597    }
 598
 599    Ok(())
 600}
 601
 602async fn randomly_mutate_active_call(
 603    client: &mut TestClient,
 604    can_hang_up: bool,
 605    rng: &Mutex<StdRng>,
 606    cx: &mut TestAppContext,
 607) -> Result<()> {
 608    let active_call = cx.read(ActiveCall::global);
 609    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 610        if rng.lock().gen_bool(0.7) {
 611            log::info!("{}: accepting incoming call", client.username);
 612            active_call
 613                .update(cx, |call, cx| call.accept_incoming(cx))
 614                .await?;
 615        } else {
 616            log::info!("{}: declining incoming call", client.username);
 617            active_call.update(cx, |call, _| call.decline_incoming())?;
 618        }
 619    } else {
 620        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 621            user_store
 622                .contacts()
 623                .iter()
 624                .filter(|contact| contact.online && !contact.busy)
 625                .cloned()
 626                .collect::<Vec<_>>()
 627        });
 628
 629        let distribution = rng.lock().gen_range(0..100);
 630        match distribution {
 631            0..=29 if !available_contacts.is_empty() => {
 632                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 633                log::info!(
 634                    "{}: inviting {}",
 635                    client.username,
 636                    contact.user.github_login
 637                );
 638                active_call
 639                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 640                    .await?;
 641            }
 642            30..=39
 643                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 644            {
 645                log::info!("{}: hanging up", client.username);
 646                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 647            }
 648            _ => {}
 649        }
 650    }
 651
 652    Ok(())
 653}
 654
 655async fn randomly_mutate_git(client: &mut TestClient, rng: &Mutex<StdRng>) {
 656    let directories = client.fs.directories().await;
 657    let mut dir_path = directories.choose(&mut *rng.lock()).unwrap().clone();
 658    if dir_path.file_name() == Some(OsStr::new(".git")) {
 659        dir_path.pop();
 660    }
 661    let mut git_dir_path = dir_path.clone();
 662    git_dir_path.push(".git");
 663
 664    if !directories.contains(&git_dir_path) {
 665        log::info!(
 666            "{}: creating git directory at {:?}",
 667            client.username,
 668            git_dir_path
 669        );
 670        client.fs.create_dir(&git_dir_path).await.unwrap();
 671    }
 672
 673    let mut child_file_paths = child_file_paths(client, &dir_path).await;
 674    let count = rng.lock().gen_range(0..=child_file_paths.len());
 675    child_file_paths.shuffle(&mut *rng.lock());
 676    child_file_paths.truncate(count);
 677
 678    let mut new_index = Vec::new();
 679    for abs_child_file_path in &child_file_paths {
 680        let child_file_path = abs_child_file_path.strip_prefix(&dir_path).unwrap();
 681        let new_base = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 682        new_index.push((child_file_path, new_base));
 683    }
 684    log::info!(
 685        "{}: updating Git index at {:?}: {:#?}",
 686        client.username,
 687        git_dir_path,
 688        new_index
 689    );
 690    client
 691        .fs
 692        .set_index_for_repo(&git_dir_path, &new_index)
 693        .await;
 694}
 695
 696async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 697    let parent_dir_path = client
 698        .fs
 699        .directories()
 700        .await
 701        .choose(&mut *rng.lock())
 702        .unwrap()
 703        .clone();
 704
 705    let is_dir = rng.lock().gen::<bool>();
 706    if is_dir {
 707        let mut dir_path = parent_dir_path.clone();
 708        dir_path.push(gen_file_name(rng));
 709        log::info!("{}: creating local dir at {:?}", client.username, dir_path);
 710        client.fs.create_dir(&dir_path).await.unwrap();
 711    } else {
 712        let child_file_paths = child_file_paths(client, &parent_dir_path).await;
 713        let create_new_file = child_file_paths.is_empty() || rng.lock().gen();
 714        let text = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 715        if create_new_file {
 716            let mut file_path = parent_dir_path.clone();
 717            file_path.push(gen_file_name(rng));
 718            file_path.set_extension("rs");
 719            log::info!(
 720                "{}: creating local file at {:?}",
 721                client.username,
 722                file_path
 723            );
 724            client
 725                .fs
 726                .create_file(&file_path, Default::default())
 727                .await
 728                .unwrap();
 729            log::info!(
 730                "{}: setting local file {:?} text to {:?}",
 731                client.username,
 732                file_path,
 733                text
 734            );
 735            client
 736                .fs
 737                .save(&file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 738                .await
 739                .unwrap();
 740        } else {
 741            let file_path = child_file_paths.choose(&mut *rng.lock()).unwrap();
 742            log::info!(
 743                "{}: setting local file {:?} text to {:?}",
 744                client.username,
 745                file_path,
 746                text
 747            );
 748            client
 749                .fs
 750                .save(file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 751                .await
 752                .unwrap();
 753        }
 754    }
 755}
 756
 757async fn randomly_mutate_projects(
 758    client: &mut TestClient,
 759    rng: &Mutex<StdRng>,
 760    cx: &mut TestAppContext,
 761) -> Result<()> {
 762    let active_call = cx.read(ActiveCall::global);
 763    let remote_projects =
 764        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 765            room.read_with(cx, |room, _| {
 766                room.remote_participants()
 767                    .values()
 768                    .flat_map(|participant| participant.projects.clone())
 769                    .collect::<Vec<_>>()
 770            })
 771        } else {
 772            Default::default()
 773        };
 774
 775    let project = if remote_projects.is_empty() || rng.lock().gen() {
 776        if client.local_projects.is_empty() || rng.lock().gen() {
 777            let paths = client.fs.paths().await;
 778            let local_project = if paths.is_empty() || rng.lock().gen() {
 779                let root_path = client.create_new_root_dir();
 780                client.fs.create_dir(&root_path).await.unwrap();
 781                client
 782                    .fs
 783                    .create_file(&root_path.join("main.rs"), Default::default())
 784                    .await
 785                    .unwrap();
 786                log::info!(
 787                    "{}: opening local project at {:?}",
 788                    client.username,
 789                    root_path
 790                );
 791                client.build_local_project(root_path, cx).await.0
 792            } else {
 793                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 794                log::info!(
 795                    "{}: opening local project at {:?}",
 796                    client.username,
 797                    root_path
 798                );
 799                client.build_local_project(root_path, cx).await.0
 800            };
 801            client.local_projects.push(local_project.clone());
 802            local_project
 803        } else {
 804            client
 805                .local_projects
 806                .choose(&mut *rng.lock())
 807                .unwrap()
 808                .clone()
 809        }
 810    } else {
 811        if client.remote_projects.is_empty() || rng.lock().gen() {
 812            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 813            let remote_project = if let Some(project) =
 814                client.remote_projects.iter().find(|project| {
 815                    project.read_with(cx, |project, _| {
 816                        project.remote_id() == Some(remote_project_id)
 817                    })
 818                }) {
 819                project.clone()
 820            } else {
 821                log::info!(
 822                    "{}: opening remote project {}",
 823                    client.username,
 824                    remote_project_id
 825                );
 826                let call = cx.read(ActiveCall::global);
 827                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 828                let remote_project = room
 829                    .update(cx, |room, cx| {
 830                        room.join_project(
 831                            remote_project_id,
 832                            client.language_registry.clone(),
 833                            FakeFs::new(cx.background().clone()),
 834                            cx,
 835                        )
 836                    })
 837                    .await?;
 838                client.remote_projects.push(remote_project.clone());
 839                remote_project
 840            };
 841
 842            remote_project
 843        } else {
 844            client
 845                .remote_projects
 846                .choose(&mut *rng.lock())
 847                .unwrap()
 848                .clone()
 849        }
 850    };
 851
 852    if active_call.read_with(cx, |call, _| call.room().is_some())
 853        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 854    {
 855        match active_call
 856            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 857            .await
 858        {
 859            Ok(project_id) => {
 860                log::info!("{}: shared project with id {}", client.username, project_id);
 861            }
 862            Err(error) => {
 863                log::error!("{}: error sharing project, {:?}", client.username, error);
 864            }
 865        }
 866    }
 867
 868    let choice = rng.lock().gen_range(0..100);
 869    match choice {
 870        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 871            let paths = client.fs.paths().await;
 872            let path = paths.choose(&mut *rng.lock()).unwrap();
 873            log::info!(
 874                "{}: finding/creating local worktree for path {:?}",
 875                client.username,
 876                path
 877            );
 878            project
 879                .update(cx, |project, cx| {
 880                    project.find_or_create_local_worktree(&path, true, cx)
 881                })
 882                .await
 883                .unwrap();
 884        }
 885        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 886            log::info!(
 887                "{}: dropping remote project {}",
 888                client.username,
 889                project.read_with(cx, |project, _| project.remote_id().unwrap())
 890            );
 891
 892            cx.update(|_| {
 893                client
 894                    .remote_projects
 895                    .retain(|remote_project| *remote_project != project);
 896                client.buffers.remove(&project);
 897                drop(project);
 898            });
 899        }
 900        _ => {}
 901    }
 902
 903    Ok(())
 904}
 905
 906async fn randomly_mutate_worktrees(
 907    client: &mut TestClient,
 908    rng: &Mutex<StdRng>,
 909    cx: &mut TestAppContext,
 910) -> Result<()> {
 911    let project = choose_random_project(client, rng).unwrap();
 912    let Some(worktree) = project.read_with(cx, |project, cx| {
 913        project
 914            .worktrees(cx)
 915            .filter(|worktree| {
 916                let worktree = worktree.read(cx);
 917                worktree.is_visible()
 918                    && worktree.entries(false).any(|e| e.is_file())
 919                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 920            })
 921            .choose(&mut *rng.lock())
 922    }) else {
 923        return Ok(())
 924    };
 925
 926    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 927        (worktree.id(), worktree.root_name().to_string())
 928    });
 929
 930    let is_dir = rng.lock().gen::<bool>();
 931    let mut new_path = PathBuf::new();
 932    new_path.push(gen_file_name(rng));
 933    if !is_dir {
 934        new_path.set_extension("rs");
 935    }
 936    log::info!(
 937        "{}: creating {:?} in worktree {} ({})",
 938        client.username,
 939        new_path,
 940        worktree_id,
 941        worktree_root_name,
 942    );
 943    project
 944        .update(cx, |project, cx| {
 945            project.create_entry((worktree_id, new_path), is_dir, cx)
 946        })
 947        .unwrap()
 948        .await?;
 949    Ok(())
 950}
 951
 952async fn randomly_query_and_mutate_buffers(
 953    client: &mut TestClient,
 954    rng: &Mutex<StdRng>,
 955    cx: &mut TestAppContext,
 956) -> Result<()> {
 957    let project = choose_random_project(client, rng).unwrap();
 958    let buffers = client.buffers.entry(project.clone()).or_default();
 959    let buffer = if buffers.is_empty() || rng.lock().gen() {
 960        let Some(worktree) = project.read_with(cx, |project, cx| {
 961            project
 962                .worktrees(cx)
 963                .filter(|worktree| {
 964                    let worktree = worktree.read(cx);
 965                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 966                })
 967                .choose(&mut *rng.lock())
 968        }) else {
 969            return Ok(());
 970        };
 971
 972        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 973            let entry = worktree
 974                .entries(false)
 975                .filter(|e| e.is_file())
 976                .choose(&mut *rng.lock())
 977                .unwrap();
 978            (
 979                worktree.root_name().to_string(),
 980                (worktree.id(), entry.path.clone()),
 981            )
 982        });
 983        log::info!(
 984            "{}: opening path {:?} in worktree {} ({})",
 985            client.username,
 986            project_path.1,
 987            project_path.0,
 988            worktree_root_name,
 989        );
 990        let buffer = project
 991            .update(cx, |project, cx| {
 992                project.open_buffer(project_path.clone(), cx)
 993            })
 994            .await?;
 995        log::info!(
 996            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 997            client.username,
 998            project_path.1,
 999            project_path.0,
1000            worktree_root_name,
1001            buffer.read_with(cx, |buffer, _| buffer.remote_id())
1002        );
1003        buffers.insert(buffer.clone());
1004        buffer
1005    } else {
1006        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
1007    };
1008
1009    let choice = rng.lock().gen_range(0..100);
1010    match choice {
1011        0..=9 => {
1012            cx.update(|cx| {
1013                log::info!(
1014                    "{}: dropping buffer {:?}",
1015                    client.username,
1016                    buffer.read(cx).file().unwrap().full_path(cx)
1017                );
1018                buffers.remove(&buffer);
1019                drop(buffer);
1020            });
1021        }
1022        10..=19 => {
1023            let completions = project.update(cx, |project, cx| {
1024                log::info!(
1025                    "{}: requesting completions for buffer {} ({:?})",
1026                    client.username,
1027                    buffer.read(cx).remote_id(),
1028                    buffer.read(cx).file().unwrap().full_path(cx)
1029                );
1030                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1031                project.completions(&buffer, offset, cx)
1032            });
1033            let completions = cx.background().spawn(async move {
1034                completions
1035                    .await
1036                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
1037            });
1038            if rng.lock().gen_bool(0.3) {
1039                log::info!("{}: detaching completions request", client.username);
1040                cx.update(|cx| completions.detach_and_log_err(cx));
1041            } else {
1042                completions.await?;
1043            }
1044        }
1045        20..=29 => {
1046            let code_actions = project.update(cx, |project, cx| {
1047                log::info!(
1048                    "{}: requesting code actions for buffer {} ({:?})",
1049                    client.username,
1050                    buffer.read(cx).remote_id(),
1051                    buffer.read(cx).file().unwrap().full_path(cx)
1052                );
1053                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
1054                project.code_actions(&buffer, range, cx)
1055            });
1056            let code_actions = cx.background().spawn(async move {
1057                code_actions
1058                    .await
1059                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
1060            });
1061            if rng.lock().gen_bool(0.3) {
1062                log::info!("{}: detaching code actions request", client.username);
1063                cx.update(|cx| code_actions.detach_and_log_err(cx));
1064            } else {
1065                code_actions.await?;
1066            }
1067        }
1068        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1069            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1070                log::info!(
1071                    "{}: saving buffer {} ({:?})",
1072                    client.username,
1073                    buffer.remote_id(),
1074                    buffer.file().unwrap().full_path(cx)
1075                );
1076                (buffer.version(), buffer.save(cx))
1077            });
1078            let save = cx.background().spawn(async move {
1079                let (saved_version, _, _) = save
1080                    .await
1081                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1082                assert!(saved_version.observed_all(&requested_version));
1083                Ok::<_, anyhow::Error>(())
1084            });
1085            if rng.lock().gen_bool(0.3) {
1086                log::info!("{}: detaching save request", client.username);
1087                cx.update(|cx| save.detach_and_log_err(cx));
1088            } else {
1089                save.await?;
1090            }
1091        }
1092        40..=44 => {
1093            let prepare_rename = project.update(cx, |project, cx| {
1094                log::info!(
1095                    "{}: preparing rename for buffer {} ({:?})",
1096                    client.username,
1097                    buffer.read(cx).remote_id(),
1098                    buffer.read(cx).file().unwrap().full_path(cx)
1099                );
1100                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1101                project.prepare_rename(buffer, offset, cx)
1102            });
1103            let prepare_rename = cx.background().spawn(async move {
1104                prepare_rename
1105                    .await
1106                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1107            });
1108            if rng.lock().gen_bool(0.3) {
1109                log::info!("{}: detaching prepare rename request", client.username);
1110                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1111            } else {
1112                prepare_rename.await?;
1113            }
1114        }
1115        45..=49 => {
1116            let definitions = project.update(cx, |project, cx| {
1117                log::info!(
1118                    "{}: requesting definitions for buffer {} ({:?})",
1119                    client.username,
1120                    buffer.read(cx).remote_id(),
1121                    buffer.read(cx).file().unwrap().full_path(cx)
1122                );
1123                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1124                project.definition(&buffer, offset, cx)
1125            });
1126            let definitions = cx.background().spawn(async move {
1127                definitions
1128                    .await
1129                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1130            });
1131            if rng.lock().gen_bool(0.3) {
1132                log::info!("{}: detaching definitions request", client.username);
1133                cx.update(|cx| definitions.detach_and_log_err(cx));
1134            } else {
1135                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1136            }
1137        }
1138        50..=54 => {
1139            let highlights = project.update(cx, |project, cx| {
1140                log::info!(
1141                    "{}: requesting highlights for buffer {} ({:?})",
1142                    client.username,
1143                    buffer.read(cx).remote_id(),
1144                    buffer.read(cx).file().unwrap().full_path(cx)
1145                );
1146                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1147                project.document_highlights(&buffer, offset, cx)
1148            });
1149            let highlights = cx.background().spawn(async move {
1150                highlights
1151                    .await
1152                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1153            });
1154            if rng.lock().gen_bool(0.3) {
1155                log::info!("{}: detaching highlights request", client.username);
1156                cx.update(|cx| highlights.detach_and_log_err(cx));
1157            } else {
1158                highlights.await?;
1159            }
1160        }
1161        55..=59 => {
1162            let search = project.update(cx, |project, cx| {
1163                let query = rng.lock().gen_range('a'..='z');
1164                log::info!("{}: project-wide search {:?}", client.username, query);
1165                project.search(SearchQuery::text(query, false, false), cx)
1166            });
1167            let search = cx.background().spawn(async move {
1168                search
1169                    .await
1170                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1171            });
1172            if rng.lock().gen_bool(0.3) {
1173                log::info!("{}: detaching search request", client.username);
1174                cx.update(|cx| search.detach_and_log_err(cx));
1175            } else {
1176                buffers.extend(search.await?.into_keys());
1177            }
1178        }
1179        _ => {
1180            buffer.update(cx, |buffer, cx| {
1181                log::info!(
1182                    "{}: updating buffer {} ({:?})",
1183                    client.username,
1184                    buffer.remote_id(),
1185                    buffer.file().unwrap().full_path(cx)
1186                );
1187                if rng.lock().gen_bool(0.7) {
1188                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1189                } else {
1190                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1191                }
1192            });
1193        }
1194    }
1195
1196    Ok(())
1197}
1198
1199fn choose_random_project(
1200    client: &mut TestClient,
1201    rng: &Mutex<StdRng>,
1202) -> Option<ModelHandle<Project>> {
1203    client
1204        .local_projects
1205        .iter()
1206        .chain(&client.remote_projects)
1207        .choose(&mut *rng.lock())
1208        .cloned()
1209}
1210
1211fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1212    let mut name = String::new();
1213    for _ in 0..10 {
1214        let letter = rng.lock().gen_range('a'..='z');
1215        name.push(letter);
1216    }
1217    name
1218}
1219
1220async fn child_file_paths(client: &TestClient, dir_path: &Path) -> Vec<PathBuf> {
1221    let mut child_paths = client.fs.read_dir(dir_path).await.unwrap();
1222    let mut child_file_paths = Vec::new();
1223    while let Some(child_path) = child_paths.next().await {
1224        let child_path = child_path.unwrap();
1225        if client.fs.is_file(&child_path).await {
1226            child_file_paths.push(child_path);
1227        }
1228    }
1229    child_file_paths
1230}