randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16, Rope};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::{
  18    distributions::{Alphanumeric, DistString},
  19    prelude::*,
  20};
  21use settings::Settings;
  22use std::{
  23    env,
  24    ffi::OsStr,
  25    path::{Path, PathBuf},
  26    sync::Arc,
  27};
  28
  29#[gpui::test(iterations = 100)]
  30async fn test_random_collaboration(
  31    cx: &mut TestAppContext,
  32    deterministic: Arc<Deterministic>,
  33    rng: StdRng,
  34) {
  35    deterministic.forbid_parking();
  36    let rng = Arc::new(Mutex::new(rng));
  37
  38    let max_peers = env::var("MAX_PEERS")
  39        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  40        .unwrap_or(5);
  41
  42    let max_operations = env::var("OPERATIONS")
  43        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  44        .unwrap_or(10);
  45
  46    let mut server = TestServer::start(&deterministic).await;
  47    let db = server.app_state.db.clone();
  48
  49    let mut available_users = Vec::new();
  50    for ix in 0..max_peers {
  51        let username = format!("user-{}", ix + 1);
  52        let user_id = db
  53            .create_user(
  54                &format!("{username}@example.com"),
  55                false,
  56                NewUserParams {
  57                    github_login: username.clone(),
  58                    github_user_id: (ix + 1) as i32,
  59                    invite_count: 0,
  60                },
  61            )
  62            .await
  63            .unwrap()
  64            .user_id;
  65        available_users.push((user_id, username));
  66    }
  67
  68    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  69        for (user_id_b, _) in &available_users[ix + 1..] {
  70            server
  71                .app_state
  72                .db
  73                .send_contact_request(*user_id_a, *user_id_b)
  74                .await
  75                .unwrap();
  76            server
  77                .app_state
  78                .db
  79                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  80                .await
  81                .unwrap();
  82        }
  83    }
  84
  85    let mut clients = Vec::new();
  86    let mut user_ids = Vec::new();
  87    let mut op_start_signals = Vec::new();
  88    let mut next_entity_id = 100000;
  89    let allow_server_restarts = rng.lock().gen_bool(0.7);
  90    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  91    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  92
  93    let mut operations = 0;
  94    while operations < max_operations {
  95        let distribution = rng.lock().gen_range(0..100);
  96        match distribution {
  97            0..=19 if !available_users.is_empty() => {
  98                let client_ix = rng.lock().gen_range(0..available_users.len());
  99                let (_, username) = available_users.remove(client_ix);
 100                log::info!("Adding new connection for {}", username);
 101                next_entity_id += 100000;
 102                let mut client_cx = TestAppContext::new(
 103                    cx.foreground_platform(),
 104                    cx.platform(),
 105                    deterministic.build_foreground(next_entity_id),
 106                    deterministic.build_background(),
 107                    cx.font_cache(),
 108                    cx.leak_detector(),
 109                    next_entity_id,
 110                    cx.function_name.clone(),
 111                );
 112
 113                client_cx.update(|cx| cx.set_global(Settings::test(cx)));
 114
 115                let op_start_signal = futures::channel::mpsc::unbounded();
 116                let client = server.create_client(&mut client_cx, &username).await;
 117                user_ids.push(client.current_user_id(&client_cx));
 118                op_start_signals.push(op_start_signal.0);
 119                clients.push(client_cx.foreground().spawn(simulate_client(
 120                    client,
 121                    op_start_signal.1,
 122                    allow_client_disconnection,
 123                    rng.clone(),
 124                    client_cx,
 125                )));
 126
 127                log::info!("Added connection for {}", username);
 128                operations += 1;
 129            }
 130
 131            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 132                let client_ix = rng.lock().gen_range(1..clients.len());
 133                log::info!(
 134                    "Simulating full disconnection of user {}",
 135                    user_ids[client_ix]
 136                );
 137                let removed_user_id = user_ids.remove(client_ix);
 138                let user_connection_ids = server
 139                    .connection_pool
 140                    .lock()
 141                    .user_connection_ids(removed_user_id)
 142                    .collect::<Vec<_>>();
 143                assert_eq!(user_connection_ids.len(), 1);
 144                let removed_peer_id = user_connection_ids[0].into();
 145                let client = clients.remove(client_ix);
 146                op_start_signals.remove(client_ix);
 147                server.forbid_connections();
 148                server.disconnect_client(removed_peer_id);
 149                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 150                deterministic.start_waiting();
 151                log::info!("Waiting for user {} to exit...", removed_user_id);
 152                let (client, mut client_cx) = client.await;
 153                deterministic.finish_waiting();
 154                server.allow_connections();
 155
 156                for project in &client.remote_projects {
 157                    project.read_with(&client_cx, |project, _| {
 158                        assert!(
 159                            project.is_read_only(),
 160                            "project {:?} should be read only",
 161                            project.remote_id()
 162                        )
 163                    });
 164                }
 165                for user_id in &user_ids {
 166                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 167                    let pool = server.connection_pool.lock();
 168                    for contact in contacts {
 169                        if let db::Contact::Accepted { user_id, .. } = contact {
 170                            if pool.is_user_online(user_id) {
 171                                assert_ne!(
 172                                    user_id, removed_user_id,
 173                                    "removed client is still a contact of another peer"
 174                                );
 175                            }
 176                        }
 177                    }
 178                }
 179
 180                log::info!("{} removed", client.username);
 181                available_users.push((removed_user_id, client.username.clone()));
 182                client_cx.update(|cx| {
 183                    cx.clear_globals();
 184                    cx.set_global(Settings::test(cx));
 185                    drop(client);
 186                });
 187
 188                operations += 1;
 189            }
 190
 191            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 192                let client_ix = rng.lock().gen_range(1..clients.len());
 193                let user_id = user_ids[client_ix];
 194                log::info!("Simulating temporary disconnection of user {}", user_id);
 195                let user_connection_ids = server
 196                    .connection_pool
 197                    .lock()
 198                    .user_connection_ids(user_id)
 199                    .collect::<Vec<_>>();
 200                assert_eq!(user_connection_ids.len(), 1);
 201                let peer_id = user_connection_ids[0].into();
 202                server.disconnect_client(peer_id);
 203                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 204                operations += 1;
 205            }
 206
 207            30..=34 if allow_server_restarts => {
 208                log::info!("Simulating server restart");
 209                server.reset().await;
 210                deterministic.advance_clock(RECEIVE_TIMEOUT);
 211                server.start().await.unwrap();
 212                deterministic.advance_clock(CLEANUP_TIMEOUT);
 213                let environment = &server.app_state.config.zed_environment;
 214                let stale_room_ids = server
 215                    .app_state
 216                    .db
 217                    .stale_room_ids(environment, server.id())
 218                    .await
 219                    .unwrap();
 220                assert_eq!(stale_room_ids, vec![]);
 221            }
 222
 223            _ if !op_start_signals.is_empty() => {
 224                while operations < max_operations && rng.lock().gen_bool(0.7) {
 225                    op_start_signals
 226                        .choose(&mut *rng.lock())
 227                        .unwrap()
 228                        .unbounded_send(())
 229                        .unwrap();
 230                    operations += 1;
 231                }
 232
 233                if rng.lock().gen_bool(0.8) {
 234                    deterministic.run_until_parked();
 235                }
 236            }
 237            _ => {}
 238        }
 239    }
 240
 241    drop(op_start_signals);
 242    deterministic.start_waiting();
 243    let clients = futures::future::join_all(clients).await;
 244    deterministic.finish_waiting();
 245    deterministic.run_until_parked();
 246
 247    for (client, client_cx) in &clients {
 248        for guest_project in &client.remote_projects {
 249            guest_project.read_with(client_cx, |guest_project, cx| {
 250                let host_project = clients.iter().find_map(|(client, cx)| {
 251                    let project = client.local_projects.iter().find(|host_project| {
 252                        host_project.read_with(cx, |host_project, _| {
 253                            host_project.remote_id() == guest_project.remote_id()
 254                        })
 255                    })?;
 256                    Some((project, cx))
 257                });
 258
 259                if !guest_project.is_read_only() {
 260                    if let Some((host_project, host_cx)) = host_project {
 261                        let host_worktree_snapshots =
 262                            host_project.read_with(host_cx, |host_project, cx| {
 263                                host_project
 264                                    .worktrees(cx)
 265                                    .map(|worktree| {
 266                                        let worktree = worktree.read(cx);
 267                                        (worktree.id(), worktree.snapshot())
 268                                    })
 269                                    .collect::<BTreeMap<_, _>>()
 270                            });
 271                        let guest_worktree_snapshots = guest_project
 272                            .worktrees(cx)
 273                            .map(|worktree| {
 274                                let worktree = worktree.read(cx);
 275                                (worktree.id(), worktree.snapshot())
 276                            })
 277                            .collect::<BTreeMap<_, _>>();
 278
 279                        assert_eq!(
 280                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 281                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 282                            "{} has different worktrees than the host",
 283                            client.username
 284                        );
 285
 286                        for (id, host_snapshot) in &host_worktree_snapshots {
 287                            let guest_snapshot = &guest_worktree_snapshots[id];
 288                            assert_eq!(
 289                                guest_snapshot.root_name(),
 290                                host_snapshot.root_name(),
 291                                "{} has different root name than the host for worktree {}",
 292                                client.username,
 293                                id
 294                            );
 295                            assert_eq!(
 296                                guest_snapshot.abs_path(),
 297                                host_snapshot.abs_path(),
 298                                "{} has different abs path than the host for worktree {}",
 299                                client.username,
 300                                id
 301                            );
 302                            assert_eq!(
 303                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 304                                host_snapshot.entries(false).collect::<Vec<_>>(),
 305                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 306                                client.username,
 307                                id,
 308                                host_snapshot.abs_path(),
 309                                host_project.read_with(host_cx, |project, _| project.remote_id())
 310                            );
 311                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 312                        }
 313                    }
 314                }
 315
 316                guest_project.check_invariants(cx);
 317            });
 318        }
 319
 320        for (guest_project, guest_buffers) in &client.buffers {
 321            let project_id = if guest_project.read_with(client_cx, |project, _| {
 322                project.is_local() || project.is_read_only()
 323            }) {
 324                continue;
 325            } else {
 326                guest_project
 327                    .read_with(client_cx, |project, _| project.remote_id())
 328                    .unwrap()
 329            };
 330            let guest_user_id = client.user_id().unwrap();
 331
 332            let host_project = clients.iter().find_map(|(client, cx)| {
 333                let project = client.local_projects.iter().find(|host_project| {
 334                    host_project.read_with(cx, |host_project, _| {
 335                        host_project.remote_id() == Some(project_id)
 336                    })
 337                })?;
 338                Some((client.user_id().unwrap(), project, cx))
 339            });
 340
 341            let (host_user_id, host_project, host_cx) =
 342                if let Some((host_user_id, host_project, host_cx)) = host_project {
 343                    (host_user_id, host_project, host_cx)
 344                } else {
 345                    continue;
 346                };
 347
 348            for guest_buffer in guest_buffers {
 349                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 350                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 351                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 352                        panic!(
 353                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 354                            client.username,
 355                            client.peer_id(),
 356                            buffer_id
 357                        )
 358                    })
 359                });
 360                let path = host_buffer
 361                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 362
 363                assert_eq!(
 364                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 365                    0,
 366                    "{}, buffer {}, path {:?} has deferred operations",
 367                    client.username,
 368                    buffer_id,
 369                    path,
 370                );
 371                assert_eq!(
 372                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 373                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 374                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 375                    client.username,
 376                    buffer_id,
 377                    path
 378                );
 379
 380                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 381                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 382                match (host_file, guest_file) {
 383                    (Some(host_file), Some(guest_file)) => {
 384                        assert_eq!(guest_file.path(), host_file.path());
 385                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 386                        assert_eq!(
 387                            guest_file.mtime(),
 388                            host_file.mtime(),
 389                            "guest {} mtime does not match host {} for path {:?} in project {}",
 390                            guest_user_id,
 391                            host_user_id,
 392                            guest_file.path(),
 393                            project_id,
 394                        );
 395                    }
 396                    (None, None) => {}
 397                    (None, _) => panic!("host's file is None, guest's isn't"),
 398                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 399                }
 400
 401                let host_diff_base =
 402                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 403                let guest_diff_base = guest_buffer
 404                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 405                assert_eq!(guest_diff_base, host_diff_base);
 406
 407                let host_is_dirty = host_buffer.read_with(host_cx, |b, _| b.is_dirty());
 408                let guest_is_dirty = host_buffer.read_with(host_cx, |b, _| b.is_dirty());
 409                assert_eq!(guest_is_dirty, host_is_dirty);
 410
 411                let host_has_conflict = host_buffer.read_with(host_cx, |b, _| b.has_conflict());
 412                let guest_has_conflict = host_buffer.read_with(host_cx, |b, _| b.has_conflict());
 413                assert_eq!(guest_has_conflict, host_has_conflict);
 414            }
 415        }
 416    }
 417
 418    for (client, mut cx) in clients {
 419        cx.update(|cx| {
 420            cx.clear_globals();
 421            cx.set_global(Settings::test(cx));
 422            drop(client);
 423        });
 424    }
 425}
 426
 427async fn simulate_client(
 428    mut client: TestClient,
 429    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 430    can_hang_up: bool,
 431    rng: Arc<Mutex<StdRng>>,
 432    mut cx: TestAppContext,
 433) -> (TestClient, TestAppContext) {
 434    // Setup language server
 435    let mut language = Language::new(
 436        LanguageConfig {
 437            name: "Rust".into(),
 438            path_suffixes: vec!["rs".to_string()],
 439            ..Default::default()
 440        },
 441        None,
 442    );
 443    let _fake_language_servers = language
 444        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 445            name: "the-fake-language-server",
 446            capabilities: lsp::LanguageServer::full_capabilities(),
 447            initializer: Some(Box::new({
 448                let rng = rng.clone();
 449                let fs = client.fs.clone();
 450                move |fake_server: &mut FakeLanguageServer| {
 451                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 452                        |_, _| async move {
 453                            Ok(Some(lsp::CompletionResponse::Array(vec![
 454                                lsp::CompletionItem {
 455                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 456                                        range: lsp::Range::new(
 457                                            lsp::Position::new(0, 0),
 458                                            lsp::Position::new(0, 0),
 459                                        ),
 460                                        new_text: "the-new-text".to_string(),
 461                                    })),
 462                                    ..Default::default()
 463                                },
 464                            ])))
 465                        },
 466                    );
 467
 468                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 469                        |_, _| async move {
 470                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 471                                lsp::CodeAction {
 472                                    title: "the-code-action".to_string(),
 473                                    ..Default::default()
 474                                },
 475                            )]))
 476                        },
 477                    );
 478
 479                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 480                        |params, _| async move {
 481                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 482                                params.position,
 483                                params.position,
 484                            ))))
 485                        },
 486                    );
 487
 488                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 489                        let fs = fs.clone();
 490                        let rng = rng.clone();
 491                        move |_, _| {
 492                            let fs = fs.clone();
 493                            let rng = rng.clone();
 494                            async move {
 495                                let files = fs.files().await;
 496                                let mut rng = rng.lock();
 497                                let count = rng.gen_range::<usize, _>(1..3);
 498                                let files = (0..count)
 499                                    .map(|_| files.choose(&mut *rng).unwrap())
 500                                    .collect::<Vec<_>>();
 501                                log::info!("LSP: Returning definitions in files {:?}", &files);
 502                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 503                                    files
 504                                        .into_iter()
 505                                        .map(|file| lsp::Location {
 506                                            uri: lsp::Url::from_file_path(file).unwrap(),
 507                                            range: Default::default(),
 508                                        })
 509                                        .collect(),
 510                                )))
 511                            }
 512                        }
 513                    });
 514
 515                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 516                        let rng = rng.clone();
 517                        move |_, _| {
 518                            let mut highlights = Vec::new();
 519                            let highlight_count = rng.lock().gen_range(1..=5);
 520                            for _ in 0..highlight_count {
 521                                let start_row = rng.lock().gen_range(0..100);
 522                                let start_column = rng.lock().gen_range(0..100);
 523                                let start = PointUtf16::new(start_row, start_column);
 524                                let end_row = rng.lock().gen_range(0..100);
 525                                let end_column = rng.lock().gen_range(0..100);
 526                                let end = PointUtf16::new(end_row, end_column);
 527                                let range = if start > end { end..start } else { start..end };
 528                                highlights.push(lsp::DocumentHighlight {
 529                                    range: range_to_lsp(range.clone()),
 530                                    kind: Some(lsp::DocumentHighlightKind::READ),
 531                                });
 532                            }
 533                            highlights.sort_unstable_by_key(|highlight| {
 534                                (highlight.range.start, highlight.range.end)
 535                            });
 536                            async move { Ok(Some(highlights)) }
 537                        }
 538                    });
 539                }
 540            })),
 541            ..Default::default()
 542        }))
 543        .await;
 544    client.language_registry.add(Arc::new(language));
 545
 546    while op_start_signal.next().await.is_some() {
 547        if let Err(error) =
 548            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 549        {
 550            log::error!("{} error: {:?}", client.username, error);
 551        }
 552
 553        cx.background().simulate_random_delay().await;
 554    }
 555    log::info!("{}: done", client.username);
 556
 557    (client, cx)
 558}
 559
 560async fn randomly_mutate_client(
 561    client: &mut TestClient,
 562    can_hang_up: bool,
 563    rng: Arc<Mutex<StdRng>>,
 564    cx: &mut TestAppContext,
 565) -> Result<()> {
 566    let choice = rng.lock().gen_range(0..100);
 567    match choice {
 568        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 569        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 570        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 571            randomly_mutate_worktrees(client, &rng, cx).await?;
 572        }
 573        60..=74 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 574            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 575        }
 576        75..=84 => randomly_mutate_git(client, &rng).await,
 577        _ => randomly_mutate_fs(client, &rng).await,
 578    }
 579
 580    Ok(())
 581}
 582
 583async fn randomly_mutate_active_call(
 584    client: &mut TestClient,
 585    can_hang_up: bool,
 586    rng: &Mutex<StdRng>,
 587    cx: &mut TestAppContext,
 588) -> Result<()> {
 589    let active_call = cx.read(ActiveCall::global);
 590    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 591        if rng.lock().gen_bool(0.7) {
 592            log::info!("{}: accepting incoming call", client.username);
 593            active_call
 594                .update(cx, |call, cx| call.accept_incoming(cx))
 595                .await?;
 596        } else {
 597            log::info!("{}: declining incoming call", client.username);
 598            active_call.update(cx, |call, _| call.decline_incoming())?;
 599        }
 600    } else {
 601        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 602            user_store
 603                .contacts()
 604                .iter()
 605                .filter(|contact| contact.online && !contact.busy)
 606                .cloned()
 607                .collect::<Vec<_>>()
 608        });
 609
 610        let distribution = rng.lock().gen_range(0..100);
 611        match distribution {
 612            0..=29 if !available_contacts.is_empty() => {
 613                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 614                log::info!(
 615                    "{}: inviting {}",
 616                    client.username,
 617                    contact.user.github_login
 618                );
 619                active_call
 620                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 621                    .await?;
 622            }
 623            30..=39
 624                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 625            {
 626                log::info!("{}: hanging up", client.username);
 627                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 628            }
 629            _ => {}
 630        }
 631    }
 632
 633    Ok(())
 634}
 635
 636async fn randomly_mutate_git(client: &mut TestClient, rng: &Mutex<StdRng>) {
 637    let directories = client.fs.directories().await;
 638    let mut dir_path = directories.choose(&mut *rng.lock()).unwrap().clone();
 639    if dir_path.file_name() == Some(OsStr::new(".git")) {
 640        dir_path.pop();
 641    }
 642    let mut git_dir_path = dir_path.clone();
 643    git_dir_path.push(".git");
 644
 645    if !directories.contains(&git_dir_path) {
 646        log::info!(
 647            "{}: creating git directory at {:?}",
 648            client.username,
 649            git_dir_path
 650        );
 651        client.fs.create_dir(&git_dir_path).await.unwrap();
 652    }
 653
 654    let mut child_file_paths = child_file_paths(client, &dir_path).await;
 655    let count = rng.lock().gen_range(0..=child_file_paths.len());
 656    child_file_paths.shuffle(&mut *rng.lock());
 657    child_file_paths.truncate(count);
 658
 659    let mut new_index = Vec::new();
 660    for abs_child_file_path in &child_file_paths {
 661        let child_file_path = abs_child_file_path.strip_prefix(&dir_path).unwrap();
 662        let new_base = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 663        new_index.push((child_file_path, new_base));
 664    }
 665    log::info!(
 666        "{}: updating Git index at {:?}: {:#?}",
 667        client.username,
 668        git_dir_path,
 669        new_index
 670    );
 671    client
 672        .fs
 673        .set_index_for_repo(&git_dir_path, &new_index)
 674        .await;
 675}
 676
 677async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 678    let parent_dir_path = client
 679        .fs
 680        .directories()
 681        .await
 682        .choose(&mut *rng.lock())
 683        .unwrap()
 684        .clone();
 685
 686    let is_dir = rng.lock().gen::<bool>();
 687    if is_dir {
 688        let mut dir_path = parent_dir_path.clone();
 689        dir_path.push(gen_file_name(rng));
 690        log::info!("{}: creating local dir at {:?}", client.username, dir_path);
 691        client.fs.create_dir(&dir_path).await.unwrap();
 692    } else {
 693        let child_file_paths = child_file_paths(client, &parent_dir_path).await;
 694        let create_new_file = child_file_paths.is_empty() || rng.lock().gen();
 695        let text = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 696        if create_new_file {
 697            let mut file_path = parent_dir_path.clone();
 698            file_path.push(gen_file_name(rng));
 699            file_path.set_extension("rs");
 700            log::info!(
 701                "{}: creating local file at {:?}",
 702                client.username,
 703                file_path
 704            );
 705            client
 706                .fs
 707                .create_file(&file_path, Default::default())
 708                .await
 709                .unwrap();
 710            log::info!(
 711                "{}: setting local file {:?} text to {:?}",
 712                client.username,
 713                file_path,
 714                text
 715            );
 716            client
 717                .fs
 718                .save(&file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 719                .await
 720                .unwrap();
 721        } else {
 722            let file_path = child_file_paths.choose(&mut *rng.lock()).unwrap();
 723            log::info!(
 724                "{}: setting local file {:?} text to {:?}",
 725                client.username,
 726                file_path,
 727                text
 728            );
 729            client
 730                .fs
 731                .save(file_path, &Rope::from(text.as_str()), fs::LineEnding::Unix)
 732                .await
 733                .unwrap();
 734        }
 735    }
 736}
 737
 738async fn randomly_mutate_projects(
 739    client: &mut TestClient,
 740    rng: &Mutex<StdRng>,
 741    cx: &mut TestAppContext,
 742) -> Result<()> {
 743    let active_call = cx.read(ActiveCall::global);
 744    let remote_projects =
 745        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 746            room.read_with(cx, |room, _| {
 747                room.remote_participants()
 748                    .values()
 749                    .flat_map(|participant| participant.projects.clone())
 750                    .collect::<Vec<_>>()
 751            })
 752        } else {
 753            Default::default()
 754        };
 755
 756    let project = if remote_projects.is_empty() || rng.lock().gen() {
 757        if client.local_projects.is_empty() || rng.lock().gen() {
 758            let paths = client.fs.paths().await;
 759            let local_project = if paths.is_empty() || rng.lock().gen() {
 760                let root_path = client.create_new_root_dir();
 761                client.fs.create_dir(&root_path).await.unwrap();
 762                client
 763                    .fs
 764                    .create_file(&root_path.join("main.rs"), Default::default())
 765                    .await
 766                    .unwrap();
 767                log::info!(
 768                    "{}: opening local project at {:?}",
 769                    client.username,
 770                    root_path
 771                );
 772                client.build_local_project(root_path, cx).await.0
 773            } else {
 774                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 775                log::info!(
 776                    "{}: opening local project at {:?}",
 777                    client.username,
 778                    root_path
 779                );
 780                client.build_local_project(root_path, cx).await.0
 781            };
 782            client.local_projects.push(local_project.clone());
 783            local_project
 784        } else {
 785            client
 786                .local_projects
 787                .choose(&mut *rng.lock())
 788                .unwrap()
 789                .clone()
 790        }
 791    } else {
 792        if client.remote_projects.is_empty() || rng.lock().gen() {
 793            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 794            let remote_project = if let Some(project) =
 795                client.remote_projects.iter().find(|project| {
 796                    project.read_with(cx, |project, _| {
 797                        project.remote_id() == Some(remote_project_id)
 798                    })
 799                }) {
 800                project.clone()
 801            } else {
 802                log::info!(
 803                    "{}: opening remote project {}",
 804                    client.username,
 805                    remote_project_id
 806                );
 807                let call = cx.read(ActiveCall::global);
 808                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 809                let remote_project = room
 810                    .update(cx, |room, cx| {
 811                        room.join_project(
 812                            remote_project_id,
 813                            client.language_registry.clone(),
 814                            FakeFs::new(cx.background().clone()),
 815                            cx,
 816                        )
 817                    })
 818                    .await?;
 819                client.remote_projects.push(remote_project.clone());
 820                remote_project
 821            };
 822
 823            remote_project
 824        } else {
 825            client
 826                .remote_projects
 827                .choose(&mut *rng.lock())
 828                .unwrap()
 829                .clone()
 830        }
 831    };
 832
 833    if active_call.read_with(cx, |call, _| call.room().is_some())
 834        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 835    {
 836        match active_call
 837            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 838            .await
 839        {
 840            Ok(project_id) => {
 841                log::info!("{}: shared project with id {}", client.username, project_id);
 842            }
 843            Err(error) => {
 844                log::error!("{}: error sharing project, {:?}", client.username, error);
 845            }
 846        }
 847    }
 848
 849    let choice = rng.lock().gen_range(0..100);
 850    match choice {
 851        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 852            let paths = client.fs.paths().await;
 853            let path = paths.choose(&mut *rng.lock()).unwrap();
 854            log::info!(
 855                "{}: finding/creating local worktree for path {:?}",
 856                client.username,
 857                path
 858            );
 859            project
 860                .update(cx, |project, cx| {
 861                    project.find_or_create_local_worktree(&path, true, cx)
 862                })
 863                .await
 864                .unwrap();
 865        }
 866        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 867            log::info!(
 868                "{}: dropping remote project {}",
 869                client.username,
 870                project.read_with(cx, |project, _| project.remote_id().unwrap())
 871            );
 872
 873            cx.update(|_| {
 874                client
 875                    .remote_projects
 876                    .retain(|remote_project| *remote_project != project);
 877                client.buffers.remove(&project);
 878                drop(project);
 879            });
 880        }
 881        _ => {}
 882    }
 883
 884    Ok(())
 885}
 886
 887async fn randomly_mutate_worktrees(
 888    client: &mut TestClient,
 889    rng: &Mutex<StdRng>,
 890    cx: &mut TestAppContext,
 891) -> Result<()> {
 892    let project = choose_random_project(client, rng).unwrap();
 893    let Some(worktree) = project.read_with(cx, |project, cx| {
 894        project
 895            .worktrees(cx)
 896            .filter(|worktree| {
 897                let worktree = worktree.read(cx);
 898                worktree.is_visible()
 899                    && worktree.entries(false).any(|e| e.is_file())
 900                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 901            })
 902            .choose(&mut *rng.lock())
 903    }) else {
 904        return Ok(())
 905    };
 906
 907    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 908        (worktree.id(), worktree.root_name().to_string())
 909    });
 910
 911    let is_dir = rng.lock().gen::<bool>();
 912    let mut new_path = PathBuf::new();
 913    new_path.push(gen_file_name(rng));
 914    if !is_dir {
 915        new_path.set_extension("rs");
 916    }
 917    log::info!(
 918        "{}: creating {:?} in worktree {} ({})",
 919        client.username,
 920        new_path,
 921        worktree_id,
 922        worktree_root_name,
 923    );
 924    project
 925        .update(cx, |project, cx| {
 926            project.create_entry((worktree_id, new_path), is_dir, cx)
 927        })
 928        .unwrap()
 929        .await?;
 930    Ok(())
 931}
 932
 933async fn randomly_query_and_mutate_buffers(
 934    client: &mut TestClient,
 935    rng: &Mutex<StdRng>,
 936    cx: &mut TestAppContext,
 937) -> Result<()> {
 938    let project = choose_random_project(client, rng).unwrap();
 939    let buffers = client.buffers.entry(project.clone()).or_default();
 940    let buffer = if buffers.is_empty() || rng.lock().gen() {
 941        let Some(worktree) = project.read_with(cx, |project, cx| {
 942            project
 943                .worktrees(cx)
 944                .filter(|worktree| {
 945                    let worktree = worktree.read(cx);
 946                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 947                })
 948                .choose(&mut *rng.lock())
 949        }) else {
 950            return Ok(());
 951        };
 952
 953        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 954            let entry = worktree
 955                .entries(false)
 956                .filter(|e| e.is_file())
 957                .choose(&mut *rng.lock())
 958                .unwrap();
 959            (
 960                worktree.root_name().to_string(),
 961                (worktree.id(), entry.path.clone()),
 962            )
 963        });
 964        log::info!(
 965            "{}: opening path {:?} in worktree {} ({})",
 966            client.username,
 967            project_path.1,
 968            project_path.0,
 969            worktree_root_name,
 970        );
 971        let buffer = project
 972            .update(cx, |project, cx| {
 973                project.open_buffer(project_path.clone(), cx)
 974            })
 975            .await?;
 976        log::info!(
 977            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 978            client.username,
 979            project_path.1,
 980            project_path.0,
 981            worktree_root_name,
 982            buffer.read_with(cx, |buffer, _| buffer.remote_id())
 983        );
 984        buffers.insert(buffer.clone());
 985        buffer
 986    } else {
 987        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
 988    };
 989
 990    let choice = rng.lock().gen_range(0..100);
 991    match choice {
 992        0..=9 => {
 993            cx.update(|cx| {
 994                log::info!(
 995                    "{}: dropping buffer {:?}",
 996                    client.username,
 997                    buffer.read(cx).file().unwrap().full_path(cx)
 998                );
 999                buffers.remove(&buffer);
1000                drop(buffer);
1001            });
1002        }
1003        10..=19 => {
1004            let completions = project.update(cx, |project, cx| {
1005                log::info!(
1006                    "{}: requesting completions for buffer {} ({:?})",
1007                    client.username,
1008                    buffer.read(cx).remote_id(),
1009                    buffer.read(cx).file().unwrap().full_path(cx)
1010                );
1011                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1012                project.completions(&buffer, offset, cx)
1013            });
1014            let completions = cx.background().spawn(async move {
1015                completions
1016                    .await
1017                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
1018            });
1019            if rng.lock().gen_bool(0.3) {
1020                log::info!("{}: detaching completions request", client.username);
1021                cx.update(|cx| completions.detach_and_log_err(cx));
1022            } else {
1023                completions.await?;
1024            }
1025        }
1026        20..=29 => {
1027            let code_actions = project.update(cx, |project, cx| {
1028                log::info!(
1029                    "{}: requesting code actions for buffer {} ({:?})",
1030                    client.username,
1031                    buffer.read(cx).remote_id(),
1032                    buffer.read(cx).file().unwrap().full_path(cx)
1033                );
1034                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
1035                project.code_actions(&buffer, range, cx)
1036            });
1037            let code_actions = cx.background().spawn(async move {
1038                code_actions
1039                    .await
1040                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
1041            });
1042            if rng.lock().gen_bool(0.3) {
1043                log::info!("{}: detaching code actions request", client.username);
1044                cx.update(|cx| code_actions.detach_and_log_err(cx));
1045            } else {
1046                code_actions.await?;
1047            }
1048        }
1049        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1050            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1051                log::info!(
1052                    "{}: saving buffer {} ({:?})",
1053                    client.username,
1054                    buffer.remote_id(),
1055                    buffer.file().unwrap().full_path(cx)
1056                );
1057                (buffer.version(), buffer.save(cx))
1058            });
1059            let save = cx.background().spawn(async move {
1060                let (saved_version, _, _) = save
1061                    .await
1062                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1063                assert!(saved_version.observed_all(&requested_version));
1064                Ok::<_, anyhow::Error>(())
1065            });
1066            if rng.lock().gen_bool(0.3) {
1067                log::info!("{}: detaching save request", client.username);
1068                cx.update(|cx| save.detach_and_log_err(cx));
1069            } else {
1070                save.await?;
1071            }
1072        }
1073        40..=44 => {
1074            let prepare_rename = project.update(cx, |project, cx| {
1075                log::info!(
1076                    "{}: preparing rename for buffer {} ({:?})",
1077                    client.username,
1078                    buffer.read(cx).remote_id(),
1079                    buffer.read(cx).file().unwrap().full_path(cx)
1080                );
1081                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1082                project.prepare_rename(buffer, offset, cx)
1083            });
1084            let prepare_rename = cx.background().spawn(async move {
1085                prepare_rename
1086                    .await
1087                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1088            });
1089            if rng.lock().gen_bool(0.3) {
1090                log::info!("{}: detaching prepare rename request", client.username);
1091                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1092            } else {
1093                prepare_rename.await?;
1094            }
1095        }
1096        45..=49 => {
1097            let definitions = project.update(cx, |project, cx| {
1098                log::info!(
1099                    "{}: requesting definitions for buffer {} ({:?})",
1100                    client.username,
1101                    buffer.read(cx).remote_id(),
1102                    buffer.read(cx).file().unwrap().full_path(cx)
1103                );
1104                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1105                project.definition(&buffer, offset, cx)
1106            });
1107            let definitions = cx.background().spawn(async move {
1108                definitions
1109                    .await
1110                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1111            });
1112            if rng.lock().gen_bool(0.3) {
1113                log::info!("{}: detaching definitions request", client.username);
1114                cx.update(|cx| definitions.detach_and_log_err(cx));
1115            } else {
1116                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1117            }
1118        }
1119        50..=54 => {
1120            let highlights = project.update(cx, |project, cx| {
1121                log::info!(
1122                    "{}: requesting highlights for buffer {} ({:?})",
1123                    client.username,
1124                    buffer.read(cx).remote_id(),
1125                    buffer.read(cx).file().unwrap().full_path(cx)
1126                );
1127                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1128                project.document_highlights(&buffer, offset, cx)
1129            });
1130            let highlights = cx.background().spawn(async move {
1131                highlights
1132                    .await
1133                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1134            });
1135            if rng.lock().gen_bool(0.3) {
1136                log::info!("{}: detaching highlights request", client.username);
1137                cx.update(|cx| highlights.detach_and_log_err(cx));
1138            } else {
1139                highlights.await?;
1140            }
1141        }
1142        55..=59 => {
1143            let search = project.update(cx, |project, cx| {
1144                let query = rng.lock().gen_range('a'..='z');
1145                log::info!("{}: project-wide search {:?}", client.username, query);
1146                project.search(SearchQuery::text(query, false, false), cx)
1147            });
1148            let search = cx.background().spawn(async move {
1149                search
1150                    .await
1151                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1152            });
1153            if rng.lock().gen_bool(0.3) {
1154                log::info!("{}: detaching search request", client.username);
1155                cx.update(|cx| search.detach_and_log_err(cx));
1156            } else {
1157                buffers.extend(search.await?.into_keys());
1158            }
1159        }
1160        _ => {
1161            buffer.update(cx, |buffer, cx| {
1162                log::info!(
1163                    "{}: updating buffer {} ({:?})",
1164                    client.username,
1165                    buffer.remote_id(),
1166                    buffer.file().unwrap().full_path(cx)
1167                );
1168                if rng.lock().gen_bool(0.7) {
1169                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1170                } else {
1171                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1172                }
1173            });
1174        }
1175    }
1176
1177    Ok(())
1178}
1179
1180fn choose_random_project(
1181    client: &mut TestClient,
1182    rng: &Mutex<StdRng>,
1183) -> Option<ModelHandle<Project>> {
1184    client
1185        .local_projects
1186        .iter()
1187        .chain(&client.remote_projects)
1188        .choose(&mut *rng.lock())
1189        .cloned()
1190}
1191
1192fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1193    let mut name = String::new();
1194    for _ in 0..10 {
1195        let letter = rng.lock().gen_range('a'..='z');
1196        name.push(letter);
1197    }
1198    name
1199}
1200
1201async fn child_file_paths(client: &TestClient, dir_path: &Path) -> Vec<PathBuf> {
1202    let mut child_paths = client.fs.read_dir(dir_path).await.unwrap();
1203    let mut child_file_paths = Vec::new();
1204    while let Some(child_path) = child_paths.next().await {
1205        let child_path = child_path.unwrap();
1206        if client.fs.is_file(&child_path).await {
1207            child_file_paths.push(child_path);
1208        }
1209    }
1210    child_file_paths
1211}