randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::{
  18    distributions::{Alphanumeric, DistString},
  19    prelude::*,
  20};
  21use std::{env, ffi::OsStr, path::PathBuf, sync::Arc};
  22
  23#[gpui::test(iterations = 100)]
  24async fn test_random_collaboration(
  25    cx: &mut TestAppContext,
  26    deterministic: Arc<Deterministic>,
  27    rng: StdRng,
  28) {
  29    deterministic.forbid_parking();
  30    let rng = Arc::new(Mutex::new(rng));
  31
  32    let max_peers = env::var("MAX_PEERS")
  33        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  34        .unwrap_or(5);
  35
  36    let max_operations = env::var("OPERATIONS")
  37        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  38        .unwrap_or(10);
  39
  40    let mut server = TestServer::start(&deterministic).await;
  41    let db = server.app_state.db.clone();
  42
  43    let mut available_users = Vec::new();
  44    for ix in 0..max_peers {
  45        let username = format!("user-{}", ix + 1);
  46        let user_id = db
  47            .create_user(
  48                &format!("{username}@example.com"),
  49                false,
  50                NewUserParams {
  51                    github_login: username.clone(),
  52                    github_user_id: (ix + 1) as i32,
  53                    invite_count: 0,
  54                },
  55            )
  56            .await
  57            .unwrap()
  58            .user_id;
  59        available_users.push((user_id, username));
  60    }
  61
  62    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  63        for (user_id_b, _) in &available_users[ix + 1..] {
  64            server
  65                .app_state
  66                .db
  67                .send_contact_request(*user_id_a, *user_id_b)
  68                .await
  69                .unwrap();
  70            server
  71                .app_state
  72                .db
  73                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  74                .await
  75                .unwrap();
  76        }
  77    }
  78
  79    let mut clients = Vec::new();
  80    let mut user_ids = Vec::new();
  81    let mut op_start_signals = Vec::new();
  82    let mut next_entity_id = 100000;
  83    let allow_server_restarts = rng.lock().gen_bool(0.7);
  84    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  85    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  86
  87    let mut operations = 0;
  88    while operations < max_operations {
  89        let distribution = rng.lock().gen_range(0..100);
  90        match distribution {
  91            0..=19 if !available_users.is_empty() => {
  92                let client_ix = rng.lock().gen_range(0..available_users.len());
  93                let (_, username) = available_users.remove(client_ix);
  94                log::info!("Adding new connection for {}", username);
  95                next_entity_id += 100000;
  96                let mut client_cx = TestAppContext::new(
  97                    cx.foreground_platform(),
  98                    cx.platform(),
  99                    deterministic.build_foreground(next_entity_id),
 100                    deterministic.build_background(),
 101                    cx.font_cache(),
 102                    cx.leak_detector(),
 103                    next_entity_id,
 104                    cx.function_name.clone(),
 105                );
 106
 107                let op_start_signal = futures::channel::mpsc::unbounded();
 108                let client = server.create_client(&mut client_cx, &username).await;
 109                user_ids.push(client.current_user_id(&client_cx));
 110                op_start_signals.push(op_start_signal.0);
 111                clients.push(client_cx.foreground().spawn(simulate_client(
 112                    client,
 113                    op_start_signal.1,
 114                    allow_client_disconnection,
 115                    rng.clone(),
 116                    client_cx,
 117                )));
 118
 119                log::info!("Added connection for {}", username);
 120                operations += 1;
 121            }
 122
 123            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 124                let client_ix = rng.lock().gen_range(1..clients.len());
 125                log::info!(
 126                    "Simulating full disconnection of user {}",
 127                    user_ids[client_ix]
 128                );
 129                let removed_user_id = user_ids.remove(client_ix);
 130                let user_connection_ids = server
 131                    .connection_pool
 132                    .lock()
 133                    .user_connection_ids(removed_user_id)
 134                    .collect::<Vec<_>>();
 135                assert_eq!(user_connection_ids.len(), 1);
 136                let removed_peer_id = user_connection_ids[0].into();
 137                let client = clients.remove(client_ix);
 138                op_start_signals.remove(client_ix);
 139                server.forbid_connections();
 140                server.disconnect_client(removed_peer_id);
 141                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 142                deterministic.start_waiting();
 143                log::info!("Waiting for user {} to exit...", removed_user_id);
 144                let (client, mut client_cx) = client.await;
 145                deterministic.finish_waiting();
 146                server.allow_connections();
 147
 148                for project in &client.remote_projects {
 149                    project.read_with(&client_cx, |project, _| {
 150                        assert!(
 151                            project.is_read_only(),
 152                            "project {:?} should be read only",
 153                            project.remote_id()
 154                        )
 155                    });
 156                }
 157                for user_id in &user_ids {
 158                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 159                    let pool = server.connection_pool.lock();
 160                    for contact in contacts {
 161                        if let db::Contact::Accepted { user_id, .. } = contact {
 162                            if pool.is_user_online(user_id) {
 163                                assert_ne!(
 164                                    user_id, removed_user_id,
 165                                    "removed client is still a contact of another peer"
 166                                );
 167                            }
 168                        }
 169                    }
 170                }
 171
 172                log::info!("{} removed", client.username);
 173                available_users.push((removed_user_id, client.username.clone()));
 174                client_cx.update(|cx| {
 175                    cx.clear_globals();
 176                    drop(client);
 177                });
 178
 179                operations += 1;
 180            }
 181
 182            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 183                let client_ix = rng.lock().gen_range(1..clients.len());
 184                let user_id = user_ids[client_ix];
 185                log::info!("Simulating temporary disconnection of user {}", user_id);
 186                let user_connection_ids = server
 187                    .connection_pool
 188                    .lock()
 189                    .user_connection_ids(user_id)
 190                    .collect::<Vec<_>>();
 191                assert_eq!(user_connection_ids.len(), 1);
 192                let peer_id = user_connection_ids[0].into();
 193                server.disconnect_client(peer_id);
 194                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 195                operations += 1;
 196            }
 197
 198            30..=34 if allow_server_restarts => {
 199                log::info!("Simulating server restart");
 200                server.reset().await;
 201                deterministic.advance_clock(RECEIVE_TIMEOUT);
 202                server.start().await.unwrap();
 203                deterministic.advance_clock(CLEANUP_TIMEOUT);
 204                let environment = &server.app_state.config.zed_environment;
 205                let stale_room_ids = server
 206                    .app_state
 207                    .db
 208                    .stale_room_ids(environment, server.id())
 209                    .await
 210                    .unwrap();
 211                assert_eq!(stale_room_ids, vec![]);
 212            }
 213
 214            _ if !op_start_signals.is_empty() => {
 215                while operations < max_operations && rng.lock().gen_bool(0.7) {
 216                    op_start_signals
 217                        .choose(&mut *rng.lock())
 218                        .unwrap()
 219                        .unbounded_send(())
 220                        .unwrap();
 221                    operations += 1;
 222                }
 223
 224                if rng.lock().gen_bool(0.8) {
 225                    deterministic.run_until_parked();
 226                }
 227            }
 228            _ => {}
 229        }
 230    }
 231
 232    drop(op_start_signals);
 233    deterministic.start_waiting();
 234    let clients = futures::future::join_all(clients).await;
 235    deterministic.finish_waiting();
 236    deterministic.run_until_parked();
 237
 238    for (client, client_cx) in &clients {
 239        for guest_project in &client.remote_projects {
 240            guest_project.read_with(client_cx, |guest_project, cx| {
 241                let host_project = clients.iter().find_map(|(client, cx)| {
 242                    let project = client.local_projects.iter().find(|host_project| {
 243                        host_project.read_with(cx, |host_project, _| {
 244                            host_project.remote_id() == guest_project.remote_id()
 245                        })
 246                    })?;
 247                    Some((project, cx))
 248                });
 249
 250                if !guest_project.is_read_only() {
 251                    if let Some((host_project, host_cx)) = host_project {
 252                        let host_worktree_snapshots =
 253                            host_project.read_with(host_cx, |host_project, cx| {
 254                                host_project
 255                                    .worktrees(cx)
 256                                    .map(|worktree| {
 257                                        let worktree = worktree.read(cx);
 258                                        (worktree.id(), worktree.snapshot())
 259                                    })
 260                                    .collect::<BTreeMap<_, _>>()
 261                            });
 262                        let guest_worktree_snapshots = guest_project
 263                            .worktrees(cx)
 264                            .map(|worktree| {
 265                                let worktree = worktree.read(cx);
 266                                (worktree.id(), worktree.snapshot())
 267                            })
 268                            .collect::<BTreeMap<_, _>>();
 269
 270                        assert_eq!(
 271                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 272                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 273                            "{} has different worktrees than the host",
 274                            client.username
 275                        );
 276
 277                        for (id, host_snapshot) in &host_worktree_snapshots {
 278                            let guest_snapshot = &guest_worktree_snapshots[id];
 279                            assert_eq!(
 280                                guest_snapshot.root_name(),
 281                                host_snapshot.root_name(),
 282                                "{} has different root name than the host for worktree {}",
 283                                client.username,
 284                                id
 285                            );
 286                            assert_eq!(
 287                                guest_snapshot.abs_path(),
 288                                host_snapshot.abs_path(),
 289                                "{} has different abs path than the host for worktree {}",
 290                                client.username,
 291                                id
 292                            );
 293                            assert_eq!(
 294                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 295                                host_snapshot.entries(false).collect::<Vec<_>>(),
 296                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 297                                client.username,
 298                                id,
 299                                host_snapshot.abs_path(),
 300                                host_project.read_with(host_cx, |project, _| project.remote_id())
 301                            );
 302                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 303                        }
 304                    }
 305                }
 306
 307                guest_project.check_invariants(cx);
 308            });
 309        }
 310
 311        for (guest_project, guest_buffers) in &client.buffers {
 312            let project_id = if guest_project.read_with(client_cx, |project, _| {
 313                project.is_local() || project.is_read_only()
 314            }) {
 315                continue;
 316            } else {
 317                guest_project
 318                    .read_with(client_cx, |project, _| project.remote_id())
 319                    .unwrap()
 320            };
 321            let guest_user_id = client.user_id().unwrap();
 322
 323            let host_project = clients.iter().find_map(|(client, cx)| {
 324                let project = client.local_projects.iter().find(|host_project| {
 325                    host_project.read_with(cx, |host_project, _| {
 326                        host_project.remote_id() == Some(project_id)
 327                    })
 328                })?;
 329                Some((client.user_id().unwrap(), project, cx))
 330            });
 331
 332            let (host_user_id, host_project, host_cx) =
 333                if let Some((host_user_id, host_project, host_cx)) = host_project {
 334                    (host_user_id, host_project, host_cx)
 335                } else {
 336                    continue;
 337                };
 338
 339            for guest_buffer in guest_buffers {
 340                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 341                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 342                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 343                        panic!(
 344                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 345                            client.username,
 346                            client.peer_id(),
 347                            buffer_id
 348                        )
 349                    })
 350                });
 351                let path = host_buffer
 352                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 353
 354                assert_eq!(
 355                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 356                    0,
 357                    "{}, buffer {}, path {:?} has deferred operations",
 358                    client.username,
 359                    buffer_id,
 360                    path,
 361                );
 362                assert_eq!(
 363                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 364                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 365                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 366                    client.username,
 367                    buffer_id,
 368                    path
 369                );
 370
 371                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 372                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 373                match (host_file, guest_file) {
 374                    (Some(host_file), Some(guest_file)) => {
 375                        assert_eq!(guest_file.path(), host_file.path());
 376                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 377                        assert_eq!(
 378                            guest_file.mtime(),
 379                            host_file.mtime(),
 380                            "guest {} mtime does not match host {} for path {:?} in project {}",
 381                            guest_user_id,
 382                            host_user_id,
 383                            guest_file.path(),
 384                            project_id,
 385                        );
 386                    }
 387                    (None, None) => {}
 388                    (None, _) => panic!("host's file is None, guest's isn't"),
 389                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 390                }
 391
 392                let host_diff_base =
 393                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 394                let guest_diff_base = guest_buffer
 395                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 396                assert_eq!(guest_diff_base, host_diff_base);
 397            }
 398        }
 399    }
 400
 401    for (client, mut cx) in clients {
 402        cx.update(|cx| {
 403            cx.clear_globals();
 404            drop(client);
 405        });
 406    }
 407}
 408
 409async fn simulate_client(
 410    mut client: TestClient,
 411    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 412    can_hang_up: bool,
 413    rng: Arc<Mutex<StdRng>>,
 414    mut cx: TestAppContext,
 415) -> (TestClient, TestAppContext) {
 416    // Setup language server
 417    let mut language = Language::new(
 418        LanguageConfig {
 419            name: "Rust".into(),
 420            path_suffixes: vec!["rs".to_string()],
 421            ..Default::default()
 422        },
 423        None,
 424    );
 425    let _fake_language_servers = language
 426        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 427            name: "the-fake-language-server",
 428            capabilities: lsp::LanguageServer::full_capabilities(),
 429            initializer: Some(Box::new({
 430                let rng = rng.clone();
 431                let fs = client.fs.clone();
 432                move |fake_server: &mut FakeLanguageServer| {
 433                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 434                        |_, _| async move {
 435                            Ok(Some(lsp::CompletionResponse::Array(vec![
 436                                lsp::CompletionItem {
 437                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 438                                        range: lsp::Range::new(
 439                                            lsp::Position::new(0, 0),
 440                                            lsp::Position::new(0, 0),
 441                                        ),
 442                                        new_text: "the-new-text".to_string(),
 443                                    })),
 444                                    ..Default::default()
 445                                },
 446                            ])))
 447                        },
 448                    );
 449
 450                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 451                        |_, _| async move {
 452                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 453                                lsp::CodeAction {
 454                                    title: "the-code-action".to_string(),
 455                                    ..Default::default()
 456                                },
 457                            )]))
 458                        },
 459                    );
 460
 461                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 462                        |params, _| async move {
 463                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 464                                params.position,
 465                                params.position,
 466                            ))))
 467                        },
 468                    );
 469
 470                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 471                        let fs = fs.clone();
 472                        let rng = rng.clone();
 473                        move |_, _| {
 474                            let fs = fs.clone();
 475                            let rng = rng.clone();
 476                            async move {
 477                                let files = fs.files().await;
 478                                let mut rng = rng.lock();
 479                                let count = rng.gen_range::<usize, _>(1..3);
 480                                let files = (0..count)
 481                                    .map(|_| files.choose(&mut *rng).unwrap())
 482                                    .collect::<Vec<_>>();
 483                                log::info!("LSP: Returning definitions in files {:?}", &files);
 484                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 485                                    files
 486                                        .into_iter()
 487                                        .map(|file| lsp::Location {
 488                                            uri: lsp::Url::from_file_path(file).unwrap(),
 489                                            range: Default::default(),
 490                                        })
 491                                        .collect(),
 492                                )))
 493                            }
 494                        }
 495                    });
 496
 497                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 498                        let rng = rng.clone();
 499                        move |_, _| {
 500                            let mut highlights = Vec::new();
 501                            let highlight_count = rng.lock().gen_range(1..=5);
 502                            for _ in 0..highlight_count {
 503                                let start_row = rng.lock().gen_range(0..100);
 504                                let start_column = rng.lock().gen_range(0..100);
 505                                let start = PointUtf16::new(start_row, start_column);
 506                                let end_row = rng.lock().gen_range(0..100);
 507                                let end_column = rng.lock().gen_range(0..100);
 508                                let end = PointUtf16::new(end_row, end_column);
 509                                let range = if start > end { end..start } else { start..end };
 510                                highlights.push(lsp::DocumentHighlight {
 511                                    range: range_to_lsp(range.clone()),
 512                                    kind: Some(lsp::DocumentHighlightKind::READ),
 513                                });
 514                            }
 515                            highlights.sort_unstable_by_key(|highlight| {
 516                                (highlight.range.start, highlight.range.end)
 517                            });
 518                            async move { Ok(Some(highlights)) }
 519                        }
 520                    });
 521                }
 522            })),
 523            ..Default::default()
 524        }))
 525        .await;
 526    client.language_registry.add(Arc::new(language));
 527
 528    while op_start_signal.next().await.is_some() {
 529        if let Err(error) =
 530            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 531        {
 532            log::error!("{} error: {:?}", client.username, error);
 533        }
 534
 535        cx.background().simulate_random_delay().await;
 536    }
 537    log::info!("{}: done", client.username);
 538
 539    (client, cx)
 540}
 541
 542async fn randomly_mutate_client(
 543    client: &mut TestClient,
 544    can_hang_up: bool,
 545    rng: Arc<Mutex<StdRng>>,
 546    cx: &mut TestAppContext,
 547) -> Result<()> {
 548    let choice = rng.lock().gen_range(0..100);
 549    match choice {
 550        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 551        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 552        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 553            randomly_mutate_worktrees(client, &rng, cx).await?;
 554        }
 555        60..=74 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 556            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 557        }
 558        75..=84 => randomly_mutate_git(client, &rng).await,
 559        _ => randomly_mutate_fs(client, &rng).await,
 560    }
 561
 562    Ok(())
 563}
 564
 565async fn randomly_mutate_active_call(
 566    client: &mut TestClient,
 567    can_hang_up: bool,
 568    rng: &Mutex<StdRng>,
 569    cx: &mut TestAppContext,
 570) -> Result<()> {
 571    let active_call = cx.read(ActiveCall::global);
 572    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 573        if rng.lock().gen_bool(0.7) {
 574            log::info!("{}: accepting incoming call", client.username);
 575            active_call
 576                .update(cx, |call, cx| call.accept_incoming(cx))
 577                .await?;
 578        } else {
 579            log::info!("{}: declining incoming call", client.username);
 580            active_call.update(cx, |call, _| call.decline_incoming())?;
 581        }
 582    } else {
 583        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 584            user_store
 585                .contacts()
 586                .iter()
 587                .filter(|contact| contact.online && !contact.busy)
 588                .cloned()
 589                .collect::<Vec<_>>()
 590        });
 591
 592        let distribution = rng.lock().gen_range(0..100);
 593        match distribution {
 594            0..=29 if !available_contacts.is_empty() => {
 595                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 596                log::info!(
 597                    "{}: inviting {}",
 598                    client.username,
 599                    contact.user.github_login
 600                );
 601                active_call
 602                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 603                    .await?;
 604            }
 605            30..=39
 606                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 607            {
 608                log::info!("{}: hanging up", client.username);
 609                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 610            }
 611            _ => {}
 612        }
 613    }
 614
 615    Ok(())
 616}
 617
 618async fn randomly_mutate_git(client: &mut TestClient, rng: &Mutex<StdRng>) {
 619    let directories = client.fs.directories().await;
 620    let mut dir_path = directories.choose(&mut *rng.lock()).unwrap().clone();
 621    if dir_path.file_name() == Some(OsStr::new(".git")) {
 622        dir_path.pop();
 623    }
 624    let mut git_dir_path = dir_path.clone();
 625    git_dir_path.push(".git");
 626
 627    if !directories.contains(&git_dir_path) {
 628        log::info!(
 629            "{}: creating git directory at {:?}",
 630            client.username,
 631            git_dir_path
 632        );
 633        client.fs.create_dir(&git_dir_path).await.unwrap();
 634    }
 635
 636    let mut child_paths = client.fs.read_dir(&dir_path).await.unwrap();
 637    let mut child_file_paths = Vec::new();
 638    while let Some(child_path) = child_paths.next().await {
 639        let child_path = child_path.unwrap();
 640        if client.fs.is_file(&child_path).await {
 641            child_file_paths.push(child_path);
 642        }
 643    }
 644    let count = rng.lock().gen_range(0..=child_file_paths.len());
 645    child_file_paths.shuffle(&mut *rng.lock());
 646    child_file_paths.truncate(count);
 647
 648    let mut new_index = Vec::new();
 649    for abs_child_file_path in &child_file_paths {
 650        let child_file_path = abs_child_file_path.strip_prefix(&dir_path).unwrap();
 651        let new_base = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 652        new_index.push((child_file_path, new_base));
 653    }
 654    log::info!(
 655        "{}: updating Git index at {:?}: {:#?}",
 656        client.username,
 657        git_dir_path,
 658        new_index
 659    );
 660    client
 661        .fs
 662        .set_index_for_repo(&git_dir_path, &new_index)
 663        .await;
 664}
 665
 666async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 667    let is_dir = rng.lock().gen::<bool>();
 668    let mut new_path = client
 669        .fs
 670        .directories()
 671        .await
 672        .choose(&mut *rng.lock())
 673        .unwrap()
 674        .clone();
 675    new_path.push(gen_file_name(rng));
 676    if is_dir {
 677        log::info!("{}: creating local dir at {:?}", client.username, new_path);
 678        client.fs.create_dir(&new_path).await.unwrap();
 679    } else {
 680        new_path.set_extension("rs");
 681        log::info!("{}: creating local file at {:?}", client.username, new_path);
 682        client
 683            .fs
 684            .create_file(&new_path, Default::default())
 685            .await
 686            .unwrap();
 687    }
 688}
 689
 690async fn randomly_mutate_projects(
 691    client: &mut TestClient,
 692    rng: &Mutex<StdRng>,
 693    cx: &mut TestAppContext,
 694) -> Result<()> {
 695    let active_call = cx.read(ActiveCall::global);
 696    let remote_projects =
 697        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 698            room.read_with(cx, |room, _| {
 699                room.remote_participants()
 700                    .values()
 701                    .flat_map(|participant| participant.projects.clone())
 702                    .collect::<Vec<_>>()
 703            })
 704        } else {
 705            Default::default()
 706        };
 707
 708    let project = if remote_projects.is_empty() || rng.lock().gen() {
 709        if client.local_projects.is_empty() || rng.lock().gen() {
 710            let paths = client.fs.paths().await;
 711            let local_project = if paths.is_empty() || rng.lock().gen() {
 712                let root_path = client.create_new_root_dir();
 713                client.fs.create_dir(&root_path).await.unwrap();
 714                client
 715                    .fs
 716                    .create_file(&root_path.join("main.rs"), Default::default())
 717                    .await
 718                    .unwrap();
 719                log::info!(
 720                    "{}: opening local project at {:?}",
 721                    client.username,
 722                    root_path
 723                );
 724                client.build_local_project(root_path, cx).await.0
 725            } else {
 726                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 727                log::info!(
 728                    "{}: opening local project at {:?}",
 729                    client.username,
 730                    root_path
 731                );
 732                client.build_local_project(root_path, cx).await.0
 733            };
 734            client.local_projects.push(local_project.clone());
 735            local_project
 736        } else {
 737            client
 738                .local_projects
 739                .choose(&mut *rng.lock())
 740                .unwrap()
 741                .clone()
 742        }
 743    } else {
 744        if client.remote_projects.is_empty() || rng.lock().gen() {
 745            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 746            let remote_project = if let Some(project) =
 747                client.remote_projects.iter().find(|project| {
 748                    project.read_with(cx, |project, _| {
 749                        project.remote_id() == Some(remote_project_id)
 750                    })
 751                }) {
 752                project.clone()
 753            } else {
 754                log::info!(
 755                    "{}: opening remote project {}",
 756                    client.username,
 757                    remote_project_id
 758                );
 759                let call = cx.read(ActiveCall::global);
 760                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 761                let remote_project = room
 762                    .update(cx, |room, cx| {
 763                        room.join_project(
 764                            remote_project_id,
 765                            client.language_registry.clone(),
 766                            FakeFs::new(cx.background().clone()),
 767                            cx,
 768                        )
 769                    })
 770                    .await?;
 771                client.remote_projects.push(remote_project.clone());
 772                remote_project
 773            };
 774
 775            remote_project
 776        } else {
 777            client
 778                .remote_projects
 779                .choose(&mut *rng.lock())
 780                .unwrap()
 781                .clone()
 782        }
 783    };
 784
 785    if active_call.read_with(cx, |call, _| call.room().is_some())
 786        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 787    {
 788        match active_call
 789            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 790            .await
 791        {
 792            Ok(project_id) => {
 793                log::info!("{}: shared project with id {}", client.username, project_id);
 794            }
 795            Err(error) => {
 796                log::error!("{}: error sharing project, {:?}", client.username, error);
 797            }
 798        }
 799    }
 800
 801    let choice = rng.lock().gen_range(0..100);
 802    match choice {
 803        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 804            let paths = client.fs.paths().await;
 805            let path = paths.choose(&mut *rng.lock()).unwrap();
 806            log::info!(
 807                "{}: finding/creating local worktree for path {:?}",
 808                client.username,
 809                path
 810            );
 811            project
 812                .update(cx, |project, cx| {
 813                    project.find_or_create_local_worktree(&path, true, cx)
 814                })
 815                .await
 816                .unwrap();
 817        }
 818        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 819            log::info!(
 820                "{}: dropping remote project {}",
 821                client.username,
 822                project.read_with(cx, |project, _| project.remote_id().unwrap())
 823            );
 824
 825            cx.update(|_| {
 826                client
 827                    .remote_projects
 828                    .retain(|remote_project| *remote_project != project);
 829                client.buffers.remove(&project);
 830                drop(project);
 831            });
 832        }
 833        _ => {}
 834    }
 835
 836    Ok(())
 837}
 838
 839async fn randomly_mutate_worktrees(
 840    client: &mut TestClient,
 841    rng: &Mutex<StdRng>,
 842    cx: &mut TestAppContext,
 843) -> Result<()> {
 844    let project = choose_random_project(client, rng).unwrap();
 845    let Some(worktree) = project.read_with(cx, |project, cx| {
 846        project
 847            .worktrees(cx)
 848            .filter(|worktree| {
 849                let worktree = worktree.read(cx);
 850                worktree.is_visible()
 851                    && worktree.entries(false).any(|e| e.is_file())
 852                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 853            })
 854            .choose(&mut *rng.lock())
 855    }) else {
 856        return Ok(())
 857    };
 858
 859    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 860        (worktree.id(), worktree.root_name().to_string())
 861    });
 862
 863    let is_dir = rng.lock().gen::<bool>();
 864    let mut new_path = PathBuf::new();
 865    new_path.push(gen_file_name(rng));
 866    if !is_dir {
 867        new_path.set_extension("rs");
 868    }
 869    log::info!(
 870        "{}: creating {:?} in worktree {} ({})",
 871        client.username,
 872        new_path,
 873        worktree_id,
 874        worktree_root_name,
 875    );
 876    project
 877        .update(cx, |project, cx| {
 878            project.create_entry((worktree_id, new_path), is_dir, cx)
 879        })
 880        .unwrap()
 881        .await?;
 882    Ok(())
 883}
 884
 885async fn randomly_query_and_mutate_buffers(
 886    client: &mut TestClient,
 887    rng: &Mutex<StdRng>,
 888    cx: &mut TestAppContext,
 889) -> Result<()> {
 890    let project = choose_random_project(client, rng).unwrap();
 891    let buffers = client.buffers.entry(project.clone()).or_default();
 892    let buffer = if buffers.is_empty() || rng.lock().gen() {
 893        let Some(worktree) = project.read_with(cx, |project, cx| {
 894            project
 895                .worktrees(cx)
 896                .filter(|worktree| {
 897                    let worktree = worktree.read(cx);
 898                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 899                })
 900                .choose(&mut *rng.lock())
 901        }) else {
 902            return Ok(());
 903        };
 904
 905        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 906            let entry = worktree
 907                .entries(false)
 908                .filter(|e| e.is_file())
 909                .choose(&mut *rng.lock())
 910                .unwrap();
 911            (
 912                worktree.root_name().to_string(),
 913                (worktree.id(), entry.path.clone()),
 914            )
 915        });
 916        log::info!(
 917            "{}: opening path {:?} in worktree {} ({})",
 918            client.username,
 919            project_path.1,
 920            project_path.0,
 921            worktree_root_name,
 922        );
 923        let buffer = project
 924            .update(cx, |project, cx| {
 925                project.open_buffer(project_path.clone(), cx)
 926            })
 927            .await?;
 928        log::info!(
 929            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 930            client.username,
 931            project_path.1,
 932            project_path.0,
 933            worktree_root_name,
 934            buffer.read_with(cx, |buffer, _| buffer.remote_id())
 935        );
 936        buffers.insert(buffer.clone());
 937        buffer
 938    } else {
 939        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
 940    };
 941
 942    let choice = rng.lock().gen_range(0..100);
 943    match choice {
 944        0..=9 => {
 945            cx.update(|cx| {
 946                log::info!(
 947                    "{}: dropping buffer {:?}",
 948                    client.username,
 949                    buffer.read(cx).file().unwrap().full_path(cx)
 950                );
 951                buffers.remove(&buffer);
 952                drop(buffer);
 953            });
 954        }
 955        10..=19 => {
 956            let completions = project.update(cx, |project, cx| {
 957                log::info!(
 958                    "{}: requesting completions for buffer {} ({:?})",
 959                    client.username,
 960                    buffer.read(cx).remote_id(),
 961                    buffer.read(cx).file().unwrap().full_path(cx)
 962                );
 963                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 964                project.completions(&buffer, offset, cx)
 965            });
 966            let completions = cx.background().spawn(async move {
 967                completions
 968                    .await
 969                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
 970            });
 971            if rng.lock().gen_bool(0.3) {
 972                log::info!("{}: detaching completions request", client.username);
 973                cx.update(|cx| completions.detach_and_log_err(cx));
 974            } else {
 975                completions.await?;
 976            }
 977        }
 978        20..=29 => {
 979            let code_actions = project.update(cx, |project, cx| {
 980                log::info!(
 981                    "{}: requesting code actions for buffer {} ({:?})",
 982                    client.username,
 983                    buffer.read(cx).remote_id(),
 984                    buffer.read(cx).file().unwrap().full_path(cx)
 985                );
 986                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
 987                project.code_actions(&buffer, range, cx)
 988            });
 989            let code_actions = cx.background().spawn(async move {
 990                code_actions
 991                    .await
 992                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
 993            });
 994            if rng.lock().gen_bool(0.3) {
 995                log::info!("{}: detaching code actions request", client.username);
 996                cx.update(|cx| code_actions.detach_and_log_err(cx));
 997            } else {
 998                code_actions.await?;
 999            }
1000        }
1001        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1002            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1003                log::info!(
1004                    "{}: saving buffer {} ({:?})",
1005                    client.username,
1006                    buffer.remote_id(),
1007                    buffer.file().unwrap().full_path(cx)
1008                );
1009                (buffer.version(), buffer.save(cx))
1010            });
1011            let save = cx.background().spawn(async move {
1012                let (saved_version, _, _) = save
1013                    .await
1014                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1015                assert!(saved_version.observed_all(&requested_version));
1016                Ok::<_, anyhow::Error>(())
1017            });
1018            if rng.lock().gen_bool(0.3) {
1019                log::info!("{}: detaching save request", client.username);
1020                cx.update(|cx| save.detach_and_log_err(cx));
1021            } else {
1022                save.await?;
1023            }
1024        }
1025        40..=44 => {
1026            let prepare_rename = project.update(cx, |project, cx| {
1027                log::info!(
1028                    "{}: preparing rename for buffer {} ({:?})",
1029                    client.username,
1030                    buffer.read(cx).remote_id(),
1031                    buffer.read(cx).file().unwrap().full_path(cx)
1032                );
1033                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1034                project.prepare_rename(buffer, offset, cx)
1035            });
1036            let prepare_rename = cx.background().spawn(async move {
1037                prepare_rename
1038                    .await
1039                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1040            });
1041            if rng.lock().gen_bool(0.3) {
1042                log::info!("{}: detaching prepare rename request", client.username);
1043                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1044            } else {
1045                prepare_rename.await?;
1046            }
1047        }
1048        45..=49 => {
1049            let definitions = project.update(cx, |project, cx| {
1050                log::info!(
1051                    "{}: requesting definitions for buffer {} ({:?})",
1052                    client.username,
1053                    buffer.read(cx).remote_id(),
1054                    buffer.read(cx).file().unwrap().full_path(cx)
1055                );
1056                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1057                project.definition(&buffer, offset, cx)
1058            });
1059            let definitions = cx.background().spawn(async move {
1060                definitions
1061                    .await
1062                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1063            });
1064            if rng.lock().gen_bool(0.3) {
1065                log::info!("{}: detaching definitions request", client.username);
1066                cx.update(|cx| definitions.detach_and_log_err(cx));
1067            } else {
1068                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1069            }
1070        }
1071        50..=54 => {
1072            let highlights = project.update(cx, |project, cx| {
1073                log::info!(
1074                    "{}: requesting highlights for buffer {} ({:?})",
1075                    client.username,
1076                    buffer.read(cx).remote_id(),
1077                    buffer.read(cx).file().unwrap().full_path(cx)
1078                );
1079                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1080                project.document_highlights(&buffer, offset, cx)
1081            });
1082            let highlights = cx.background().spawn(async move {
1083                highlights
1084                    .await
1085                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1086            });
1087            if rng.lock().gen_bool(0.3) {
1088                log::info!("{}: detaching highlights request", client.username);
1089                cx.update(|cx| highlights.detach_and_log_err(cx));
1090            } else {
1091                highlights.await?;
1092            }
1093        }
1094        55..=59 => {
1095            let search = project.update(cx, |project, cx| {
1096                let query = rng.lock().gen_range('a'..='z');
1097                log::info!("{}: project-wide search {:?}", client.username, query);
1098                project.search(SearchQuery::text(query, false, false), cx)
1099            });
1100            let search = cx.background().spawn(async move {
1101                search
1102                    .await
1103                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1104            });
1105            if rng.lock().gen_bool(0.3) {
1106                log::info!("{}: detaching search request", client.username);
1107                cx.update(|cx| search.detach_and_log_err(cx));
1108            } else {
1109                buffers.extend(search.await?.into_keys());
1110            }
1111        }
1112        _ => {
1113            buffer.update(cx, |buffer, cx| {
1114                log::info!(
1115                    "{}: updating buffer {} ({:?})",
1116                    client.username,
1117                    buffer.remote_id(),
1118                    buffer.file().unwrap().full_path(cx)
1119                );
1120                if rng.lock().gen_bool(0.7) {
1121                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1122                } else {
1123                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1124                }
1125            });
1126        }
1127    }
1128
1129    Ok(())
1130}
1131
1132fn choose_random_project(
1133    client: &mut TestClient,
1134    rng: &Mutex<StdRng>,
1135) -> Option<ModelHandle<Project>> {
1136    client
1137        .local_projects
1138        .iter()
1139        .chain(&client.remote_projects)
1140        .choose(&mut *rng.lock())
1141        .cloned()
1142}
1143
1144fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1145    let mut name = String::new();
1146    for _ in 0..10 {
1147        let letter = rng.lock().gen_range('a'..='z');
1148        name.push(letter);
1149    }
1150    name
1151}