randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::{
  18    distributions::{Alphanumeric, DistString},
  19    prelude::*,
  20};
  21use settings::Settings;
  22use std::{env, ffi::OsStr, path::PathBuf, sync::Arc};
  23
  24#[gpui::test(iterations = 100)]
  25async fn test_random_collaboration(
  26    cx: &mut TestAppContext,
  27    deterministic: Arc<Deterministic>,
  28    rng: StdRng,
  29) {
  30    deterministic.forbid_parking();
  31    let rng = Arc::new(Mutex::new(rng));
  32
  33    let max_peers = env::var("MAX_PEERS")
  34        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  35        .unwrap_or(5);
  36
  37    let max_operations = env::var("OPERATIONS")
  38        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  39        .unwrap_or(10);
  40
  41    let mut server = TestServer::start(&deterministic).await;
  42    let db = server.app_state.db.clone();
  43
  44    let mut available_users = Vec::new();
  45    for ix in 0..max_peers {
  46        let username = format!("user-{}", ix + 1);
  47        let user_id = db
  48            .create_user(
  49                &format!("{username}@example.com"),
  50                false,
  51                NewUserParams {
  52                    github_login: username.clone(),
  53                    github_user_id: (ix + 1) as i32,
  54                    invite_count: 0,
  55                },
  56            )
  57            .await
  58            .unwrap()
  59            .user_id;
  60        available_users.push((user_id, username));
  61    }
  62
  63    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
  64        for (user_id_b, _) in &available_users[ix + 1..] {
  65            server
  66                .app_state
  67                .db
  68                .send_contact_request(*user_id_a, *user_id_b)
  69                .await
  70                .unwrap();
  71            server
  72                .app_state
  73                .db
  74                .respond_to_contact_request(*user_id_b, *user_id_a, true)
  75                .await
  76                .unwrap();
  77        }
  78    }
  79
  80    let mut clients = Vec::new();
  81    let mut user_ids = Vec::new();
  82    let mut op_start_signals = Vec::new();
  83    let mut next_entity_id = 100000;
  84    let allow_server_restarts = rng.lock().gen_bool(0.7);
  85    let allow_client_reconnection = rng.lock().gen_bool(0.7);
  86    let allow_client_disconnection = rng.lock().gen_bool(0.1);
  87
  88    let mut operations = 0;
  89    while operations < max_operations {
  90        let distribution = rng.lock().gen_range(0..100);
  91        match distribution {
  92            0..=19 if !available_users.is_empty() => {
  93                let client_ix = rng.lock().gen_range(0..available_users.len());
  94                let (_, username) = available_users.remove(client_ix);
  95                log::info!("Adding new connection for {}", username);
  96                next_entity_id += 100000;
  97                let mut client_cx = TestAppContext::new(
  98                    cx.foreground_platform(),
  99                    cx.platform(),
 100                    deterministic.build_foreground(next_entity_id),
 101                    deterministic.build_background(),
 102                    cx.font_cache(),
 103                    cx.leak_detector(),
 104                    next_entity_id,
 105                    cx.function_name.clone(),
 106                );
 107
 108                client_cx.update(|cx| cx.set_global(Settings::test(cx)));
 109
 110                let op_start_signal = futures::channel::mpsc::unbounded();
 111                let client = server.create_client(&mut client_cx, &username).await;
 112                user_ids.push(client.current_user_id(&client_cx));
 113                op_start_signals.push(op_start_signal.0);
 114                clients.push(client_cx.foreground().spawn(simulate_client(
 115                    client,
 116                    op_start_signal.1,
 117                    allow_client_disconnection,
 118                    rng.clone(),
 119                    client_cx,
 120                )));
 121
 122                log::info!("Added connection for {}", username);
 123                operations += 1;
 124            }
 125
 126            20..=24 if clients.len() > 1 && allow_client_disconnection => {
 127                let client_ix = rng.lock().gen_range(1..clients.len());
 128                log::info!(
 129                    "Simulating full disconnection of user {}",
 130                    user_ids[client_ix]
 131                );
 132                let removed_user_id = user_ids.remove(client_ix);
 133                let user_connection_ids = server
 134                    .connection_pool
 135                    .lock()
 136                    .user_connection_ids(removed_user_id)
 137                    .collect::<Vec<_>>();
 138                assert_eq!(user_connection_ids.len(), 1);
 139                let removed_peer_id = user_connection_ids[0].into();
 140                let client = clients.remove(client_ix);
 141                op_start_signals.remove(client_ix);
 142                server.forbid_connections();
 143                server.disconnect_client(removed_peer_id);
 144                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 145                deterministic.start_waiting();
 146                log::info!("Waiting for user {} to exit...", removed_user_id);
 147                let (client, mut client_cx) = client.await;
 148                deterministic.finish_waiting();
 149                server.allow_connections();
 150
 151                for project in &client.remote_projects {
 152                    project.read_with(&client_cx, |project, _| {
 153                        assert!(
 154                            project.is_read_only(),
 155                            "project {:?} should be read only",
 156                            project.remote_id()
 157                        )
 158                    });
 159                }
 160                for user_id in &user_ids {
 161                    let contacts = server.app_state.db.get_contacts(*user_id).await.unwrap();
 162                    let pool = server.connection_pool.lock();
 163                    for contact in contacts {
 164                        if let db::Contact::Accepted { user_id, .. } = contact {
 165                            if pool.is_user_online(user_id) {
 166                                assert_ne!(
 167                                    user_id, removed_user_id,
 168                                    "removed client is still a contact of another peer"
 169                                );
 170                            }
 171                        }
 172                    }
 173                }
 174
 175                log::info!("{} removed", client.username);
 176                available_users.push((removed_user_id, client.username.clone()));
 177                client_cx.update(|cx| {
 178                    cx.clear_globals();
 179                    cx.set_global(Settings::test(cx));
 180                    drop(client);
 181                });
 182
 183                operations += 1;
 184            }
 185
 186            25..=29 if clients.len() > 1 && allow_client_reconnection => {
 187                let client_ix = rng.lock().gen_range(1..clients.len());
 188                let user_id = user_ids[client_ix];
 189                log::info!("Simulating temporary disconnection of user {}", user_id);
 190                let user_connection_ids = server
 191                    .connection_pool
 192                    .lock()
 193                    .user_connection_ids(user_id)
 194                    .collect::<Vec<_>>();
 195                assert_eq!(user_connection_ids.len(), 1);
 196                let peer_id = user_connection_ids[0].into();
 197                server.disconnect_client(peer_id);
 198                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 199                operations += 1;
 200            }
 201
 202            30..=34 if allow_server_restarts => {
 203                log::info!("Simulating server restart");
 204                server.reset().await;
 205                deterministic.advance_clock(RECEIVE_TIMEOUT);
 206                server.start().await.unwrap();
 207                deterministic.advance_clock(CLEANUP_TIMEOUT);
 208                let environment = &server.app_state.config.zed_environment;
 209                let stale_room_ids = server
 210                    .app_state
 211                    .db
 212                    .stale_room_ids(environment, server.id())
 213                    .await
 214                    .unwrap();
 215                assert_eq!(stale_room_ids, vec![]);
 216            }
 217
 218            _ if !op_start_signals.is_empty() => {
 219                while operations < max_operations && rng.lock().gen_bool(0.7) {
 220                    op_start_signals
 221                        .choose(&mut *rng.lock())
 222                        .unwrap()
 223                        .unbounded_send(())
 224                        .unwrap();
 225                    operations += 1;
 226                }
 227
 228                if rng.lock().gen_bool(0.8) {
 229                    deterministic.run_until_parked();
 230                }
 231            }
 232            _ => {}
 233        }
 234    }
 235
 236    drop(op_start_signals);
 237    deterministic.start_waiting();
 238    let clients = futures::future::join_all(clients).await;
 239    deterministic.finish_waiting();
 240    deterministic.run_until_parked();
 241
 242    for (client, client_cx) in &clients {
 243        for guest_project in &client.remote_projects {
 244            guest_project.read_with(client_cx, |guest_project, cx| {
 245                let host_project = clients.iter().find_map(|(client, cx)| {
 246                    let project = client.local_projects.iter().find(|host_project| {
 247                        host_project.read_with(cx, |host_project, _| {
 248                            host_project.remote_id() == guest_project.remote_id()
 249                        })
 250                    })?;
 251                    Some((project, cx))
 252                });
 253
 254                if !guest_project.is_read_only() {
 255                    if let Some((host_project, host_cx)) = host_project {
 256                        let host_worktree_snapshots =
 257                            host_project.read_with(host_cx, |host_project, cx| {
 258                                host_project
 259                                    .worktrees(cx)
 260                                    .map(|worktree| {
 261                                        let worktree = worktree.read(cx);
 262                                        (worktree.id(), worktree.snapshot())
 263                                    })
 264                                    .collect::<BTreeMap<_, _>>()
 265                            });
 266                        let guest_worktree_snapshots = guest_project
 267                            .worktrees(cx)
 268                            .map(|worktree| {
 269                                let worktree = worktree.read(cx);
 270                                (worktree.id(), worktree.snapshot())
 271                            })
 272                            .collect::<BTreeMap<_, _>>();
 273
 274                        assert_eq!(
 275                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 276                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 277                            "{} has different worktrees than the host",
 278                            client.username
 279                        );
 280
 281                        for (id, host_snapshot) in &host_worktree_snapshots {
 282                            let guest_snapshot = &guest_worktree_snapshots[id];
 283                            assert_eq!(
 284                                guest_snapshot.root_name(),
 285                                host_snapshot.root_name(),
 286                                "{} has different root name than the host for worktree {}",
 287                                client.username,
 288                                id
 289                            );
 290                            assert_eq!(
 291                                guest_snapshot.abs_path(),
 292                                host_snapshot.abs_path(),
 293                                "{} has different abs path than the host for worktree {}",
 294                                client.username,
 295                                id
 296                            );
 297                            assert_eq!(
 298                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 299                                host_snapshot.entries(false).collect::<Vec<_>>(),
 300                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 301                                client.username,
 302                                id,
 303                                host_snapshot.abs_path(),
 304                                host_project.read_with(host_cx, |project, _| project.remote_id())
 305                            );
 306                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 307                        }
 308                    }
 309                }
 310
 311                guest_project.check_invariants(cx);
 312            });
 313        }
 314
 315        for (guest_project, guest_buffers) in &client.buffers {
 316            let project_id = if guest_project.read_with(client_cx, |project, _| {
 317                project.is_local() || project.is_read_only()
 318            }) {
 319                continue;
 320            } else {
 321                guest_project
 322                    .read_with(client_cx, |project, _| project.remote_id())
 323                    .unwrap()
 324            };
 325            let guest_user_id = client.user_id().unwrap();
 326
 327            let host_project = clients.iter().find_map(|(client, cx)| {
 328                let project = client.local_projects.iter().find(|host_project| {
 329                    host_project.read_with(cx, |host_project, _| {
 330                        host_project.remote_id() == Some(project_id)
 331                    })
 332                })?;
 333                Some((client.user_id().unwrap(), project, cx))
 334            });
 335
 336            let (host_user_id, host_project, host_cx) =
 337                if let Some((host_user_id, host_project, host_cx)) = host_project {
 338                    (host_user_id, host_project, host_cx)
 339                } else {
 340                    continue;
 341                };
 342
 343            for guest_buffer in guest_buffers {
 344                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 345                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 346                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 347                        panic!(
 348                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 349                            client.username,
 350                            client.peer_id(),
 351                            buffer_id
 352                        )
 353                    })
 354                });
 355                let path = host_buffer
 356                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 357
 358                assert_eq!(
 359                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 360                    0,
 361                    "{}, buffer {}, path {:?} has deferred operations",
 362                    client.username,
 363                    buffer_id,
 364                    path,
 365                );
 366                assert_eq!(
 367                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 368                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 369                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 370                    client.username,
 371                    buffer_id,
 372                    path
 373                );
 374
 375                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 376                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 377                match (host_file, guest_file) {
 378                    (Some(host_file), Some(guest_file)) => {
 379                        assert_eq!(guest_file.path(), host_file.path());
 380                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 381                        assert_eq!(
 382                            guest_file.mtime(),
 383                            host_file.mtime(),
 384                            "guest {} mtime does not match host {} for path {:?} in project {}",
 385                            guest_user_id,
 386                            host_user_id,
 387                            guest_file.path(),
 388                            project_id,
 389                        );
 390                    }
 391                    (None, None) => {}
 392                    (None, _) => panic!("host's file is None, guest's isn't"),
 393                    (_, None) => panic!("guest's file is None, hosts's isn't"),
 394                }
 395
 396                let host_diff_base =
 397                    host_buffer.read_with(host_cx, |b, _| b.diff_base().map(ToString::to_string));
 398                let guest_diff_base = guest_buffer
 399                    .read_with(client_cx, |b, _| b.diff_base().map(ToString::to_string));
 400                assert_eq!(guest_diff_base, host_diff_base);
 401            }
 402        }
 403    }
 404
 405    for (client, mut cx) in clients {
 406        cx.update(|cx| {
 407            cx.clear_globals();
 408            cx.set_global(Settings::test(cx));
 409            drop(client);
 410        });
 411    }
 412}
 413
 414async fn simulate_client(
 415    mut client: TestClient,
 416    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<()>,
 417    can_hang_up: bool,
 418    rng: Arc<Mutex<StdRng>>,
 419    mut cx: TestAppContext,
 420) -> (TestClient, TestAppContext) {
 421    // Setup language server
 422    let mut language = Language::new(
 423        LanguageConfig {
 424            name: "Rust".into(),
 425            path_suffixes: vec!["rs".to_string()],
 426            ..Default::default()
 427        },
 428        None,
 429    );
 430    let _fake_language_servers = language
 431        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 432            name: "the-fake-language-server",
 433            capabilities: lsp::LanguageServer::full_capabilities(),
 434            initializer: Some(Box::new({
 435                let rng = rng.clone();
 436                let fs = client.fs.clone();
 437                move |fake_server: &mut FakeLanguageServer| {
 438                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 439                        |_, _| async move {
 440                            Ok(Some(lsp::CompletionResponse::Array(vec![
 441                                lsp::CompletionItem {
 442                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 443                                        range: lsp::Range::new(
 444                                            lsp::Position::new(0, 0),
 445                                            lsp::Position::new(0, 0),
 446                                        ),
 447                                        new_text: "the-new-text".to_string(),
 448                                    })),
 449                                    ..Default::default()
 450                                },
 451                            ])))
 452                        },
 453                    );
 454
 455                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 456                        |_, _| async move {
 457                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 458                                lsp::CodeAction {
 459                                    title: "the-code-action".to_string(),
 460                                    ..Default::default()
 461                                },
 462                            )]))
 463                        },
 464                    );
 465
 466                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 467                        |params, _| async move {
 468                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 469                                params.position,
 470                                params.position,
 471                            ))))
 472                        },
 473                    );
 474
 475                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 476                        let fs = fs.clone();
 477                        let rng = rng.clone();
 478                        move |_, _| {
 479                            let fs = fs.clone();
 480                            let rng = rng.clone();
 481                            async move {
 482                                let files = fs.files().await;
 483                                let mut rng = rng.lock();
 484                                let count = rng.gen_range::<usize, _>(1..3);
 485                                let files = (0..count)
 486                                    .map(|_| files.choose(&mut *rng).unwrap())
 487                                    .collect::<Vec<_>>();
 488                                log::info!("LSP: Returning definitions in files {:?}", &files);
 489                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 490                                    files
 491                                        .into_iter()
 492                                        .map(|file| lsp::Location {
 493                                            uri: lsp::Url::from_file_path(file).unwrap(),
 494                                            range: Default::default(),
 495                                        })
 496                                        .collect(),
 497                                )))
 498                            }
 499                        }
 500                    });
 501
 502                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 503                        let rng = rng.clone();
 504                        move |_, _| {
 505                            let mut highlights = Vec::new();
 506                            let highlight_count = rng.lock().gen_range(1..=5);
 507                            for _ in 0..highlight_count {
 508                                let start_row = rng.lock().gen_range(0..100);
 509                                let start_column = rng.lock().gen_range(0..100);
 510                                let start = PointUtf16::new(start_row, start_column);
 511                                let end_row = rng.lock().gen_range(0..100);
 512                                let end_column = rng.lock().gen_range(0..100);
 513                                let end = PointUtf16::new(end_row, end_column);
 514                                let range = if start > end { end..start } else { start..end };
 515                                highlights.push(lsp::DocumentHighlight {
 516                                    range: range_to_lsp(range.clone()),
 517                                    kind: Some(lsp::DocumentHighlightKind::READ),
 518                                });
 519                            }
 520                            highlights.sort_unstable_by_key(|highlight| {
 521                                (highlight.range.start, highlight.range.end)
 522                            });
 523                            async move { Ok(Some(highlights)) }
 524                        }
 525                    });
 526                }
 527            })),
 528            ..Default::default()
 529        }))
 530        .await;
 531    client.language_registry.add(Arc::new(language));
 532
 533    while op_start_signal.next().await.is_some() {
 534        if let Err(error) =
 535            randomly_mutate_client(&mut client, can_hang_up, rng.clone(), &mut cx).await
 536        {
 537            log::error!("{} error: {:?}", client.username, error);
 538        }
 539
 540        cx.background().simulate_random_delay().await;
 541    }
 542    log::info!("{}: done", client.username);
 543
 544    (client, cx)
 545}
 546
 547async fn randomly_mutate_client(
 548    client: &mut TestClient,
 549    can_hang_up: bool,
 550    rng: Arc<Mutex<StdRng>>,
 551    cx: &mut TestAppContext,
 552) -> Result<()> {
 553    let choice = rng.lock().gen_range(0..100);
 554    match choice {
 555        0..=19 => randomly_mutate_active_call(client, can_hang_up, &rng, cx).await?,
 556        20..=49 => randomly_mutate_projects(client, &rng, cx).await?,
 557        50..=59 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 558            randomly_mutate_worktrees(client, &rng, cx).await?;
 559        }
 560        60..=74 if !client.local_projects.is_empty() || !client.remote_projects.is_empty() => {
 561            randomly_query_and_mutate_buffers(client, &rng, cx).await?;
 562        }
 563        75..=84 => randomly_mutate_git(client, &rng).await,
 564        _ => randomly_mutate_fs(client, &rng).await,
 565    }
 566
 567    Ok(())
 568}
 569
 570async fn randomly_mutate_active_call(
 571    client: &mut TestClient,
 572    can_hang_up: bool,
 573    rng: &Mutex<StdRng>,
 574    cx: &mut TestAppContext,
 575) -> Result<()> {
 576    let active_call = cx.read(ActiveCall::global);
 577    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 578        if rng.lock().gen_bool(0.7) {
 579            log::info!("{}: accepting incoming call", client.username);
 580            active_call
 581                .update(cx, |call, cx| call.accept_incoming(cx))
 582                .await?;
 583        } else {
 584            log::info!("{}: declining incoming call", client.username);
 585            active_call.update(cx, |call, _| call.decline_incoming())?;
 586        }
 587    } else {
 588        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 589            user_store
 590                .contacts()
 591                .iter()
 592                .filter(|contact| contact.online && !contact.busy)
 593                .cloned()
 594                .collect::<Vec<_>>()
 595        });
 596
 597        let distribution = rng.lock().gen_range(0..100);
 598        match distribution {
 599            0..=29 if !available_contacts.is_empty() => {
 600                let contact = available_contacts.choose(&mut *rng.lock()).unwrap();
 601                log::info!(
 602                    "{}: inviting {}",
 603                    client.username,
 604                    contact.user.github_login
 605                );
 606                active_call
 607                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 608                    .await?;
 609            }
 610            30..=39
 611                if can_hang_up && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 612            {
 613                log::info!("{}: hanging up", client.username);
 614                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 615            }
 616            _ => {}
 617        }
 618    }
 619
 620    Ok(())
 621}
 622
 623async fn randomly_mutate_git(client: &mut TestClient, rng: &Mutex<StdRng>) {
 624    let directories = client.fs.directories().await;
 625    let mut dir_path = directories.choose(&mut *rng.lock()).unwrap().clone();
 626    if dir_path.file_name() == Some(OsStr::new(".git")) {
 627        dir_path.pop();
 628    }
 629    let mut git_dir_path = dir_path.clone();
 630    git_dir_path.push(".git");
 631
 632    if !directories.contains(&git_dir_path) {
 633        log::info!(
 634            "{}: creating git directory at {:?}",
 635            client.username,
 636            git_dir_path
 637        );
 638        client.fs.create_dir(&git_dir_path).await.unwrap();
 639    }
 640
 641    let mut child_paths = client.fs.read_dir(&dir_path).await.unwrap();
 642    let mut child_file_paths = Vec::new();
 643    while let Some(child_path) = child_paths.next().await {
 644        let child_path = child_path.unwrap();
 645        if client.fs.is_file(&child_path).await {
 646            child_file_paths.push(child_path);
 647        }
 648    }
 649    let count = rng.lock().gen_range(0..=child_file_paths.len());
 650    child_file_paths.shuffle(&mut *rng.lock());
 651    child_file_paths.truncate(count);
 652
 653    let mut new_index = Vec::new();
 654    for abs_child_file_path in &child_file_paths {
 655        let child_file_path = abs_child_file_path.strip_prefix(&dir_path).unwrap();
 656        let new_base = Alphanumeric.sample_string(&mut *rng.lock(), 16);
 657        new_index.push((child_file_path, new_base));
 658    }
 659    log::info!(
 660        "{}: updating Git index at {:?}: {:#?}",
 661        client.username,
 662        git_dir_path,
 663        new_index
 664    );
 665    client
 666        .fs
 667        .set_index_for_repo(&git_dir_path, &new_index)
 668        .await;
 669}
 670
 671async fn randomly_mutate_fs(client: &mut TestClient, rng: &Mutex<StdRng>) {
 672    let is_dir = rng.lock().gen::<bool>();
 673    let mut new_path = client
 674        .fs
 675        .directories()
 676        .await
 677        .choose(&mut *rng.lock())
 678        .unwrap()
 679        .clone();
 680    new_path.push(gen_file_name(rng));
 681    if is_dir {
 682        log::info!("{}: creating local dir at {:?}", client.username, new_path);
 683        client.fs.create_dir(&new_path).await.unwrap();
 684    } else {
 685        new_path.set_extension("rs");
 686        log::info!("{}: creating local file at {:?}", client.username, new_path);
 687        client
 688            .fs
 689            .create_file(&new_path, Default::default())
 690            .await
 691            .unwrap();
 692    }
 693}
 694
 695async fn randomly_mutate_projects(
 696    client: &mut TestClient,
 697    rng: &Mutex<StdRng>,
 698    cx: &mut TestAppContext,
 699) -> Result<()> {
 700    let active_call = cx.read(ActiveCall::global);
 701    let remote_projects =
 702        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 703            room.read_with(cx, |room, _| {
 704                room.remote_participants()
 705                    .values()
 706                    .flat_map(|participant| participant.projects.clone())
 707                    .collect::<Vec<_>>()
 708            })
 709        } else {
 710            Default::default()
 711        };
 712
 713    let project = if remote_projects.is_empty() || rng.lock().gen() {
 714        if client.local_projects.is_empty() || rng.lock().gen() {
 715            let paths = client.fs.paths().await;
 716            let local_project = if paths.is_empty() || rng.lock().gen() {
 717                let root_path = client.create_new_root_dir();
 718                client.fs.create_dir(&root_path).await.unwrap();
 719                client
 720                    .fs
 721                    .create_file(&root_path.join("main.rs"), Default::default())
 722                    .await
 723                    .unwrap();
 724                log::info!(
 725                    "{}: opening local project at {:?}",
 726                    client.username,
 727                    root_path
 728                );
 729                client.build_local_project(root_path, cx).await.0
 730            } else {
 731                let root_path = paths.choose(&mut *rng.lock()).unwrap();
 732                log::info!(
 733                    "{}: opening local project at {:?}",
 734                    client.username,
 735                    root_path
 736                );
 737                client.build_local_project(root_path, cx).await.0
 738            };
 739            client.local_projects.push(local_project.clone());
 740            local_project
 741        } else {
 742            client
 743                .local_projects
 744                .choose(&mut *rng.lock())
 745                .unwrap()
 746                .clone()
 747        }
 748    } else {
 749        if client.remote_projects.is_empty() || rng.lock().gen() {
 750            let remote_project_id = remote_projects.choose(&mut *rng.lock()).unwrap().id;
 751            let remote_project = if let Some(project) =
 752                client.remote_projects.iter().find(|project| {
 753                    project.read_with(cx, |project, _| {
 754                        project.remote_id() == Some(remote_project_id)
 755                    })
 756                }) {
 757                project.clone()
 758            } else {
 759                log::info!(
 760                    "{}: opening remote project {}",
 761                    client.username,
 762                    remote_project_id
 763                );
 764                let call = cx.read(ActiveCall::global);
 765                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 766                let remote_project = room
 767                    .update(cx, |room, cx| {
 768                        room.join_project(
 769                            remote_project_id,
 770                            client.language_registry.clone(),
 771                            FakeFs::new(cx.background().clone()),
 772                            cx,
 773                        )
 774                    })
 775                    .await?;
 776                client.remote_projects.push(remote_project.clone());
 777                remote_project
 778            };
 779
 780            remote_project
 781        } else {
 782            client
 783                .remote_projects
 784                .choose(&mut *rng.lock())
 785                .unwrap()
 786                .clone()
 787        }
 788    };
 789
 790    if active_call.read_with(cx, |call, _| call.room().is_some())
 791        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 792    {
 793        match active_call
 794            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 795            .await
 796        {
 797            Ok(project_id) => {
 798                log::info!("{}: shared project with id {}", client.username, project_id);
 799            }
 800            Err(error) => {
 801                log::error!("{}: error sharing project, {:?}", client.username, error);
 802            }
 803        }
 804    }
 805
 806    let choice = rng.lock().gen_range(0..100);
 807    match choice {
 808        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 809            let paths = client.fs.paths().await;
 810            let path = paths.choose(&mut *rng.lock()).unwrap();
 811            log::info!(
 812                "{}: finding/creating local worktree for path {:?}",
 813                client.username,
 814                path
 815            );
 816            project
 817                .update(cx, |project, cx| {
 818                    project.find_or_create_local_worktree(&path, true, cx)
 819                })
 820                .await
 821                .unwrap();
 822        }
 823        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 824            log::info!(
 825                "{}: dropping remote project {}",
 826                client.username,
 827                project.read_with(cx, |project, _| project.remote_id().unwrap())
 828            );
 829
 830            cx.update(|_| {
 831                client
 832                    .remote_projects
 833                    .retain(|remote_project| *remote_project != project);
 834                client.buffers.remove(&project);
 835                drop(project);
 836            });
 837        }
 838        _ => {}
 839    }
 840
 841    Ok(())
 842}
 843
 844async fn randomly_mutate_worktrees(
 845    client: &mut TestClient,
 846    rng: &Mutex<StdRng>,
 847    cx: &mut TestAppContext,
 848) -> Result<()> {
 849    let project = choose_random_project(client, rng).unwrap();
 850    let Some(worktree) = project.read_with(cx, |project, cx| {
 851        project
 852            .worktrees(cx)
 853            .filter(|worktree| {
 854                let worktree = worktree.read(cx);
 855                worktree.is_visible()
 856                    && worktree.entries(false).any(|e| e.is_file())
 857                    && worktree.root_entry().map_or(false, |e| e.is_dir())
 858            })
 859            .choose(&mut *rng.lock())
 860    }) else {
 861        return Ok(())
 862    };
 863
 864    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
 865        (worktree.id(), worktree.root_name().to_string())
 866    });
 867
 868    let is_dir = rng.lock().gen::<bool>();
 869    let mut new_path = PathBuf::new();
 870    new_path.push(gen_file_name(rng));
 871    if !is_dir {
 872        new_path.set_extension("rs");
 873    }
 874    log::info!(
 875        "{}: creating {:?} in worktree {} ({})",
 876        client.username,
 877        new_path,
 878        worktree_id,
 879        worktree_root_name,
 880    );
 881    project
 882        .update(cx, |project, cx| {
 883            project.create_entry((worktree_id, new_path), is_dir, cx)
 884        })
 885        .unwrap()
 886        .await?;
 887    Ok(())
 888}
 889
 890async fn randomly_query_and_mutate_buffers(
 891    client: &mut TestClient,
 892    rng: &Mutex<StdRng>,
 893    cx: &mut TestAppContext,
 894) -> Result<()> {
 895    let project = choose_random_project(client, rng).unwrap();
 896    let buffers = client.buffers.entry(project.clone()).or_default();
 897    let buffer = if buffers.is_empty() || rng.lock().gen() {
 898        let Some(worktree) = project.read_with(cx, |project, cx| {
 899            project
 900                .worktrees(cx)
 901                .filter(|worktree| {
 902                    let worktree = worktree.read(cx);
 903                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
 904                })
 905                .choose(&mut *rng.lock())
 906        }) else {
 907            return Ok(());
 908        };
 909
 910        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
 911            let entry = worktree
 912                .entries(false)
 913                .filter(|e| e.is_file())
 914                .choose(&mut *rng.lock())
 915                .unwrap();
 916            (
 917                worktree.root_name().to_string(),
 918                (worktree.id(), entry.path.clone()),
 919            )
 920        });
 921        log::info!(
 922            "{}: opening path {:?} in worktree {} ({})",
 923            client.username,
 924            project_path.1,
 925            project_path.0,
 926            worktree_root_name,
 927        );
 928        let buffer = project
 929            .update(cx, |project, cx| {
 930                project.open_buffer(project_path.clone(), cx)
 931            })
 932            .await?;
 933        log::info!(
 934            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
 935            client.username,
 936            project_path.1,
 937            project_path.0,
 938            worktree_root_name,
 939            buffer.read_with(cx, |buffer, _| buffer.remote_id())
 940        );
 941        buffers.insert(buffer.clone());
 942        buffer
 943    } else {
 944        buffers.iter().choose(&mut *rng.lock()).unwrap().clone()
 945    };
 946
 947    let choice = rng.lock().gen_range(0..100);
 948    match choice {
 949        0..=9 => {
 950            cx.update(|cx| {
 951                log::info!(
 952                    "{}: dropping buffer {:?}",
 953                    client.username,
 954                    buffer.read(cx).file().unwrap().full_path(cx)
 955                );
 956                buffers.remove(&buffer);
 957                drop(buffer);
 958            });
 959        }
 960        10..=19 => {
 961            let completions = project.update(cx, |project, cx| {
 962                log::info!(
 963                    "{}: requesting completions for buffer {} ({:?})",
 964                    client.username,
 965                    buffer.read(cx).remote_id(),
 966                    buffer.read(cx).file().unwrap().full_path(cx)
 967                );
 968                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
 969                project.completions(&buffer, offset, cx)
 970            });
 971            let completions = cx.background().spawn(async move {
 972                completions
 973                    .await
 974                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
 975            });
 976            if rng.lock().gen_bool(0.3) {
 977                log::info!("{}: detaching completions request", client.username);
 978                cx.update(|cx| completions.detach_and_log_err(cx));
 979            } else {
 980                completions.await?;
 981            }
 982        }
 983        20..=29 => {
 984            let code_actions = project.update(cx, |project, cx| {
 985                log::info!(
 986                    "{}: requesting code actions for buffer {} ({:?})",
 987                    client.username,
 988                    buffer.read(cx).remote_id(),
 989                    buffer.read(cx).file().unwrap().full_path(cx)
 990                );
 991                let range = buffer.read(cx).random_byte_range(0, &mut *rng.lock());
 992                project.code_actions(&buffer, range, cx)
 993            });
 994            let code_actions = cx.background().spawn(async move {
 995                code_actions
 996                    .await
 997                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
 998            });
 999            if rng.lock().gen_bool(0.3) {
1000                log::info!("{}: detaching code actions request", client.username);
1001                cx.update(|cx| code_actions.detach_and_log_err(cx));
1002            } else {
1003                code_actions.await?;
1004            }
1005        }
1006        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1007            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1008                log::info!(
1009                    "{}: saving buffer {} ({:?})",
1010                    client.username,
1011                    buffer.remote_id(),
1012                    buffer.file().unwrap().full_path(cx)
1013                );
1014                (buffer.version(), buffer.save(cx))
1015            });
1016            let save = cx.background().spawn(async move {
1017                let (saved_version, _, _) = save
1018                    .await
1019                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1020                assert!(saved_version.observed_all(&requested_version));
1021                Ok::<_, anyhow::Error>(())
1022            });
1023            if rng.lock().gen_bool(0.3) {
1024                log::info!("{}: detaching save request", client.username);
1025                cx.update(|cx| save.detach_and_log_err(cx));
1026            } else {
1027                save.await?;
1028            }
1029        }
1030        40..=44 => {
1031            let prepare_rename = project.update(cx, |project, cx| {
1032                log::info!(
1033                    "{}: preparing rename for buffer {} ({:?})",
1034                    client.username,
1035                    buffer.read(cx).remote_id(),
1036                    buffer.read(cx).file().unwrap().full_path(cx)
1037                );
1038                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1039                project.prepare_rename(buffer, offset, cx)
1040            });
1041            let prepare_rename = cx.background().spawn(async move {
1042                prepare_rename
1043                    .await
1044                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1045            });
1046            if rng.lock().gen_bool(0.3) {
1047                log::info!("{}: detaching prepare rename request", client.username);
1048                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1049            } else {
1050                prepare_rename.await?;
1051            }
1052        }
1053        45..=49 => {
1054            let definitions = project.update(cx, |project, cx| {
1055                log::info!(
1056                    "{}: requesting definitions for buffer {} ({:?})",
1057                    client.username,
1058                    buffer.read(cx).remote_id(),
1059                    buffer.read(cx).file().unwrap().full_path(cx)
1060                );
1061                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1062                project.definition(&buffer, offset, cx)
1063            });
1064            let definitions = cx.background().spawn(async move {
1065                definitions
1066                    .await
1067                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1068            });
1069            if rng.lock().gen_bool(0.3) {
1070                log::info!("{}: detaching definitions request", client.username);
1071                cx.update(|cx| definitions.detach_and_log_err(cx));
1072            } else {
1073                buffers.extend(definitions.await?.into_iter().map(|loc| loc.target.buffer));
1074            }
1075        }
1076        50..=54 => {
1077            let highlights = project.update(cx, |project, cx| {
1078                log::info!(
1079                    "{}: requesting highlights for buffer {} ({:?})",
1080                    client.username,
1081                    buffer.read(cx).remote_id(),
1082                    buffer.read(cx).file().unwrap().full_path(cx)
1083                );
1084                let offset = rng.lock().gen_range(0..=buffer.read(cx).len());
1085                project.document_highlights(&buffer, offset, cx)
1086            });
1087            let highlights = cx.background().spawn(async move {
1088                highlights
1089                    .await
1090                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1091            });
1092            if rng.lock().gen_bool(0.3) {
1093                log::info!("{}: detaching highlights request", client.username);
1094                cx.update(|cx| highlights.detach_and_log_err(cx));
1095            } else {
1096                highlights.await?;
1097            }
1098        }
1099        55..=59 => {
1100            let search = project.update(cx, |project, cx| {
1101                let query = rng.lock().gen_range('a'..='z');
1102                log::info!("{}: project-wide search {:?}", client.username, query);
1103                project.search(SearchQuery::text(query, false, false), cx)
1104            });
1105            let search = cx.background().spawn(async move {
1106                search
1107                    .await
1108                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1109            });
1110            if rng.lock().gen_bool(0.3) {
1111                log::info!("{}: detaching search request", client.username);
1112                cx.update(|cx| search.detach_and_log_err(cx));
1113            } else {
1114                buffers.extend(search.await?.into_keys());
1115            }
1116        }
1117        _ => {
1118            buffer.update(cx, |buffer, cx| {
1119                log::info!(
1120                    "{}: updating buffer {} ({:?})",
1121                    client.username,
1122                    buffer.remote_id(),
1123                    buffer.file().unwrap().full_path(cx)
1124                );
1125                if rng.lock().gen_bool(0.7) {
1126                    buffer.randomly_edit(&mut *rng.lock(), 5, cx);
1127                } else {
1128                    buffer.randomly_undo_redo(&mut *rng.lock(), cx);
1129                }
1130            });
1131        }
1132    }
1133
1134    Ok(())
1135}
1136
1137fn choose_random_project(
1138    client: &mut TestClient,
1139    rng: &Mutex<StdRng>,
1140) -> Option<ModelHandle<Project>> {
1141    client
1142        .local_projects
1143        .iter()
1144        .chain(&client.remote_projects)
1145        .choose(&mut *rng.lock())
1146        .cloned()
1147}
1148
1149fn gen_file_name(rng: &Mutex<StdRng>) -> String {
1150    let mut name = String::new();
1151    for _ in 0..10 {
1152        let letter = rng.lock().gen_range('a'..='z');
1153        name.push(letter);
1154    }
1155    name
1156}