randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::{FakeFs, Fs as _};
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::prelude::*;
  18use std::{env, path::PathBuf, rc::Rc, sync::Arc};
  19
  20struct TestPlan {
  21    rng: StdRng,
  22    allow_server_restarts: bool,
  23    allow_client_reconnection: bool,
  24    allow_client_disconnection: bool,
  25}
  26
  27#[derive(Debug)]
  28enum Operation {
  29    AddConnection {
  30        user_id: UserId,
  31    },
  32    RemoveConnection {
  33        user_id: UserId,
  34    },
  35    BounceConnection {
  36        user_id: UserId,
  37    },
  38    RestartServer,
  39    RunUntilParked,
  40    MutateClient {
  41        user_id: UserId,
  42        operation: ClientOperation,
  43    },
  44}
  45
  46#[derive(Debug)]
  47enum ClientOperation {
  48    AcceptIncomingCall,
  49    RejectIncomingCall,
  50    LeaveCall,
  51    InviteContactToCall {
  52        user_id: UserId,
  53    },
  54    OpenLocalProject {
  55        first_root_path: PathBuf,
  56    },
  57    OpenRemoteProject {
  58        host_id: UserId,
  59        first_root_name: String,
  60    },
  61    AddWorktreeToProject {
  62        first_root_path: PathBuf,
  63        new_root_path: PathBuf,
  64    },
  65    CloseProject {
  66        id: u64,
  67    },
  68}
  69
  70impl TestPlan {
  71    async fn next_operation(
  72        &mut self,
  73        clients: &[(Rc<TestClient>, TestAppContext)],
  74        offline_users: &[(UserId, String)],
  75    ) -> Operation {
  76        let operation = loop {
  77            break match self.rng.gen_range(0..100) {
  78                0..=9 if !offline_users.is_empty() => {
  79                    let user_id = offline_users[self.rng.gen_range(0..offline_users.len())].0;
  80                    Operation::AddConnection { user_id }
  81                }
  82                10..=14 if clients.len() > 1 && self.allow_client_disconnection => {
  83                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
  84                    let user_id = client.current_user_id(cx);
  85                    Operation::RemoveConnection { user_id }
  86                }
  87                15..=19 if clients.len() > 1 && self.allow_client_reconnection => {
  88                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
  89                    let user_id = client.current_user_id(cx);
  90                    Operation::BounceConnection { user_id }
  91                }
  92                20..=24 if self.allow_server_restarts => Operation::RestartServer,
  93                25..=29 => Operation::RunUntilParked,
  94                _ if !clients.is_empty() => {
  95                    let ix = self.rng.gen_range(0..clients.len());
  96                    let (client, cx) = &clients[ix];
  97                    let user_id = client.current_user_id(cx);
  98                    let operation = self.next_client_operation(clients, ix).await;
  99                    Operation::MutateClient { user_id, operation }
 100                }
 101                _ => continue,
 102            };
 103        };
 104        operation
 105    }
 106
 107    async fn next_client_operation(
 108        &mut self,
 109        clients: &[(Rc<TestClient>, TestAppContext)],
 110        client_ix: usize,
 111    ) -> ClientOperation {
 112        let (client, cx) = &clients[client_ix];
 113        let call = cx.read(ActiveCall::global);
 114
 115        loop {
 116            match self.rng.gen_range(0..100) {
 117                // Respond to an incoming call
 118                0..=19 => {
 119                    if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 120                        return if self.rng.gen_bool(0.7) {
 121                            ClientOperation::AcceptIncomingCall
 122                        } else {
 123                            ClientOperation::RejectIncomingCall
 124                        };
 125                    }
 126                }
 127
 128                // Invite a contact to the current call
 129                20..=29 => {
 130                    let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 131                        user_store
 132                            .contacts()
 133                            .iter()
 134                            .filter(|contact| contact.online && !contact.busy)
 135                            .cloned()
 136                            .collect::<Vec<_>>()
 137                    });
 138                    if !available_contacts.is_empty() {
 139                        let contact = available_contacts.choose(&mut self.rng).unwrap();
 140                        return ClientOperation::InviteContactToCall {
 141                            user_id: UserId(contact.user.id as i32),
 142                        };
 143                    }
 144                }
 145
 146                // Leave the current call
 147                30..=39 => {
 148                    if self.allow_client_disconnection
 149                        && call.read_with(cx, |call, _| call.room().is_some())
 150                    {
 151                        return ClientOperation::LeaveCall;
 152                    }
 153                }
 154
 155                // Open a remote project
 156                40..=49 => {
 157                    if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
 158                        let remote_projects = room.read_with(cx, |room, _| {
 159                            room.remote_participants()
 160                                .values()
 161                                .flat_map(|participant| {
 162                                    participant.projects.iter().map(|project| {
 163                                        (
 164                                            UserId::from_proto(participant.user.id),
 165                                            project.worktree_root_names[0].clone(),
 166                                        )
 167                                    })
 168                                })
 169                                .collect::<Vec<_>>()
 170                        });
 171                        if !remote_projects.is_empty() {
 172                            let (host_id, first_root_name) =
 173                                remote_projects.choose(&mut self.rng).unwrap().clone();
 174                            return ClientOperation::OpenRemoteProject {
 175                                host_id,
 176                                first_root_name,
 177                            };
 178                        }
 179                    }
 180                }
 181
 182                // Open a local project
 183                50..=59 => {
 184                    let paths = client.fs.paths().await;
 185                    let first_root_path = if paths.is_empty() || self.rng.gen() {
 186                        client.create_new_root_dir()
 187                    } else {
 188                        paths.choose(&mut self.rng).unwrap().clone()
 189                    };
 190                    return ClientOperation::OpenLocalProject { first_root_path };
 191                }
 192
 193                // Add a worktree to a local project
 194                60..=69 if !client.local_projects().is_empty() => {
 195                    let project = client
 196                        .local_projects()
 197                        .choose(&mut self.rng)
 198                        .unwrap()
 199                        .clone();
 200
 201                    let first_root_path = project.read_with(cx, |project, cx| {
 202                        project
 203                            .visible_worktrees(cx)
 204                            .next()
 205                            .unwrap()
 206                            .read(cx)
 207                            .abs_path()
 208                            .to_path_buf()
 209                    });
 210
 211                    let paths = client.fs.paths().await;
 212                    let new_root_path = if paths.is_empty() || self.rng.gen() {
 213                        client.create_new_root_dir()
 214                    } else {
 215                        paths.choose(&mut self.rng).unwrap().clone()
 216                    };
 217
 218                    return ClientOperation::AddWorktreeToProject {
 219                        first_root_path,
 220                        new_root_path,
 221                    };
 222                }
 223
 224                _ => continue,
 225            };
 226        }
 227    }
 228}
 229
 230#[gpui::test(iterations = 100)]
 231async fn test_random_collaboration(
 232    cx: &mut TestAppContext,
 233    deterministic: Arc<Deterministic>,
 234    mut rng: StdRng,
 235) {
 236    deterministic.forbid_parking();
 237
 238    let max_peers = env::var("MAX_PEERS")
 239        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
 240        .unwrap_or(5);
 241
 242    let max_operations = env::var("OPERATIONS")
 243        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
 244        .unwrap_or(10);
 245
 246    let mut server = TestServer::start(&deterministic).await;
 247    let db = server.app_state.db.clone();
 248
 249    let mut available_users = Vec::new();
 250    for ix in 0..max_peers {
 251        let username = format!("user-{}", ix + 1);
 252        let user_id = db
 253            .create_user(
 254                &format!("{username}@example.com"),
 255                false,
 256                NewUserParams {
 257                    github_login: username.clone(),
 258                    github_user_id: (ix + 1) as i32,
 259                    invite_count: 0,
 260                },
 261            )
 262            .await
 263            .unwrap()
 264            .user_id;
 265        available_users.push((user_id, username));
 266    }
 267
 268    let plan = Arc::new(Mutex::new(TestPlan {
 269        allow_server_restarts: rng.gen_bool(0.7),
 270        allow_client_reconnection: rng.gen_bool(0.7),
 271        allow_client_disconnection: rng.gen_bool(0.1),
 272        rng,
 273    }));
 274
 275    for (ix, (user_id_a, _)) in available_users.iter().enumerate() {
 276        for (user_id_b, _) in &available_users[ix + 1..] {
 277            server
 278                .app_state
 279                .db
 280                .send_contact_request(*user_id_a, *user_id_b)
 281                .await
 282                .unwrap();
 283            server
 284                .app_state
 285                .db
 286                .respond_to_contact_request(*user_id_b, *user_id_a, true)
 287                .await
 288                .unwrap();
 289        }
 290    }
 291
 292    let mut clients = Vec::new();
 293    let mut client_tasks = Vec::new();
 294    let mut op_start_signals = Vec::new();
 295    let mut next_entity_id = 100000;
 296
 297    for _ in 0..max_operations {
 298        let next_operation = plan.lock().next_operation(&clients, &available_users).await;
 299        match next_operation {
 300            Operation::AddConnection { user_id } => {
 301                let user_ix = available_users
 302                    .iter()
 303                    .position(|(id, _)| *id == user_id)
 304                    .unwrap();
 305                let (_, username) = available_users.remove(user_ix);
 306                log::info!("Adding new connection for {}", username);
 307                next_entity_id += 100000;
 308                let mut client_cx = TestAppContext::new(
 309                    cx.foreground_platform(),
 310                    cx.platform(),
 311                    deterministic.build_foreground(next_entity_id),
 312                    deterministic.build_background(),
 313                    cx.font_cache(),
 314                    cx.leak_detector(),
 315                    next_entity_id,
 316                    cx.function_name.clone(),
 317                );
 318
 319                let op_start_signal = futures::channel::mpsc::unbounded();
 320                let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 321                op_start_signals.push(op_start_signal.0);
 322                clients.push((client.clone(), client_cx.clone()));
 323                client_tasks.push(client_cx.foreground().spawn(simulate_client(
 324                    client,
 325                    op_start_signal.1,
 326                    plan.clone(),
 327                    client_cx,
 328                )));
 329
 330                log::info!("Added connection for {}", username);
 331            }
 332
 333            Operation::RemoveConnection { user_id } => {
 334                log::info!("Simulating full disconnection of user {}", user_id);
 335                let client_ix = clients
 336                    .iter()
 337                    .position(|(client, cx)| client.current_user_id(cx) == user_id)
 338                    .unwrap();
 339                let user_connection_ids = server
 340                    .connection_pool
 341                    .lock()
 342                    .user_connection_ids(user_id)
 343                    .collect::<Vec<_>>();
 344                assert_eq!(user_connection_ids.len(), 1);
 345                let removed_peer_id = user_connection_ids[0].into();
 346                let (client, mut client_cx) = clients.remove(client_ix);
 347                let client_task = client_tasks.remove(client_ix);
 348                op_start_signals.remove(client_ix);
 349                server.forbid_connections();
 350                server.disconnect_client(removed_peer_id);
 351                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 352                deterministic.start_waiting();
 353                log::info!("Waiting for user {} to exit...", user_id);
 354                client_task.await;
 355                deterministic.finish_waiting();
 356                server.allow_connections();
 357
 358                for project in client.remote_projects().iter() {
 359                    project.read_with(&client_cx, |project, _| {
 360                        assert!(
 361                            project.is_read_only(),
 362                            "project {:?} should be read only",
 363                            project.remote_id()
 364                        )
 365                    });
 366                }
 367
 368                for (client, cx) in &clients {
 369                    let contacts = server
 370                        .app_state
 371                        .db
 372                        .get_contacts(client.current_user_id(cx))
 373                        .await
 374                        .unwrap();
 375                    let pool = server.connection_pool.lock();
 376                    for contact in contacts {
 377                        if let db::Contact::Accepted { user_id: id, .. } = contact {
 378                            if pool.is_user_online(id) {
 379                                assert_ne!(
 380                                    id, user_id,
 381                                    "removed client is still a contact of another peer"
 382                                );
 383                            }
 384                        }
 385                    }
 386                }
 387
 388                log::info!("{} removed", client.username);
 389                available_users.push((user_id, client.username.clone()));
 390                client_cx.update(|cx| {
 391                    cx.clear_globals();
 392                    drop(client);
 393                });
 394            }
 395
 396            Operation::BounceConnection { user_id } => {
 397                log::info!("Simulating temporary disconnection of user {}", user_id);
 398                let user_connection_ids = server
 399                    .connection_pool
 400                    .lock()
 401                    .user_connection_ids(user_id)
 402                    .collect::<Vec<_>>();
 403                assert_eq!(user_connection_ids.len(), 1);
 404                let peer_id = user_connection_ids[0].into();
 405                server.disconnect_client(peer_id);
 406                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 407            }
 408
 409            Operation::RestartServer => {
 410                log::info!("Simulating server restart");
 411                server.reset().await;
 412                deterministic.advance_clock(RECEIVE_TIMEOUT);
 413                server.start().await.unwrap();
 414                deterministic.advance_clock(CLEANUP_TIMEOUT);
 415                let environment = &server.app_state.config.zed_environment;
 416                let stale_room_ids = server
 417                    .app_state
 418                    .db
 419                    .stale_room_ids(environment, server.id())
 420                    .await
 421                    .unwrap();
 422                assert_eq!(stale_room_ids, vec![]);
 423            }
 424
 425            Operation::RunUntilParked => {
 426                deterministic.run_until_parked();
 427            }
 428
 429            Operation::MutateClient { user_id, operation } => {
 430                let client_ix = clients
 431                    .iter()
 432                    .position(|(client, cx)| client.current_user_id(cx) == user_id)
 433                    .unwrap();
 434                op_start_signals[client_ix]
 435                    .unbounded_send(operation)
 436                    .unwrap();
 437            }
 438        }
 439    }
 440
 441    drop(op_start_signals);
 442    deterministic.start_waiting();
 443    futures::future::join_all(client_tasks).await;
 444    deterministic.finish_waiting();
 445    deterministic.run_until_parked();
 446
 447    for (client, client_cx) in &clients {
 448        for guest_project in client.remote_projects().iter() {
 449            guest_project.read_with(client_cx, |guest_project, cx| {
 450                let host_project = clients.iter().find_map(|(client, cx)| {
 451                    let project = client
 452                        .local_projects()
 453                        .iter()
 454                        .find(|host_project| {
 455                            host_project.read_with(cx, |host_project, _| {
 456                                host_project.remote_id() == guest_project.remote_id()
 457                            })
 458                        })?
 459                        .clone();
 460                    Some((project, cx))
 461                });
 462
 463                if !guest_project.is_read_only() {
 464                    if let Some((host_project, host_cx)) = host_project {
 465                        let host_worktree_snapshots =
 466                            host_project.read_with(host_cx, |host_project, cx| {
 467                                host_project
 468                                    .worktrees(cx)
 469                                    .map(|worktree| {
 470                                        let worktree = worktree.read(cx);
 471                                        (worktree.id(), worktree.snapshot())
 472                                    })
 473                                    .collect::<BTreeMap<_, _>>()
 474                            });
 475                        let guest_worktree_snapshots = guest_project
 476                            .worktrees(cx)
 477                            .map(|worktree| {
 478                                let worktree = worktree.read(cx);
 479                                (worktree.id(), worktree.snapshot())
 480                            })
 481                            .collect::<BTreeMap<_, _>>();
 482
 483                        assert_eq!(
 484                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 485                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 486                            "{} has different worktrees than the host",
 487                            client.username
 488                        );
 489
 490                        for (id, host_snapshot) in &host_worktree_snapshots {
 491                            let guest_snapshot = &guest_worktree_snapshots[id];
 492                            assert_eq!(
 493                                guest_snapshot.root_name(),
 494                                host_snapshot.root_name(),
 495                                "{} has different root name than the host for worktree {}",
 496                                client.username,
 497                                id
 498                            );
 499                            assert_eq!(
 500                                guest_snapshot.abs_path(),
 501                                host_snapshot.abs_path(),
 502                                "{} has different abs path than the host for worktree {}",
 503                                client.username,
 504                                id
 505                            );
 506                            assert_eq!(
 507                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 508                                host_snapshot.entries(false).collect::<Vec<_>>(),
 509                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 510                                client.username,
 511                                id,
 512                                host_snapshot.abs_path(),
 513                                host_project.read_with(host_cx, |project, _| project.remote_id())
 514                            );
 515                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 516                        }
 517                    }
 518                }
 519
 520                guest_project.check_invariants(cx);
 521            });
 522        }
 523
 524        let buffers = client.buffers().clone();
 525        for (guest_project, guest_buffers) in &buffers {
 526            let project_id = if guest_project.read_with(client_cx, |project, _| {
 527                project.is_local() || project.is_read_only()
 528            }) {
 529                continue;
 530            } else {
 531                guest_project
 532                    .read_with(client_cx, |project, _| project.remote_id())
 533                    .unwrap()
 534            };
 535            let guest_user_id = client.user_id().unwrap();
 536
 537            let host_project = clients.iter().find_map(|(client, cx)| {
 538                let project = client
 539                    .local_projects()
 540                    .iter()
 541                    .find(|host_project| {
 542                        host_project.read_with(cx, |host_project, _| {
 543                            host_project.remote_id() == Some(project_id)
 544                        })
 545                    })?
 546                    .clone();
 547                Some((client.user_id().unwrap(), project, cx))
 548            });
 549
 550            let (host_user_id, host_project, host_cx) =
 551                if let Some((host_user_id, host_project, host_cx)) = host_project {
 552                    (host_user_id, host_project, host_cx)
 553                } else {
 554                    continue;
 555                };
 556
 557            for guest_buffer in guest_buffers {
 558                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 559                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 560                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 561                        panic!(
 562                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 563                            client.username,
 564                            client.peer_id(),
 565                            buffer_id
 566                        )
 567                    })
 568                });
 569                let path = host_buffer
 570                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 571
 572                assert_eq!(
 573                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 574                    0,
 575                    "{}, buffer {}, path {:?} has deferred operations",
 576                    client.username,
 577                    buffer_id,
 578                    path,
 579                );
 580                assert_eq!(
 581                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 582                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 583                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 584                    client.username,
 585                    buffer_id,
 586                    path
 587                );
 588
 589                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 590                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 591                match (host_file, guest_file) {
 592                    (Some(host_file), Some(guest_file)) => {
 593                        assert_eq!(guest_file.path(), host_file.path());
 594                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 595                        assert_eq!(
 596                            guest_file.mtime(),
 597                            host_file.mtime(),
 598                            "guest {} mtime does not match host {} for path {:?} in project {}",
 599                            guest_user_id,
 600                            host_user_id,
 601                            guest_file.path(),
 602                            project_id,
 603                        );
 604                    }
 605                    (None, None) => {}
 606                    (None, _) => panic!("host's file is None, guest's isn't "),
 607                    (_, None) => panic!("guest's file is None, hosts's isn't "),
 608                }
 609            }
 610        }
 611    }
 612
 613    for (client, mut cx) in clients {
 614        cx.update(|cx| {
 615            cx.clear_globals();
 616            drop(client);
 617        });
 618    }
 619}
 620
 621async fn simulate_client(
 622    client: Rc<TestClient>,
 623    mut op_start_signal: futures::channel::mpsc::UnboundedReceiver<ClientOperation>,
 624    plan: Arc<Mutex<TestPlan>>,
 625    mut cx: TestAppContext,
 626) {
 627    // Setup language server
 628    let mut language = Language::new(
 629        LanguageConfig {
 630            name: "Rust".into(),
 631            path_suffixes: vec!["rs".to_string()],
 632            ..Default::default()
 633        },
 634        None,
 635    );
 636    let _fake_language_servers = language
 637        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 638            name: "the-fake-language-server",
 639            capabilities: lsp::LanguageServer::full_capabilities(),
 640            initializer: Some(Box::new({
 641                let plan = plan.clone();
 642                let fs = client.fs.clone();
 643                move |fake_server: &mut FakeLanguageServer| {
 644                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 645                        |_, _| async move {
 646                            Ok(Some(lsp::CompletionResponse::Array(vec![
 647                                lsp::CompletionItem {
 648                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 649                                        range: lsp::Range::new(
 650                                            lsp::Position::new(0, 0),
 651                                            lsp::Position::new(0, 0),
 652                                        ),
 653                                        new_text: "the-new-text".to_string(),
 654                                    })),
 655                                    ..Default::default()
 656                                },
 657                            ])))
 658                        },
 659                    );
 660
 661                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 662                        |_, _| async move {
 663                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 664                                lsp::CodeAction {
 665                                    title: "the-code-action".to_string(),
 666                                    ..Default::default()
 667                                },
 668                            )]))
 669                        },
 670                    );
 671
 672                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 673                        |params, _| async move {
 674                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 675                                params.position,
 676                                params.position,
 677                            ))))
 678                        },
 679                    );
 680
 681                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 682                        let fs = fs.clone();
 683                        let plan = plan.clone();
 684                        move |_, _| {
 685                            let fs = fs.clone();
 686                            let plan = plan.clone();
 687                            async move {
 688                                let files = fs.files().await;
 689                                let mut plan = plan.lock();
 690                                let count = plan.rng.gen_range::<usize, _>(1..3);
 691                                let files = (0..count)
 692                                    .map(|_| files.choose(&mut plan.rng).unwrap())
 693                                    .collect::<Vec<_>>();
 694                                log::info!("LSP: Returning definitions in files {:?}", &files);
 695                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 696                                    files
 697                                        .into_iter()
 698                                        .map(|file| lsp::Location {
 699                                            uri: lsp::Url::from_file_path(file).unwrap(),
 700                                            range: Default::default(),
 701                                        })
 702                                        .collect(),
 703                                )))
 704                            }
 705                        }
 706                    });
 707
 708                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 709                        let plan = plan.clone();
 710                        move |_, _| {
 711                            let mut highlights = Vec::new();
 712                            let highlight_count = plan.lock().rng.gen_range(1..=5);
 713                            for _ in 0..highlight_count {
 714                                let start_row = plan.lock().rng.gen_range(0..100);
 715                                let start_column = plan.lock().rng.gen_range(0..100);
 716                                let start = PointUtf16::new(start_row, start_column);
 717                                let end_row = plan.lock().rng.gen_range(0..100);
 718                                let end_column = plan.lock().rng.gen_range(0..100);
 719                                let end = PointUtf16::new(end_row, end_column);
 720                                let range = if start > end { end..start } else { start..end };
 721                                highlights.push(lsp::DocumentHighlight {
 722                                    range: range_to_lsp(range.clone()),
 723                                    kind: Some(lsp::DocumentHighlightKind::READ),
 724                                });
 725                            }
 726                            highlights.sort_unstable_by_key(|highlight| {
 727                                (highlight.range.start, highlight.range.end)
 728                            });
 729                            async move { Ok(Some(highlights)) }
 730                        }
 731                    });
 732                }
 733            })),
 734            ..Default::default()
 735        }))
 736        .await;
 737    client.language_registry.add(Arc::new(language));
 738
 739    while op_start_signal.next().await.is_some() {
 740        if let Err(error) = randomly_mutate_client(&client, plan.clone(), &mut cx).await {
 741            log::error!("{} error: {:?}", client.username, error);
 742        }
 743
 744        cx.background().simulate_random_delay().await;
 745    }
 746    log::info!("{}: done", client.username);
 747}
 748
 749// async fn apply_client_operation(
 750//     client: &mut TestClient,
 751//     plan: Arc<Mutex<TestPlan>>,
 752//     operation: ClientOperation,
 753//     cx: &mut TestAppContext,
 754// ) -> Result<()> {
 755//     match operation {
 756//         ClientOperation::AcceptIncomingCall => todo!(),
 757//         ClientOperation::RejectIncomingCall => todo!(),
 758//         ClientOperation::OpenLocalProject { path } => todo!(),
 759//         ClientOperation::AddWorktreeToProject {
 760//             existing_path,
 761//             new_path,
 762//         } => todo!(),
 763//         ClientOperation::CloseProject { existing_path } => todo!(),
 764//     }
 765// }
 766
 767async fn randomly_mutate_client(
 768    client: &Rc<TestClient>,
 769    plan: Arc<Mutex<TestPlan>>,
 770    cx: &mut TestAppContext,
 771) -> Result<()> {
 772    let choice = plan.lock().rng.gen_range(0..100);
 773    match choice {
 774        0..=19 => randomly_mutate_active_call(client, &plan, cx).await?,
 775        20..=49 => randomly_mutate_projects(client, &plan, cx).await?,
 776        50..=59 if !client.local_projects().is_empty() || !client.remote_projects().is_empty() => {
 777            randomly_mutate_worktrees(client, &plan, cx).await?;
 778        }
 779        60..=84 if !client.local_projects().is_empty() || !client.remote_projects().is_empty() => {
 780            randomly_query_and_mutate_buffers(client, &plan, cx).await?;
 781        }
 782        _ => randomly_mutate_fs(client, &plan).await,
 783    }
 784
 785    Ok(())
 786}
 787
 788async fn randomly_mutate_active_call(
 789    client: &TestClient,
 790    plan: &Arc<Mutex<TestPlan>>,
 791    cx: &mut TestAppContext,
 792) -> Result<()> {
 793    let active_call = cx.read(ActiveCall::global);
 794    if active_call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 795        if plan.lock().rng.gen_bool(0.7) {
 796            log::info!("{}: accepting incoming call", client.username);
 797            active_call
 798                .update(cx, |call, cx| call.accept_incoming(cx))
 799                .await?;
 800        } else {
 801            log::info!("{}: declining incoming call", client.username);
 802            active_call.update(cx, |call, _| call.decline_incoming())?;
 803        }
 804    } else {
 805        let available_contacts = client.user_store.read_with(cx, |user_store, _| {
 806            user_store
 807                .contacts()
 808                .iter()
 809                .filter(|contact| contact.online && !contact.busy)
 810                .cloned()
 811                .collect::<Vec<_>>()
 812        });
 813
 814        let distribution = plan.lock().rng.gen_range(0..100);
 815        match distribution {
 816            0..=29 if !available_contacts.is_empty() => {
 817                let contact = available_contacts.choose(&mut plan.lock().rng).unwrap();
 818                log::info!(
 819                    "{}: inviting {}",
 820                    client.username,
 821                    contact.user.github_login
 822                );
 823                active_call
 824                    .update(cx, |call, cx| call.invite(contact.user.id, None, cx))
 825                    .await?;
 826            }
 827            30..=39
 828                if plan.lock().allow_client_disconnection
 829                    && active_call.read_with(cx, |call, _| call.room().is_some()) =>
 830            {
 831                log::info!("{}: hanging up", client.username);
 832                active_call.update(cx, |call, cx| call.hang_up(cx))?;
 833            }
 834            _ => {}
 835        }
 836    }
 837
 838    Ok(())
 839}
 840
 841async fn randomly_mutate_fs(client: &TestClient, plan: &Arc<Mutex<TestPlan>>) {
 842    let is_dir = plan.lock().rng.gen::<bool>();
 843    let mut new_path = client
 844        .fs
 845        .directories()
 846        .await
 847        .choose(&mut plan.lock().rng)
 848        .unwrap()
 849        .clone();
 850    new_path.push(gen_file_name(&mut plan.lock().rng));
 851    if is_dir {
 852        log::info!("{}: creating local dir at {:?}", client.username, new_path);
 853        client.fs.create_dir(&new_path).await.unwrap();
 854    } else {
 855        new_path.set_extension("rs");
 856        log::info!("{}: creating local file at {:?}", client.username, new_path);
 857        client
 858            .fs
 859            .create_file(&new_path, Default::default())
 860            .await
 861            .unwrap();
 862    }
 863}
 864
 865async fn randomly_mutate_projects(
 866    client: &TestClient,
 867    plan: &Arc<Mutex<TestPlan>>,
 868    cx: &mut TestAppContext,
 869) -> Result<()> {
 870    let active_call = cx.read(ActiveCall::global);
 871    let remote_projects =
 872        if let Some(room) = active_call.read_with(cx, |call, _| call.room().cloned()) {
 873            room.read_with(cx, |room, _| {
 874                room.remote_participants()
 875                    .values()
 876                    .flat_map(|participant| participant.projects.clone())
 877                    .collect::<Vec<_>>()
 878            })
 879        } else {
 880            Default::default()
 881        };
 882
 883    let project = if remote_projects.is_empty() || plan.lock().rng.gen() {
 884        if client.local_projects().is_empty() || plan.lock().rng.gen() {
 885            let paths = client.fs.paths().await;
 886            let local_project = if paths.is_empty() || plan.lock().rng.gen() {
 887                let root_path = client.create_new_root_dir();
 888                client.fs.create_dir(&root_path).await.unwrap();
 889                client
 890                    .fs
 891                    .create_file(&root_path.join("main.rs"), Default::default())
 892                    .await
 893                    .unwrap();
 894                log::info!(
 895                    "{}: opening local project at {:?}",
 896                    client.username,
 897                    root_path
 898                );
 899                client.build_local_project(root_path, cx).await.0
 900            } else {
 901                let root_path = paths.choose(&mut plan.lock().rng).unwrap();
 902                log::info!(
 903                    "{}: opening local project at {:?}",
 904                    client.username,
 905                    root_path
 906                );
 907                client.build_local_project(root_path, cx).await.0
 908            };
 909            client.local_projects_mut().push(local_project.clone());
 910            local_project
 911        } else {
 912            client
 913                .local_projects()
 914                .choose(&mut plan.lock().rng)
 915                .unwrap()
 916                .clone()
 917        }
 918    } else {
 919        if client.remote_projects().is_empty() || plan.lock().rng.gen() {
 920            let remote_project_id = remote_projects.choose(&mut plan.lock().rng).unwrap().id;
 921            let remote_projects = client.remote_projects().clone();
 922            let remote_project = if let Some(project) = remote_projects
 923                .iter()
 924                .find(|project| {
 925                    project.read_with(cx, |project, _| {
 926                        project.remote_id() == Some(remote_project_id)
 927                    })
 928                })
 929                .cloned()
 930            {
 931                project
 932            } else {
 933                log::info!(
 934                    "{}: opening remote project {}",
 935                    client.username,
 936                    remote_project_id
 937                );
 938                let call = cx.read(ActiveCall::global);
 939                let room = call.read_with(cx, |call, _| call.room().unwrap().clone());
 940                let remote_project = room
 941                    .update(cx, |room, cx| {
 942                        room.join_project(
 943                            remote_project_id,
 944                            client.language_registry.clone(),
 945                            FakeFs::new(cx.background().clone()),
 946                            cx,
 947                        )
 948                    })
 949                    .await?;
 950                client.remote_projects_mut().push(remote_project.clone());
 951                remote_project
 952            };
 953
 954            remote_project
 955        } else {
 956            client
 957                .remote_projects()
 958                .choose(&mut plan.lock().rng)
 959                .unwrap()
 960                .clone()
 961        }
 962    };
 963
 964    if active_call.read_with(cx, |call, _| call.room().is_some())
 965        && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 966    {
 967        match active_call
 968            .update(cx, |call, cx| call.share_project(project.clone(), cx))
 969            .await
 970        {
 971            Ok(project_id) => {
 972                log::info!("{}: shared project with id {}", client.username, project_id);
 973            }
 974            Err(error) => {
 975                log::error!("{}: error sharing project, {:?}", client.username, error);
 976            }
 977        }
 978    }
 979
 980    let choice = plan.lock().rng.gen_range(0..100);
 981    match choice {
 982        0..=19 if project.read_with(cx, |project, _| project.is_local()) => {
 983            let paths = client.fs.paths().await;
 984            let path = paths.choose(&mut plan.lock().rng).unwrap();
 985            log::info!(
 986                "{}: finding/creating local worktree for path {:?}",
 987                client.username,
 988                path
 989            );
 990            project
 991                .update(cx, |project, cx| {
 992                    project.find_or_create_local_worktree(&path, true, cx)
 993                })
 994                .await
 995                .unwrap();
 996        }
 997        20..=24 if project.read_with(cx, |project, _| project.is_remote()) => {
 998            log::info!(
 999                "{}: dropping remote project {}",
1000                client.username,
1001                project.read_with(cx, |project, _| project.remote_id().unwrap())
1002            );
1003
1004            cx.update(|_| {
1005                client
1006                    .remote_projects_mut()
1007                    .retain(|remote_project| *remote_project != project);
1008                client.buffers().remove(&project);
1009                drop(project);
1010            });
1011        }
1012        _ => {}
1013    }
1014
1015    Ok(())
1016}
1017
1018async fn randomly_mutate_worktrees(
1019    client: &TestClient,
1020    plan: &Arc<Mutex<TestPlan>>,
1021    cx: &mut TestAppContext,
1022) -> Result<()> {
1023    let project = choose_random_project(client, &mut plan.lock().rng).unwrap();
1024    let Some(worktree) = project.read_with(cx, |project, cx| {
1025        project
1026            .worktrees(cx)
1027            .filter(|worktree| {
1028                let worktree = worktree.read(cx);
1029                worktree.is_visible()
1030                    && worktree.entries(false).any(|e| e.is_file())
1031                    && worktree.root_entry().map_or(false, |e| e.is_dir())
1032            })
1033            .choose(&mut plan.lock().rng)
1034    }) else {
1035        return Ok(())
1036    };
1037
1038    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
1039        (worktree.id(), worktree.root_name().to_string())
1040    });
1041
1042    let is_dir = plan.lock().rng.gen::<bool>();
1043    let mut new_path = PathBuf::new();
1044    new_path.push(gen_file_name(&mut plan.lock().rng));
1045    if !is_dir {
1046        new_path.set_extension("rs");
1047    }
1048    log::info!(
1049        "{}: creating {:?} in worktree {} ({})",
1050        client.username,
1051        new_path,
1052        worktree_id,
1053        worktree_root_name,
1054    );
1055    project
1056        .update(cx, |project, cx| {
1057            project.create_entry((worktree_id, new_path), is_dir, cx)
1058        })
1059        .unwrap()
1060        .await?;
1061    Ok(())
1062}
1063
1064async fn randomly_query_and_mutate_buffers(
1065    client: &TestClient,
1066    plan: &Arc<Mutex<TestPlan>>,
1067    cx: &mut TestAppContext,
1068) -> Result<()> {
1069    let project = choose_random_project(client, &mut plan.lock().rng).unwrap();
1070    let has_buffers_for_project = !client.buffers_for_project(&project).is_empty();
1071    let buffer = if !has_buffers_for_project || plan.lock().rng.gen() {
1072        let Some(worktree) = project.read_with(cx, |project, cx| {
1073            project
1074                .worktrees(cx)
1075                .filter(|worktree| {
1076                    let worktree = worktree.read(cx);
1077                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
1078                })
1079                .choose(&mut plan.lock().rng)
1080        }) else {
1081            return Ok(());
1082        };
1083
1084        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
1085            let entry = worktree
1086                .entries(false)
1087                .filter(|e| e.is_file())
1088                .choose(&mut plan.lock().rng)
1089                .unwrap();
1090            (
1091                worktree.root_name().to_string(),
1092                (worktree.id(), entry.path.clone()),
1093            )
1094        });
1095        log::info!(
1096            "{}: opening path {:?} in worktree {} ({})",
1097            client.username,
1098            project_path.1,
1099            project_path.0,
1100            worktree_root_name,
1101        );
1102        let buffer = project
1103            .update(cx, |project, cx| {
1104                project.open_buffer(project_path.clone(), cx)
1105            })
1106            .await?;
1107        log::info!(
1108            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
1109            client.username,
1110            project_path.1,
1111            project_path.0,
1112            worktree_root_name,
1113            buffer.read_with(cx, |buffer, _| buffer.remote_id())
1114        );
1115        client.buffers_for_project(&project).insert(buffer.clone());
1116        buffer
1117    } else {
1118        client
1119            .buffers_for_project(&project)
1120            .iter()
1121            .choose(&mut plan.lock().rng)
1122            .unwrap()
1123            .clone()
1124    };
1125
1126    let choice = plan.lock().rng.gen_range(0..100);
1127    match choice {
1128        0..=9 => {
1129            cx.update(|cx| {
1130                log::info!(
1131                    "{}: dropping buffer {:?}",
1132                    client.username,
1133                    buffer.read(cx).file().unwrap().full_path(cx)
1134                );
1135                client.buffers_for_project(&project).remove(&buffer);
1136                drop(buffer);
1137            });
1138        }
1139        10..=19 => {
1140            let completions = project.update(cx, |project, cx| {
1141                log::info!(
1142                    "{}: requesting completions for buffer {} ({:?})",
1143                    client.username,
1144                    buffer.read(cx).remote_id(),
1145                    buffer.read(cx).file().unwrap().full_path(cx)
1146                );
1147                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1148                project.completions(&buffer, offset, cx)
1149            });
1150            let completions = cx.background().spawn(async move {
1151                completions
1152                    .await
1153                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
1154            });
1155            if plan.lock().rng.gen_bool(0.3) {
1156                log::info!("{}: detaching completions request", client.username);
1157                cx.update(|cx| completions.detach_and_log_err(cx));
1158            } else {
1159                completions.await?;
1160            }
1161        }
1162        20..=29 => {
1163            let code_actions = project.update(cx, |project, cx| {
1164                log::info!(
1165                    "{}: requesting code actions for buffer {} ({:?})",
1166                    client.username,
1167                    buffer.read(cx).remote_id(),
1168                    buffer.read(cx).file().unwrap().full_path(cx)
1169                );
1170                let range = buffer.read(cx).random_byte_range(0, &mut plan.lock().rng);
1171                project.code_actions(&buffer, range, cx)
1172            });
1173            let code_actions = cx.background().spawn(async move {
1174                code_actions
1175                    .await
1176                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
1177            });
1178            if plan.lock().rng.gen_bool(0.3) {
1179                log::info!("{}: detaching code actions request", client.username);
1180                cx.update(|cx| code_actions.detach_and_log_err(cx));
1181            } else {
1182                code_actions.await?;
1183            }
1184        }
1185        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1186            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1187                log::info!(
1188                    "{}: saving buffer {} ({:?})",
1189                    client.username,
1190                    buffer.remote_id(),
1191                    buffer.file().unwrap().full_path(cx)
1192                );
1193                (buffer.version(), buffer.save(cx))
1194            });
1195            let save = cx.background().spawn(async move {
1196                let (saved_version, _, _) = save
1197                    .await
1198                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1199                assert!(saved_version.observed_all(&requested_version));
1200                Ok::<_, anyhow::Error>(())
1201            });
1202            if plan.lock().rng.gen_bool(0.3) {
1203                log::info!("{}: detaching save request", client.username);
1204                cx.update(|cx| save.detach_and_log_err(cx));
1205            } else {
1206                save.await?;
1207            }
1208        }
1209        40..=44 => {
1210            let prepare_rename = project.update(cx, |project, cx| {
1211                log::info!(
1212                    "{}: preparing rename for buffer {} ({:?})",
1213                    client.username,
1214                    buffer.read(cx).remote_id(),
1215                    buffer.read(cx).file().unwrap().full_path(cx)
1216                );
1217                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1218                project.prepare_rename(buffer, offset, cx)
1219            });
1220            let prepare_rename = cx.background().spawn(async move {
1221                prepare_rename
1222                    .await
1223                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1224            });
1225            if plan.lock().rng.gen_bool(0.3) {
1226                log::info!("{}: detaching prepare rename request", client.username);
1227                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1228            } else {
1229                prepare_rename.await?;
1230            }
1231        }
1232        45..=49 => {
1233            let definitions = project.update(cx, |project, cx| {
1234                log::info!(
1235                    "{}: requesting definitions for buffer {} ({:?})",
1236                    client.username,
1237                    buffer.read(cx).remote_id(),
1238                    buffer.read(cx).file().unwrap().full_path(cx)
1239                );
1240                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1241                project.definition(&buffer, offset, cx)
1242            });
1243            let definitions = cx.background().spawn(async move {
1244                definitions
1245                    .await
1246                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1247            });
1248            if plan.lock().rng.gen_bool(0.3) {
1249                log::info!("{}: detaching definitions request", client.username);
1250                cx.update(|cx| definitions.detach_and_log_err(cx));
1251            } else {
1252                let definitions = definitions.await?;
1253                client
1254                    .buffers_for_project(&project)
1255                    .extend(definitions.into_iter().map(|loc| loc.target.buffer));
1256            }
1257        }
1258        50..=54 => {
1259            let highlights = project.update(cx, |project, cx| {
1260                log::info!(
1261                    "{}: requesting highlights for buffer {} ({:?})",
1262                    client.username,
1263                    buffer.read(cx).remote_id(),
1264                    buffer.read(cx).file().unwrap().full_path(cx)
1265                );
1266                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1267                project.document_highlights(&buffer, offset, cx)
1268            });
1269            let highlights = cx.background().spawn(async move {
1270                highlights
1271                    .await
1272                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1273            });
1274            if plan.lock().rng.gen_bool(0.3) {
1275                log::info!("{}: detaching highlights request", client.username);
1276                cx.update(|cx| highlights.detach_and_log_err(cx));
1277            } else {
1278                highlights.await?;
1279            }
1280        }
1281        55..=59 => {
1282            let search = project.update(cx, |project, cx| {
1283                let query = plan.lock().rng.gen_range('a'..='z');
1284                log::info!("{}: project-wide search {:?}", client.username, query);
1285                project.search(SearchQuery::text(query, false, false), cx)
1286            });
1287            let search = cx.background().spawn(async move {
1288                search
1289                    .await
1290                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1291            });
1292            if plan.lock().rng.gen_bool(0.3) {
1293                log::info!("{}: detaching search request", client.username);
1294                cx.update(|cx| search.detach_and_log_err(cx));
1295            } else {
1296                let search = search.await?;
1297                client
1298                    .buffers_for_project(&project)
1299                    .extend(search.into_keys());
1300            }
1301        }
1302        _ => {
1303            buffer.update(cx, |buffer, cx| {
1304                log::info!(
1305                    "{}: updating buffer {} ({:?})",
1306                    client.username,
1307                    buffer.remote_id(),
1308                    buffer.file().unwrap().full_path(cx)
1309                );
1310                if plan.lock().rng.gen_bool(0.7) {
1311                    buffer.randomly_edit(&mut plan.lock().rng, 5, cx);
1312                } else {
1313                    buffer.randomly_undo_redo(&mut plan.lock().rng, cx);
1314                }
1315            });
1316        }
1317    }
1318
1319    Ok(())
1320}
1321
1322fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1323    client
1324        .local_projects()
1325        .iter()
1326        .chain(client.remote_projects().iter())
1327        .choose(rng)
1328        .cloned()
1329}
1330
1331fn gen_file_name(rng: &mut StdRng) -> String {
1332    let mut name = String::new();
1333    for _ in 0..10 {
1334        let letter = rng.gen_range('a'..='z');
1335        name.push(letter);
1336    }
1337    name
1338}