randomized_integration_tests.rs

   1use crate::{
   2    db::{self, NewUserParams, UserId},
   3    rpc::{CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
   4    tests::{TestClient, TestServer},
   5};
   6use anyhow::{anyhow, Result};
   7use call::ActiveCall;
   8use client::RECEIVE_TIMEOUT;
   9use collections::BTreeMap;
  10use fs::Fs as _;
  11use futures::StreamExt as _;
  12use gpui::{executor::Deterministic, ModelHandle, TestAppContext};
  13use language::{range_to_lsp, FakeLspAdapter, Language, LanguageConfig, PointUtf16};
  14use lsp::FakeLanguageServer;
  15use parking_lot::Mutex;
  16use project::{search::SearchQuery, Project};
  17use rand::prelude::*;
  18use std::{
  19    env,
  20    ops::Range,
  21    path::{Path, PathBuf},
  22    rc::Rc,
  23    sync::Arc,
  24};
  25
  26#[gpui::test(iterations = 100)]
  27async fn test_random_collaboration(
  28    cx: &mut TestAppContext,
  29    deterministic: Arc<Deterministic>,
  30    mut rng: StdRng,
  31) {
  32    deterministic.forbid_parking();
  33
  34    let max_peers = env::var("MAX_PEERS")
  35        .map(|i| i.parse().expect("invalid `MAX_PEERS` variable"))
  36        .unwrap_or(5);
  37
  38    let max_operations = env::var("OPERATIONS")
  39        .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
  40        .unwrap_or(10);
  41
  42    let mut server = TestServer::start(&deterministic).await;
  43    let db = server.app_state.db.clone();
  44
  45    let mut users = Vec::new();
  46    for ix in 0..max_peers {
  47        let username = format!("user-{}", ix + 1);
  48        let user_id = db
  49            .create_user(
  50                &format!("{username}@example.com"),
  51                false,
  52                NewUserParams {
  53                    github_login: username.clone(),
  54                    github_user_id: (ix + 1) as i32,
  55                    invite_count: 0,
  56                },
  57            )
  58            .await
  59            .unwrap()
  60            .user_id;
  61        users.push(UserTestPlan {
  62            user_id,
  63            username,
  64            online: false,
  65            next_root_id: 0,
  66        });
  67    }
  68
  69    for (ix, user_a) in users.iter().enumerate() {
  70        for user_b in &users[ix + 1..] {
  71            server
  72                .app_state
  73                .db
  74                .send_contact_request(user_a.user_id, user_b.user_id)
  75                .await
  76                .unwrap();
  77            server
  78                .app_state
  79                .db
  80                .respond_to_contact_request(user_b.user_id, user_a.user_id, true)
  81                .await
  82                .unwrap();
  83        }
  84    }
  85
  86    let plan = Arc::new(Mutex::new(TestPlan {
  87        users,
  88        allow_server_restarts: rng.gen_bool(0.7),
  89        allow_client_reconnection: rng.gen_bool(0.7),
  90        allow_client_disconnection: rng.gen_bool(0.1),
  91        rng,
  92    }));
  93
  94    let mut clients = Vec::new();
  95    let mut client_tasks = Vec::new();
  96    let mut operation_channels = Vec::new();
  97    let mut next_entity_id = 100000;
  98
  99    let mut i = 0;
 100    while i < max_operations {
 101        let next_operation = plan.lock().next_operation(&clients).await;
 102        match next_operation {
 103            Operation::AddConnection { user_id } => {
 104                let username = {
 105                    let mut plan = plan.lock();
 106                    let mut user = plan.user(user_id);
 107                    user.online = true;
 108                    user.username.clone()
 109                };
 110                log::info!("Adding new connection for {}", username);
 111                next_entity_id += 100000;
 112                let mut client_cx = TestAppContext::new(
 113                    cx.foreground_platform(),
 114                    cx.platform(),
 115                    deterministic.build_foreground(next_entity_id),
 116                    deterministic.build_background(),
 117                    cx.font_cache(),
 118                    cx.leak_detector(),
 119                    next_entity_id,
 120                    cx.function_name.clone(),
 121                );
 122
 123                let (operation_tx, operation_rx) = futures::channel::mpsc::unbounded();
 124                let client = Rc::new(server.create_client(&mut client_cx, &username).await);
 125                operation_channels.push(operation_tx);
 126                clients.push((client.clone(), client_cx.clone()));
 127                client_tasks.push(client_cx.foreground().spawn(simulate_client(
 128                    client,
 129                    operation_rx,
 130                    plan.clone(),
 131                    client_cx,
 132                )));
 133
 134                log::info!("Added connection for {}", username);
 135                i += 1;
 136            }
 137
 138            Operation::RemoveConnection { user_id } => {
 139                log::info!("Simulating full disconnection of user {}", user_id);
 140                let client_ix = clients
 141                    .iter()
 142                    .position(|(client, cx)| client.current_user_id(cx) == user_id)
 143                    .unwrap();
 144                let user_connection_ids = server
 145                    .connection_pool
 146                    .lock()
 147                    .user_connection_ids(user_id)
 148                    .collect::<Vec<_>>();
 149                assert_eq!(user_connection_ids.len(), 1);
 150                let removed_peer_id = user_connection_ids[0].into();
 151                let (client, mut client_cx) = clients.remove(client_ix);
 152                let client_task = client_tasks.remove(client_ix);
 153                operation_channels.remove(client_ix);
 154                server.forbid_connections();
 155                server.disconnect_client(removed_peer_id);
 156                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 157                deterministic.start_waiting();
 158                log::info!("Waiting for user {} to exit...", user_id);
 159                client_task.await;
 160                deterministic.finish_waiting();
 161                server.allow_connections();
 162
 163                for project in client.remote_projects().iter() {
 164                    project.read_with(&client_cx, |project, _| {
 165                        assert!(
 166                            project.is_read_only(),
 167                            "project {:?} should be read only",
 168                            project.remote_id()
 169                        )
 170                    });
 171                }
 172
 173                for (client, cx) in &clients {
 174                    let contacts = server
 175                        .app_state
 176                        .db
 177                        .get_contacts(client.current_user_id(cx))
 178                        .await
 179                        .unwrap();
 180                    let pool = server.connection_pool.lock();
 181                    for contact in contacts {
 182                        if let db::Contact::Accepted { user_id: id, .. } = contact {
 183                            if pool.is_user_online(id) {
 184                                assert_ne!(
 185                                    id, user_id,
 186                                    "removed client is still a contact of another peer"
 187                                );
 188                            }
 189                        }
 190                    }
 191                }
 192
 193                log::info!("{} removed", client.username);
 194                plan.lock().user(user_id).online = false;
 195                client_cx.update(|cx| {
 196                    cx.clear_globals();
 197                    drop(client);
 198                });
 199                i += 1;
 200            }
 201
 202            Operation::BounceConnection { user_id } => {
 203                log::info!("Simulating temporary disconnection of user {}", user_id);
 204                let user_connection_ids = server
 205                    .connection_pool
 206                    .lock()
 207                    .user_connection_ids(user_id)
 208                    .collect::<Vec<_>>();
 209                assert_eq!(user_connection_ids.len(), 1);
 210                let peer_id = user_connection_ids[0].into();
 211                server.disconnect_client(peer_id);
 212                deterministic.advance_clock(RECEIVE_TIMEOUT + RECONNECT_TIMEOUT);
 213                i += 1;
 214            }
 215
 216            Operation::RestartServer => {
 217                log::info!("Simulating server restart");
 218                server.reset().await;
 219                deterministic.advance_clock(RECEIVE_TIMEOUT);
 220                server.start().await.unwrap();
 221                deterministic.advance_clock(CLEANUP_TIMEOUT);
 222                let environment = &server.app_state.config.zed_environment;
 223                let stale_room_ids = server
 224                    .app_state
 225                    .db
 226                    .stale_room_ids(environment, server.id())
 227                    .await
 228                    .unwrap();
 229                assert_eq!(stale_room_ids, vec![]);
 230                i += 1;
 231            }
 232
 233            Operation::RunUntilParked => {
 234                deterministic.run_until_parked();
 235            }
 236
 237            Operation::MutateClients(user_ids) => {
 238                for user_id in user_ids {
 239                    let client_ix = clients
 240                        .iter()
 241                        .position(|(client, cx)| client.current_user_id(cx) == user_id)
 242                        .unwrap();
 243                    operation_channels[client_ix].unbounded_send(()).unwrap();
 244                    i += 1;
 245                }
 246            }
 247        }
 248    }
 249
 250    drop(operation_channels);
 251    deterministic.start_waiting();
 252    futures::future::join_all(client_tasks).await;
 253    deterministic.finish_waiting();
 254    deterministic.run_until_parked();
 255
 256    for (client, client_cx) in &clients {
 257        for guest_project in client.remote_projects().iter() {
 258            guest_project.read_with(client_cx, |guest_project, cx| {
 259                let host_project = clients.iter().find_map(|(client, cx)| {
 260                    let project = client
 261                        .local_projects()
 262                        .iter()
 263                        .find(|host_project| {
 264                            host_project.read_with(cx, |host_project, _| {
 265                                host_project.remote_id() == guest_project.remote_id()
 266                            })
 267                        })?
 268                        .clone();
 269                    Some((project, cx))
 270                });
 271
 272                if !guest_project.is_read_only() {
 273                    if let Some((host_project, host_cx)) = host_project {
 274                        let host_worktree_snapshots =
 275                            host_project.read_with(host_cx, |host_project, cx| {
 276                                host_project
 277                                    .worktrees(cx)
 278                                    .map(|worktree| {
 279                                        let worktree = worktree.read(cx);
 280                                        (worktree.id(), worktree.snapshot())
 281                                    })
 282                                    .collect::<BTreeMap<_, _>>()
 283                            });
 284                        let guest_worktree_snapshots = guest_project
 285                            .worktrees(cx)
 286                            .map(|worktree| {
 287                                let worktree = worktree.read(cx);
 288                                (worktree.id(), worktree.snapshot())
 289                            })
 290                            .collect::<BTreeMap<_, _>>();
 291
 292                        assert_eq!(
 293                            guest_worktree_snapshots.keys().collect::<Vec<_>>(),
 294                            host_worktree_snapshots.keys().collect::<Vec<_>>(),
 295                            "{} has different worktrees than the host",
 296                            client.username
 297                        );
 298
 299                        for (id, host_snapshot) in &host_worktree_snapshots {
 300                            let guest_snapshot = &guest_worktree_snapshots[id];
 301                            assert_eq!(
 302                                guest_snapshot.root_name(),
 303                                host_snapshot.root_name(),
 304                                "{} has different root name than the host for worktree {}",
 305                                client.username,
 306                                id
 307                            );
 308                            assert_eq!(
 309                                guest_snapshot.abs_path(),
 310                                host_snapshot.abs_path(),
 311                                "{} has different abs path than the host for worktree {}",
 312                                client.username,
 313                                id
 314                            );
 315                            assert_eq!(
 316                                guest_snapshot.entries(false).collect::<Vec<_>>(),
 317                                host_snapshot.entries(false).collect::<Vec<_>>(),
 318                                "{} has different snapshot than the host for worktree {} ({:?}) and project {:?}",
 319                                client.username,
 320                                id,
 321                                host_snapshot.abs_path(),
 322                                host_project.read_with(host_cx, |project, _| project.remote_id())
 323                            );
 324                            assert_eq!(guest_snapshot.scan_id(), host_snapshot.scan_id());
 325                        }
 326                    }
 327                }
 328
 329                guest_project.check_invariants(cx);
 330            });
 331        }
 332
 333        let buffers = client.buffers().clone();
 334        for (guest_project, guest_buffers) in &buffers {
 335            let project_id = if guest_project.read_with(client_cx, |project, _| {
 336                project.is_local() || project.is_read_only()
 337            }) {
 338                continue;
 339            } else {
 340                guest_project
 341                    .read_with(client_cx, |project, _| project.remote_id())
 342                    .unwrap()
 343            };
 344            let guest_user_id = client.user_id().unwrap();
 345
 346            let host_project = clients.iter().find_map(|(client, cx)| {
 347                let project = client
 348                    .local_projects()
 349                    .iter()
 350                    .find(|host_project| {
 351                        host_project.read_with(cx, |host_project, _| {
 352                            host_project.remote_id() == Some(project_id)
 353                        })
 354                    })?
 355                    .clone();
 356                Some((client.user_id().unwrap(), project, cx))
 357            });
 358
 359            let (host_user_id, host_project, host_cx) =
 360                if let Some((host_user_id, host_project, host_cx)) = host_project {
 361                    (host_user_id, host_project, host_cx)
 362                } else {
 363                    continue;
 364                };
 365
 366            for guest_buffer in guest_buffers {
 367                let buffer_id = guest_buffer.read_with(client_cx, |buffer, _| buffer.remote_id());
 368                let host_buffer = host_project.read_with(host_cx, |project, cx| {
 369                    project.buffer_for_id(buffer_id, cx).unwrap_or_else(|| {
 370                        panic!(
 371                            "host does not have buffer for guest:{}, peer:{:?}, id:{}",
 372                            client.username,
 373                            client.peer_id(),
 374                            buffer_id
 375                        )
 376                    })
 377                });
 378                let path = host_buffer
 379                    .read_with(host_cx, |buffer, cx| buffer.file().unwrap().full_path(cx));
 380
 381                assert_eq!(
 382                    guest_buffer.read_with(client_cx, |buffer, _| buffer.deferred_ops_len()),
 383                    0,
 384                    "{}, buffer {}, path {:?} has deferred operations",
 385                    client.username,
 386                    buffer_id,
 387                    path,
 388                );
 389                assert_eq!(
 390                    guest_buffer.read_with(client_cx, |buffer, _| buffer.text()),
 391                    host_buffer.read_with(host_cx, |buffer, _| buffer.text()),
 392                    "{}, buffer {}, path {:?}, differs from the host's buffer",
 393                    client.username,
 394                    buffer_id,
 395                    path
 396                );
 397
 398                let host_file = host_buffer.read_with(host_cx, |b, _| b.file().cloned());
 399                let guest_file = guest_buffer.read_with(client_cx, |b, _| b.file().cloned());
 400                match (host_file, guest_file) {
 401                    (Some(host_file), Some(guest_file)) => {
 402                        assert_eq!(guest_file.path(), host_file.path());
 403                        assert_eq!(guest_file.is_deleted(), host_file.is_deleted());
 404                        assert_eq!(
 405                            guest_file.mtime(),
 406                            host_file.mtime(),
 407                            "guest {} mtime does not match host {} for path {:?} in project {}",
 408                            guest_user_id,
 409                            host_user_id,
 410                            guest_file.path(),
 411                            project_id,
 412                        );
 413                    }
 414                    (None, None) => {}
 415                    (None, _) => panic!("host's file is None, guest's isn't "),
 416                    (_, None) => panic!("guest's file is None, hosts's isn't "),
 417                }
 418            }
 419        }
 420    }
 421
 422    for (client, mut cx) in clients {
 423        cx.update(|cx| {
 424            cx.clear_globals();
 425            drop(client);
 426        });
 427    }
 428}
 429
 430struct TestPlan {
 431    rng: StdRng,
 432    users: Vec<UserTestPlan>,
 433    allow_server_restarts: bool,
 434    allow_client_reconnection: bool,
 435    allow_client_disconnection: bool,
 436}
 437
 438struct UserTestPlan {
 439    user_id: UserId,
 440    username: String,
 441    next_root_id: usize,
 442    online: bool,
 443}
 444
 445#[derive(Debug)]
 446enum Operation {
 447    AddConnection { user_id: UserId },
 448    RemoveConnection { user_id: UserId },
 449    BounceConnection { user_id: UserId },
 450    RestartServer,
 451    RunUntilParked,
 452    MutateClients(Vec<UserId>),
 453}
 454
 455#[derive(Debug)]
 456enum ClientOperation {
 457    AcceptIncomingCall,
 458    RejectIncomingCall,
 459    LeaveCall,
 460    InviteContactToCall {
 461        user_id: UserId,
 462    },
 463    OpenLocalProject {
 464        first_root_name: String,
 465    },
 466    OpenRemoteProject {
 467        host_id: UserId,
 468        first_root_name: String,
 469    },
 470    AddWorktreeToProject {
 471        project_root_name: String,
 472        new_root_path: PathBuf,
 473    },
 474    CloseRemoteProject {
 475        project_root_name: String,
 476    },
 477    OpenBuffer {
 478        project_root_name: String,
 479        full_path: PathBuf,
 480    },
 481    EditBuffer {
 482        project_root_name: String,
 483        full_path: PathBuf,
 484        edits: Vec<(Range<usize>, Arc<str>)>,
 485    },
 486    Other,
 487}
 488
 489impl TestPlan {
 490    async fn next_operation(&mut self, clients: &[(Rc<TestClient>, TestAppContext)]) -> Operation {
 491        let operation = loop {
 492            break match self.rng.gen_range(0..100) {
 493                0..=19 if clients.len() < self.users.len() => {
 494                    let user = self
 495                        .users
 496                        .iter()
 497                        .filter(|u| !u.online)
 498                        .choose(&mut self.rng)
 499                        .unwrap();
 500                    Operation::AddConnection {
 501                        user_id: user.user_id,
 502                    }
 503                }
 504                20..=24 if clients.len() > 1 && self.allow_client_disconnection => {
 505                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
 506                    let user_id = client.current_user_id(cx);
 507                    Operation::RemoveConnection { user_id }
 508                }
 509                25..=29 if clients.len() > 1 && self.allow_client_reconnection => {
 510                    let (client, cx) = &clients[self.rng.gen_range(0..clients.len())];
 511                    let user_id = client.current_user_id(cx);
 512                    Operation::BounceConnection { user_id }
 513                }
 514                30..=34 if self.allow_server_restarts && clients.len() > 1 => {
 515                    Operation::RestartServer
 516                }
 517                35..=39 => Operation::RunUntilParked,
 518                _ if !clients.is_empty() => {
 519                    let user_ids = (0..self.rng.gen_range(0..10))
 520                        .map(|_| {
 521                            let ix = self.rng.gen_range(0..clients.len());
 522                            let (client, cx) = &clients[ix];
 523                            client.current_user_id(cx)
 524                        })
 525                        .collect();
 526                    Operation::MutateClients(user_ids)
 527                }
 528                _ => continue,
 529            };
 530        };
 531        operation
 532    }
 533
 534    async fn next_client_operation(
 535        &mut self,
 536        client: &TestClient,
 537        cx: &TestAppContext,
 538    ) -> ClientOperation {
 539        let user_id = client.current_user_id(cx);
 540        let call = cx.read(ActiveCall::global);
 541        let operation = loop {
 542            match self.rng.gen_range(0..100) {
 543                // Mutate the call
 544                0..=19 => match self.rng.gen_range(0..100_u32) {
 545                    // Respond to an incoming call
 546                    0..=39 => {
 547                        if call.read_with(cx, |call, _| call.incoming().borrow().is_some()) {
 548                            break if self.rng.gen_bool(0.7) {
 549                                ClientOperation::AcceptIncomingCall
 550                            } else {
 551                                ClientOperation::RejectIncomingCall
 552                            };
 553                        }
 554                    }
 555
 556                    // Invite a contact to the current call
 557                    30..=89 => {
 558                        let available_contacts =
 559                            client.user_store.read_with(cx, |user_store, _| {
 560                                user_store
 561                                    .contacts()
 562                                    .iter()
 563                                    .filter(|contact| contact.online && !contact.busy)
 564                                    .cloned()
 565                                    .collect::<Vec<_>>()
 566                            });
 567                        if !available_contacts.is_empty() {
 568                            let contact = available_contacts.choose(&mut self.rng).unwrap();
 569                            break ClientOperation::InviteContactToCall {
 570                                user_id: UserId(contact.user.id as i32),
 571                            };
 572                        }
 573                    }
 574
 575                    // Leave the current call
 576                    90.. => {
 577                        if self.allow_client_disconnection
 578                            && call.read_with(cx, |call, _| call.room().is_some())
 579                        {
 580                            break ClientOperation::LeaveCall;
 581                        }
 582                    }
 583                },
 584
 585                // Mutate projects
 586                20..=39 => match self.rng.gen_range(0..100_u32) {
 587                    // Open a remote project
 588                    0..=30 => {
 589                        if let Some(room) = call.read_with(cx, |call, _| call.room().cloned()) {
 590                            let remote_projects = room.read_with(cx, |room, _| {
 591                                room.remote_participants()
 592                                    .values()
 593                                    .flat_map(|participant| {
 594                                        participant.projects.iter().map(|project| {
 595                                            (
 596                                                UserId::from_proto(participant.user.id),
 597                                                project.worktree_root_names[0].clone(),
 598                                            )
 599                                        })
 600                                    })
 601                                    .collect::<Vec<_>>()
 602                            });
 603                            if !remote_projects.is_empty() {
 604                                let (host_id, first_root_name) =
 605                                    remote_projects.choose(&mut self.rng).unwrap().clone();
 606                                break ClientOperation::OpenRemoteProject {
 607                                    host_id,
 608                                    first_root_name,
 609                                };
 610                            }
 611                        }
 612                    }
 613
 614                    // Close a remote project
 615                    31..=40 => {
 616                        if !client.remote_projects().is_empty() {
 617                            let project = client
 618                                .remote_projects()
 619                                .choose(&mut self.rng)
 620                                .unwrap()
 621                                .clone();
 622                            let first_root_name = root_name_for_project(&project, cx);
 623                            break ClientOperation::CloseRemoteProject {
 624                                project_root_name: first_root_name,
 625                            };
 626                        }
 627                    }
 628
 629                    // Open a local project
 630                    41..=60 => {
 631                        let first_root_name = self.next_root_dir_name(user_id);
 632                        break ClientOperation::OpenLocalProject { first_root_name };
 633                    }
 634
 635                    // Add a worktree to a local project
 636                    61.. => {
 637                        if !client.local_projects().is_empty() {
 638                            let project = client
 639                                .local_projects()
 640                                .choose(&mut self.rng)
 641                                .unwrap()
 642                                .clone();
 643                            let project_root_name = root_name_for_project(&project, cx);
 644
 645                            let mut paths = client.fs.paths().await;
 646                            paths.remove(0);
 647                            let new_root_path = if paths.is_empty() || self.rng.gen() {
 648                                Path::new("/").join(&self.next_root_dir_name(user_id))
 649                            } else {
 650                                paths.choose(&mut self.rng).unwrap().clone()
 651                            };
 652
 653                            break ClientOperation::AddWorktreeToProject {
 654                                project_root_name,
 655                                new_root_path,
 656                            };
 657                        }
 658                    }
 659                },
 660
 661                // Mutate buffers
 662                40..=79 => {
 663                    let Some(project) = choose_random_project(client, &mut self.rng) else { continue };
 664                    let project_root_name = root_name_for_project(&project, cx);
 665
 666                    match self.rng.gen_range(0..100_u32) {
 667                        // Manipulate an existing buffer
 668                        0..=80 => {
 669                            let Some(buffer) = client
 670                                .buffers_for_project(&project)
 671                                .iter()
 672                                .choose(&mut self.rng)
 673                                .cloned() else { continue };
 674
 675                            match self.rng.gen_range(0..100_u32) {
 676                                0..=9 => {
 677                                    let (full_path, edits) = buffer.read_with(cx, |buffer, cx| {
 678                                        (
 679                                            buffer.file().unwrap().full_path(cx),
 680                                            buffer.get_random_edits(&mut self.rng, 3),
 681                                        )
 682                                    });
 683                                    break ClientOperation::EditBuffer {
 684                                        project_root_name,
 685                                        full_path,
 686                                        edits,
 687                                    };
 688                                }
 689                                _ => {}
 690                            }
 691                        }
 692
 693                        // Open a buffer
 694                        81.. => {
 695                            let worktree = project.read_with(cx, |project, cx| {
 696                                project
 697                                    .worktrees(cx)
 698                                    .filter(|worktree| {
 699                                        let worktree = worktree.read(cx);
 700                                        worktree.is_visible()
 701                                            && worktree.entries(false).any(|e| e.is_file())
 702                                    })
 703                                    .choose(&mut self.rng)
 704                            });
 705                            let Some(worktree) = worktree else { continue };
 706                            let full_path = worktree.read_with(cx, |worktree, _| {
 707                                let entry = worktree
 708                                    .entries(false)
 709                                    .filter(|e| e.is_file())
 710                                    .choose(&mut self.rng)
 711                                    .unwrap();
 712                                if entry.path.as_ref() == Path::new("") {
 713                                    Path::new(worktree.root_name()).into()
 714                                } else {
 715                                    Path::new(worktree.root_name()).join(&entry.path)
 716                                }
 717                            });
 718                            break ClientOperation::OpenBuffer {
 719                                project_root_name,
 720                                full_path,
 721                            };
 722                        }
 723                    }
 724                }
 725
 726                _ => break ClientOperation::Other,
 727            }
 728        };
 729        operation
 730    }
 731
 732    fn next_root_dir_name(&mut self, user_id: UserId) -> String {
 733        let user_ix = self
 734            .users
 735            .iter()
 736            .position(|user| user.user_id == user_id)
 737            .unwrap();
 738        let root_id = util::post_inc(&mut self.users[user_ix].next_root_id);
 739        format!("dir-{user_id}-{root_id}")
 740    }
 741
 742    fn user(&mut self, user_id: UserId) -> &mut UserTestPlan {
 743        let ix = self
 744            .users
 745            .iter()
 746            .position(|user| user.user_id == user_id)
 747            .unwrap();
 748        &mut self.users[ix]
 749    }
 750}
 751
 752async fn simulate_client(
 753    client: Rc<TestClient>,
 754    mut operation_rx: futures::channel::mpsc::UnboundedReceiver<()>,
 755    plan: Arc<Mutex<TestPlan>>,
 756    mut cx: TestAppContext,
 757) {
 758    // Setup language server
 759    let mut language = Language::new(
 760        LanguageConfig {
 761            name: "Rust".into(),
 762            path_suffixes: vec!["rs".to_string()],
 763            ..Default::default()
 764        },
 765        None,
 766    );
 767    let _fake_language_servers = language
 768        .set_fake_lsp_adapter(Arc::new(FakeLspAdapter {
 769            name: "the-fake-language-server",
 770            capabilities: lsp::LanguageServer::full_capabilities(),
 771            initializer: Some(Box::new({
 772                let plan = plan.clone();
 773                let fs = client.fs.clone();
 774                move |fake_server: &mut FakeLanguageServer| {
 775                    fake_server.handle_request::<lsp::request::Completion, _, _>(
 776                        |_, _| async move {
 777                            Ok(Some(lsp::CompletionResponse::Array(vec![
 778                                lsp::CompletionItem {
 779                                    text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
 780                                        range: lsp::Range::new(
 781                                            lsp::Position::new(0, 0),
 782                                            lsp::Position::new(0, 0),
 783                                        ),
 784                                        new_text: "the-new-text".to_string(),
 785                                    })),
 786                                    ..Default::default()
 787                                },
 788                            ])))
 789                        },
 790                    );
 791
 792                    fake_server.handle_request::<lsp::request::CodeActionRequest, _, _>(
 793                        |_, _| async move {
 794                            Ok(Some(vec![lsp::CodeActionOrCommand::CodeAction(
 795                                lsp::CodeAction {
 796                                    title: "the-code-action".to_string(),
 797                                    ..Default::default()
 798                                },
 799                            )]))
 800                        },
 801                    );
 802
 803                    fake_server.handle_request::<lsp::request::PrepareRenameRequest, _, _>(
 804                        |params, _| async move {
 805                            Ok(Some(lsp::PrepareRenameResponse::Range(lsp::Range::new(
 806                                params.position,
 807                                params.position,
 808                            ))))
 809                        },
 810                    );
 811
 812                    fake_server.handle_request::<lsp::request::GotoDefinition, _, _>({
 813                        let fs = fs.clone();
 814                        let plan = plan.clone();
 815                        move |_, _| {
 816                            let fs = fs.clone();
 817                            let plan = plan.clone();
 818                            async move {
 819                                let files = fs.files().await;
 820                                let mut plan = plan.lock();
 821                                let count = plan.rng.gen_range::<usize, _>(1..3);
 822                                let files = (0..count)
 823                                    .map(|_| files.choose(&mut plan.rng).unwrap())
 824                                    .collect::<Vec<_>>();
 825                                log::info!("LSP: Returning definitions in files {:?}", &files);
 826                                Ok(Some(lsp::GotoDefinitionResponse::Array(
 827                                    files
 828                                        .into_iter()
 829                                        .map(|file| lsp::Location {
 830                                            uri: lsp::Url::from_file_path(file).unwrap(),
 831                                            range: Default::default(),
 832                                        })
 833                                        .collect(),
 834                                )))
 835                            }
 836                        }
 837                    });
 838
 839                    fake_server.handle_request::<lsp::request::DocumentHighlightRequest, _, _>({
 840                        let plan = plan.clone();
 841                        move |_, _| {
 842                            let mut highlights = Vec::new();
 843                            let highlight_count = plan.lock().rng.gen_range(1..=5);
 844                            for _ in 0..highlight_count {
 845                                let start_row = plan.lock().rng.gen_range(0..100);
 846                                let start_column = plan.lock().rng.gen_range(0..100);
 847                                let start = PointUtf16::new(start_row, start_column);
 848                                let end_row = plan.lock().rng.gen_range(0..100);
 849                                let end_column = plan.lock().rng.gen_range(0..100);
 850                                let end = PointUtf16::new(end_row, end_column);
 851                                let range = if start > end { end..start } else { start..end };
 852                                highlights.push(lsp::DocumentHighlight {
 853                                    range: range_to_lsp(range.clone()),
 854                                    kind: Some(lsp::DocumentHighlightKind::READ),
 855                                });
 856                            }
 857                            highlights.sort_unstable_by_key(|highlight| {
 858                                (highlight.range.start, highlight.range.end)
 859                            });
 860                            async move { Ok(Some(highlights)) }
 861                        }
 862                    });
 863                }
 864            })),
 865            ..Default::default()
 866        }))
 867        .await;
 868    client.language_registry.add(Arc::new(language));
 869
 870    while operation_rx.next().await.is_some() {
 871        let operation = plan.lock().next_client_operation(&client, &cx).await;
 872        if let Err(error) = apply_client_operation(&client, plan.clone(), operation, &mut cx).await
 873        {
 874            log::error!("{} error: {:?}", client.username, error);
 875        }
 876
 877        cx.background().simulate_random_delay().await;
 878    }
 879    log::info!("{}: done", client.username);
 880}
 881
 882async fn apply_client_operation(
 883    client: &TestClient,
 884    plan: Arc<Mutex<TestPlan>>,
 885    operation: ClientOperation,
 886    cx: &mut TestAppContext,
 887) -> Result<()> {
 888    match operation {
 889        ClientOperation::AcceptIncomingCall => {
 890            log::info!("{}: accepting incoming call", client.username);
 891            let active_call = cx.read(ActiveCall::global);
 892            active_call
 893                .update(cx, |call, cx| call.accept_incoming(cx))
 894                .await?;
 895        }
 896
 897        ClientOperation::RejectIncomingCall => {
 898            log::info!("{}: declining incoming call", client.username);
 899            let active_call = cx.read(ActiveCall::global);
 900            active_call.update(cx, |call, _| call.decline_incoming())?;
 901        }
 902
 903        ClientOperation::LeaveCall => {
 904            log::info!("{}: hanging up", client.username);
 905            let active_call = cx.read(ActiveCall::global);
 906            active_call.update(cx, |call, cx| call.hang_up(cx))?;
 907        }
 908
 909        ClientOperation::InviteContactToCall { user_id } => {
 910            log::info!("{}: inviting {}", client.username, user_id,);
 911            let active_call = cx.read(ActiveCall::global);
 912            active_call
 913                .update(cx, |call, cx| call.invite(user_id.to_proto(), None, cx))
 914                .await?;
 915        }
 916
 917        ClientOperation::OpenLocalProject { first_root_name } => {
 918            log::info!(
 919                "{}: opening local project at {:?}",
 920                client.username,
 921                first_root_name
 922            );
 923            let root_path = Path::new("/").join(&first_root_name);
 924            client.fs.create_dir(&root_path).await.unwrap();
 925            client
 926                .fs
 927                .create_file(&root_path.join("main.rs"), Default::default())
 928                .await
 929                .unwrap();
 930            let project = client.build_local_project(root_path, cx).await.0;
 931
 932            let active_call = cx.read(ActiveCall::global);
 933            if active_call.read_with(cx, |call, _| call.room().is_some())
 934                && project.read_with(cx, |project, _| project.is_local() && !project.is_shared())
 935            {
 936                match active_call
 937                    .update(cx, |call, cx| call.share_project(project.clone(), cx))
 938                    .await
 939                {
 940                    Ok(project_id) => {
 941                        log::info!(
 942                            "{}: shared project {} with id {}",
 943                            client.username,
 944                            first_root_name,
 945                            project_id
 946                        );
 947                    }
 948                    Err(error) => {
 949                        log::error!(
 950                            "{}: error sharing project {}: {:?}",
 951                            client.username,
 952                            first_root_name,
 953                            error
 954                        );
 955                    }
 956                }
 957            }
 958
 959            client.local_projects_mut().push(project.clone());
 960        }
 961
 962        ClientOperation::AddWorktreeToProject {
 963            project_root_name,
 964            new_root_path,
 965        } => {
 966            log::info!(
 967                "{}: finding/creating local worktree at {:?} to project with root path {}",
 968                client.username,
 969                new_root_path,
 970                project_root_name
 971            );
 972            let project = project_for_root_name(client, &project_root_name, cx)
 973                .expect("invalid project in test operation");
 974            if !client.fs.paths().await.contains(&new_root_path) {
 975                client.fs.create_dir(&new_root_path).await.unwrap();
 976            }
 977            project
 978                .update(cx, |project, cx| {
 979                    project.find_or_create_local_worktree(&new_root_path, true, cx)
 980                })
 981                .await
 982                .unwrap();
 983        }
 984
 985        ClientOperation::CloseRemoteProject { project_root_name } => {
 986            log::info!(
 987                "{}: dropping project with root path {}",
 988                client.username,
 989                project_root_name,
 990            );
 991            let ix = project_ix_for_root_name(&*client.remote_projects(), &project_root_name, cx)
 992                .expect("invalid project in test operation");
 993            client.remote_projects_mut().remove(ix);
 994        }
 995
 996        ClientOperation::OpenRemoteProject {
 997            host_id,
 998            first_root_name,
 999        } => {
1000            log::info!(
1001                "{}: joining remote project of user {}, root name {}",
1002                client.username,
1003                host_id,
1004                first_root_name,
1005            );
1006            let active_call = cx.read(ActiveCall::global);
1007            let project_id = active_call
1008                .read_with(cx, |call, cx| {
1009                    let room = call.room().cloned()?;
1010                    let participant = room
1011                        .read(cx)
1012                        .remote_participants()
1013                        .get(&host_id.to_proto())?;
1014                    let project = participant
1015                        .projects
1016                        .iter()
1017                        .find(|project| project.worktree_root_names[0] == first_root_name)?;
1018                    Some(project.id)
1019                })
1020                .expect("invalid project in test operation");
1021            let project = client.build_remote_project(project_id, cx).await;
1022            client.remote_projects_mut().push(project);
1023        }
1024
1025        ClientOperation::OpenBuffer {
1026            project_root_name,
1027            full_path,
1028        } => {
1029            log::info!(
1030                "{}: opening path {:?} in project {}",
1031                client.username,
1032                full_path,
1033                project_root_name,
1034            );
1035            let project = project_for_root_name(client, &project_root_name, cx)
1036                .expect("invalid project in test operation");
1037            let mut components = full_path.components();
1038            let root_name = components.next().unwrap().as_os_str().to_str().unwrap();
1039            let path = components.as_path();
1040            let worktree_id = project
1041                .read_with(cx, |project, cx| {
1042                    project.worktrees(cx).find_map(|worktree| {
1043                        let worktree = worktree.read(cx);
1044                        if worktree.root_name() == root_name {
1045                            Some(worktree.id())
1046                        } else {
1047                            None
1048                        }
1049                    })
1050                })
1051                .expect("invalid buffer path in test operation");
1052            let buffer = project
1053                .update(cx, |project, cx| {
1054                    project.open_buffer((worktree_id, &path), cx)
1055                })
1056                .await?;
1057            client.buffers_for_project(&project).insert(buffer);
1058        }
1059
1060        ClientOperation::EditBuffer {
1061            project_root_name,
1062            full_path,
1063            edits,
1064        } => {
1065            log::info!(
1066                "{}: editing buffer {:?} in project {} with {:?}",
1067                client.username,
1068                full_path,
1069                project_root_name,
1070                edits
1071            );
1072            let project = project_for_root_name(client, &project_root_name, cx)
1073                .expect("invalid project in test operation");
1074            let buffer = client
1075                .buffers_for_project(&project)
1076                .iter()
1077                .find(|buffer| {
1078                    buffer.read_with(cx, |buffer, cx| {
1079                        buffer.file().unwrap().full_path(cx) == full_path
1080                    })
1081                })
1082                .cloned()
1083                .expect("invalid buffer path in test operation");
1084            buffer.update(cx, |buffer, cx| {
1085                buffer.edit(edits, None, cx);
1086            });
1087        }
1088
1089        _ => {
1090            let choice = plan.lock().rng.gen_range(0..100);
1091            match choice {
1092                50..=59
1093                    if !client.local_projects().is_empty()
1094                        || !client.remote_projects().is_empty() =>
1095                {
1096                    randomly_mutate_worktrees(client, &plan, cx).await?;
1097                }
1098                60..=84
1099                    if !client.local_projects().is_empty()
1100                        || !client.remote_projects().is_empty() =>
1101                {
1102                    randomly_query_and_mutate_buffers(client, &plan, cx).await?;
1103                }
1104                _ => randomly_mutate_fs(client, &plan).await,
1105            }
1106        }
1107    }
1108    Ok(())
1109}
1110
1111fn project_for_root_name(
1112    client: &TestClient,
1113    root_name: &str,
1114    cx: &TestAppContext,
1115) -> Option<ModelHandle<Project>> {
1116    if let Some(ix) = project_ix_for_root_name(&*client.local_projects(), root_name, cx) {
1117        return Some(client.local_projects()[ix].clone());
1118    }
1119    if let Some(ix) = project_ix_for_root_name(&*client.remote_projects(), root_name, cx) {
1120        return Some(client.remote_projects()[ix].clone());
1121    }
1122    None
1123}
1124
1125fn project_ix_for_root_name(
1126    projects: &[ModelHandle<Project>],
1127    root_name: &str,
1128    cx: &TestAppContext,
1129) -> Option<usize> {
1130    projects.iter().position(|project| {
1131        project.read_with(cx, |project, cx| {
1132            let worktree = project.visible_worktrees(cx).next().unwrap();
1133            worktree.read(cx).root_name() == root_name
1134        })
1135    })
1136}
1137
1138fn root_name_for_project(project: &ModelHandle<Project>, cx: &TestAppContext) -> String {
1139    project.read_with(cx, |project, cx| {
1140        project
1141            .visible_worktrees(cx)
1142            .next()
1143            .unwrap()
1144            .read(cx)
1145            .root_name()
1146            .to_string()
1147    })
1148}
1149
1150async fn randomly_mutate_fs(client: &TestClient, plan: &Arc<Mutex<TestPlan>>) {
1151    let is_dir = plan.lock().rng.gen::<bool>();
1152    let mut new_path = client
1153        .fs
1154        .directories()
1155        .await
1156        .choose(&mut plan.lock().rng)
1157        .unwrap()
1158        .clone();
1159    new_path.push(gen_file_name(&mut plan.lock().rng));
1160    if is_dir {
1161        log::info!("{}: creating local dir at {:?}", client.username, new_path);
1162        client.fs.create_dir(&new_path).await.unwrap();
1163    } else {
1164        new_path.set_extension("rs");
1165        log::info!("{}: creating local file at {:?}", client.username, new_path);
1166        client
1167            .fs
1168            .create_file(&new_path, Default::default())
1169            .await
1170            .unwrap();
1171    }
1172}
1173
1174async fn randomly_mutate_worktrees(
1175    client: &TestClient,
1176    plan: &Arc<Mutex<TestPlan>>,
1177    cx: &mut TestAppContext,
1178) -> Result<()> {
1179    let project = choose_random_project(client, &mut plan.lock().rng).unwrap();
1180    let Some(worktree) = project.read_with(cx, |project, cx| {
1181        project
1182            .worktrees(cx)
1183            .filter(|worktree| {
1184                let worktree = worktree.read(cx);
1185                worktree.is_visible()
1186                    && worktree.entries(false).any(|e| e.is_file())
1187                    && worktree.root_entry().map_or(false, |e| e.is_dir())
1188            })
1189            .choose(&mut plan.lock().rng)
1190    }) else {
1191        return Ok(())
1192    };
1193
1194    let (worktree_id, worktree_root_name) = worktree.read_with(cx, |worktree, _| {
1195        (worktree.id(), worktree.root_name().to_string())
1196    });
1197
1198    let is_dir = plan.lock().rng.gen::<bool>();
1199    let mut new_path = PathBuf::new();
1200    new_path.push(gen_file_name(&mut plan.lock().rng));
1201    if !is_dir {
1202        new_path.set_extension("rs");
1203    }
1204    log::info!(
1205        "{}: creating {:?} in worktree {} ({})",
1206        client.username,
1207        new_path,
1208        worktree_id,
1209        worktree_root_name,
1210    );
1211    project
1212        .update(cx, |project, cx| {
1213            project.create_entry((worktree_id, new_path), is_dir, cx)
1214        })
1215        .unwrap()
1216        .await?;
1217    Ok(())
1218}
1219
1220async fn randomly_query_and_mutate_buffers(
1221    client: &TestClient,
1222    plan: &Arc<Mutex<TestPlan>>,
1223    cx: &mut TestAppContext,
1224) -> Result<()> {
1225    let project = choose_random_project(client, &mut plan.lock().rng).unwrap();
1226    let has_buffers_for_project = !client.buffers_for_project(&project).is_empty();
1227    let buffer = if !has_buffers_for_project || plan.lock().rng.gen() {
1228        let Some(worktree) = project.read_with(cx, |project, cx| {
1229            project
1230                .worktrees(cx)
1231                .filter(|worktree| {
1232                    let worktree = worktree.read(cx);
1233                    worktree.is_visible() && worktree.entries(false).any(|e| e.is_file())
1234                })
1235                .choose(&mut plan.lock().rng)
1236        }) else {
1237            return Ok(());
1238        };
1239
1240        let (worktree_root_name, project_path) = worktree.read_with(cx, |worktree, _| {
1241            let entry = worktree
1242                .entries(false)
1243                .filter(|e| e.is_file())
1244                .choose(&mut plan.lock().rng)
1245                .unwrap();
1246            (
1247                worktree.root_name().to_string(),
1248                (worktree.id(), entry.path.clone()),
1249            )
1250        });
1251        log::info!(
1252            "{}: opening path {:?} in worktree {} ({})",
1253            client.username,
1254            project_path.1,
1255            project_path.0,
1256            worktree_root_name,
1257        );
1258        let buffer = project
1259            .update(cx, |project, cx| {
1260                project.open_buffer(project_path.clone(), cx)
1261            })
1262            .await?;
1263        log::info!(
1264            "{}: opened path {:?} in worktree {} ({}) with buffer id {}",
1265            client.username,
1266            project_path.1,
1267            project_path.0,
1268            worktree_root_name,
1269            buffer.read_with(cx, |buffer, _| buffer.remote_id())
1270        );
1271        client.buffers_for_project(&project).insert(buffer.clone());
1272        buffer
1273    } else {
1274        client
1275            .buffers_for_project(&project)
1276            .iter()
1277            .choose(&mut plan.lock().rng)
1278            .unwrap()
1279            .clone()
1280    };
1281
1282    let choice = plan.lock().rng.gen_range(0..100);
1283    match choice {
1284        0..=9 => {
1285            cx.update(|cx| {
1286                log::info!(
1287                    "{}: dropping buffer {:?}",
1288                    client.username,
1289                    buffer.read(cx).file().unwrap().full_path(cx)
1290                );
1291                client.buffers_for_project(&project).remove(&buffer);
1292                drop(buffer);
1293            });
1294        }
1295        10..=19 => {
1296            let completions = project.update(cx, |project, cx| {
1297                log::info!(
1298                    "{}: requesting completions for buffer {} ({:?})",
1299                    client.username,
1300                    buffer.read(cx).remote_id(),
1301                    buffer.read(cx).file().unwrap().full_path(cx)
1302                );
1303                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1304                project.completions(&buffer, offset, cx)
1305            });
1306            let completions = cx.background().spawn(async move {
1307                completions
1308                    .await
1309                    .map_err(|err| anyhow!("completions request failed: {:?}", err))
1310            });
1311            if plan.lock().rng.gen_bool(0.3) {
1312                log::info!("{}: detaching completions request", client.username);
1313                cx.update(|cx| completions.detach_and_log_err(cx));
1314            } else {
1315                completions.await?;
1316            }
1317        }
1318        20..=29 => {
1319            let code_actions = project.update(cx, |project, cx| {
1320                log::info!(
1321                    "{}: requesting code actions for buffer {} ({:?})",
1322                    client.username,
1323                    buffer.read(cx).remote_id(),
1324                    buffer.read(cx).file().unwrap().full_path(cx)
1325                );
1326                let range = buffer.read(cx).random_byte_range(0, &mut plan.lock().rng);
1327                project.code_actions(&buffer, range, cx)
1328            });
1329            let code_actions = cx.background().spawn(async move {
1330                code_actions
1331                    .await
1332                    .map_err(|err| anyhow!("code actions request failed: {:?}", err))
1333            });
1334            if plan.lock().rng.gen_bool(0.3) {
1335                log::info!("{}: detaching code actions request", client.username);
1336                cx.update(|cx| code_actions.detach_and_log_err(cx));
1337            } else {
1338                code_actions.await?;
1339            }
1340        }
1341        30..=39 if buffer.read_with(cx, |buffer, _| buffer.is_dirty()) => {
1342            let (requested_version, save) = buffer.update(cx, |buffer, cx| {
1343                log::info!(
1344                    "{}: saving buffer {} ({:?})",
1345                    client.username,
1346                    buffer.remote_id(),
1347                    buffer.file().unwrap().full_path(cx)
1348                );
1349                (buffer.version(), buffer.save(cx))
1350            });
1351            let save = cx.background().spawn(async move {
1352                let (saved_version, _, _) = save
1353                    .await
1354                    .map_err(|err| anyhow!("save request failed: {:?}", err))?;
1355                assert!(saved_version.observed_all(&requested_version));
1356                Ok::<_, anyhow::Error>(())
1357            });
1358            if plan.lock().rng.gen_bool(0.3) {
1359                log::info!("{}: detaching save request", client.username);
1360                cx.update(|cx| save.detach_and_log_err(cx));
1361            } else {
1362                save.await?;
1363            }
1364        }
1365        40..=44 => {
1366            let prepare_rename = project.update(cx, |project, cx| {
1367                log::info!(
1368                    "{}: preparing rename for buffer {} ({:?})",
1369                    client.username,
1370                    buffer.read(cx).remote_id(),
1371                    buffer.read(cx).file().unwrap().full_path(cx)
1372                );
1373                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1374                project.prepare_rename(buffer, offset, cx)
1375            });
1376            let prepare_rename = cx.background().spawn(async move {
1377                prepare_rename
1378                    .await
1379                    .map_err(|err| anyhow!("prepare rename request failed: {:?}", err))
1380            });
1381            if plan.lock().rng.gen_bool(0.3) {
1382                log::info!("{}: detaching prepare rename request", client.username);
1383                cx.update(|cx| prepare_rename.detach_and_log_err(cx));
1384            } else {
1385                prepare_rename.await?;
1386            }
1387        }
1388        45..=49 => {
1389            let definitions = project.update(cx, |project, cx| {
1390                log::info!(
1391                    "{}: requesting definitions for buffer {} ({:?})",
1392                    client.username,
1393                    buffer.read(cx).remote_id(),
1394                    buffer.read(cx).file().unwrap().full_path(cx)
1395                );
1396                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1397                project.definition(&buffer, offset, cx)
1398            });
1399            let definitions = cx.background().spawn(async move {
1400                definitions
1401                    .await
1402                    .map_err(|err| anyhow!("definitions request failed: {:?}", err))
1403            });
1404            if plan.lock().rng.gen_bool(0.3) {
1405                log::info!("{}: detaching definitions request", client.username);
1406                cx.update(|cx| definitions.detach_and_log_err(cx));
1407            } else {
1408                let definitions = definitions.await?;
1409                client
1410                    .buffers_for_project(&project)
1411                    .extend(definitions.into_iter().map(|loc| loc.target.buffer));
1412            }
1413        }
1414        50..=54 => {
1415            let highlights = project.update(cx, |project, cx| {
1416                log::info!(
1417                    "{}: requesting highlights for buffer {} ({:?})",
1418                    client.username,
1419                    buffer.read(cx).remote_id(),
1420                    buffer.read(cx).file().unwrap().full_path(cx)
1421                );
1422                let offset = plan.lock().rng.gen_range(0..=buffer.read(cx).len());
1423                project.document_highlights(&buffer, offset, cx)
1424            });
1425            let highlights = cx.background().spawn(async move {
1426                highlights
1427                    .await
1428                    .map_err(|err| anyhow!("highlights request failed: {:?}", err))
1429            });
1430            if plan.lock().rng.gen_bool(0.3) {
1431                log::info!("{}: detaching highlights request", client.username);
1432                cx.update(|cx| highlights.detach_and_log_err(cx));
1433            } else {
1434                highlights.await?;
1435            }
1436        }
1437        55..=59 => {
1438            let search = project.update(cx, |project, cx| {
1439                let query = plan.lock().rng.gen_range('a'..='z');
1440                log::info!("{}: project-wide search {:?}", client.username, query);
1441                project.search(SearchQuery::text(query, false, false), cx)
1442            });
1443            let search = cx.background().spawn(async move {
1444                search
1445                    .await
1446                    .map_err(|err| anyhow!("search request failed: {:?}", err))
1447            });
1448            if plan.lock().rng.gen_bool(0.3) {
1449                log::info!("{}: detaching search request", client.username);
1450                cx.update(|cx| search.detach_and_log_err(cx));
1451            } else {
1452                let search = search.await?;
1453                client
1454                    .buffers_for_project(&project)
1455                    .extend(search.into_keys());
1456            }
1457        }
1458        _ => {
1459            buffer.update(cx, |buffer, cx| {
1460                log::info!(
1461                    "{}: updating buffer {} ({:?})",
1462                    client.username,
1463                    buffer.remote_id(),
1464                    buffer.file().unwrap().full_path(cx)
1465                );
1466                if plan.lock().rng.gen_bool(0.7) {
1467                    buffer.randomly_edit(&mut plan.lock().rng, 5, cx);
1468                } else {
1469                    buffer.randomly_undo_redo(&mut plan.lock().rng, cx);
1470                }
1471            });
1472        }
1473    }
1474
1475    Ok(())
1476}
1477
1478fn choose_random_project(client: &TestClient, rng: &mut StdRng) -> Option<ModelHandle<Project>> {
1479    client
1480        .local_projects()
1481        .iter()
1482        .chain(client.remote_projects().iter())
1483        .choose(rng)
1484        .cloned()
1485}
1486
1487fn gen_file_name(rng: &mut StdRng) -> String {
1488    let mut name = String::new();
1489    for _ in 0..10 {
1490        let letter = rng.gen_range('a'..='z');
1491        name.push(letter);
1492    }
1493    name
1494}