diff --git a/crates/zed/src/visual_test_runner.rs b/crates/zed/src/visual_test_runner.rs index b36342bcf8e7e3658157063cfa3928f625017981..d257df7e7133e910f3e18d20eaf687c4aafb381e 100644 --- a/crates/zed/src/visual_test_runner.rs +++ b/crates/zed/src/visual_test_runner.rs @@ -1,7 +1,7 @@ //! Visual Test Runner //! -//! This binary runs visual tests on the main thread, which is required on macOS -//! because App construction must happen on the main thread. +//! This binary runs visual regression tests for Zed's UI. It captures screenshots +//! of real Zed windows and compares them against baseline images. //! //! ## Prerequisites //! @@ -15,28 +15,49 @@ //! //! ## Usage //! +//! Run the visual tests: //! cargo run -p zed --bin visual_test_runner --features visual-tests //! -//! ## Environment variables +//! Update baseline images (when UI intentionally changes): +//! UPDATE_BASELINE=1 cargo run -p zed --bin visual_test_runner --features visual-tests //! -//! VISUAL_TEST_OUTPUT_DIR - Directory to save screenshots (default: target/visual_tests) +//! ## Environment Variables +//! +//! UPDATE_BASELINE - Set to update baseline images instead of comparing +//! VISUAL_TEST_OUTPUT_DIR - Directory to save test output (default: target/visual_tests) -use anyhow::Result; +use anyhow::{Context, Result}; use gpui::{ AppContext as _, Application, Bounds, Window, WindowBounds, WindowHandle, WindowOptions, point, px, size, }; +use image::RgbaImage; +use project_panel::ProjectPanel; use settings::SettingsStore; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use workspace::{AppState, Workspace}; +/// Baseline images are stored relative to this file +const BASELINE_DIR: &str = "crates/zed/test_fixtures/visual_tests"; + +/// Threshold for image comparison (0.0 to 1.0) +/// Images must match at least this percentage to pass +const MATCH_THRESHOLD: f64 = 0.99; + fn main() { env_logger::builder() .filter_level(log::LevelFilter::Info) .init(); - println!("=== Visual Test Runner ===\n"); + let update_baseline = std::env::var("UPDATE_BASELINE").is_ok(); + + if update_baseline { + println!("=== Visual Test Runner (UPDATE MODE) ===\n"); + println!("Baseline images will be updated.\n"); + } else { + println!("=== Visual Test Runner ===\n"); + } // Create a temporary directory for test files let temp_dir = tempfile::tempdir().expect("Failed to create temp directory"); @@ -44,141 +65,385 @@ fn main() { std::fs::create_dir_all(&project_path).expect("Failed to create project directory"); // Create test files in the real filesystem - println!("Setting up test project at: {:?}", project_path); create_test_files(&project_path); let project_path_clone = project_path.clone(); - Application::new().run(move |cx| { - println!("Initializing Zed..."); - - // Initialize settings store first (required by theme and other subsystems) - let settings_store = SettingsStore::test(cx); - cx.set_global(settings_store); - - // Create AppState using the production-like initialization - let app_state = init_app_state(cx); - - // Initialize all Zed subsystems - gpui_tokio::init(cx); - theme::init(theme::LoadThemes::JustBase, cx); - client::init(&app_state.client, cx); - audio::init(cx); - workspace::init(app_state.clone(), cx); - release_channel::init(semver::Version::new(0, 0, 0), cx); - command_palette::init(cx); - editor::init(cx); - project_panel::init(cx); - outline_panel::init(cx); - terminal_view::init(cx); - image_viewer::init(cx); - search::init(cx); - - println!("Opening Zed workspace..."); - - // Open a real Zed workspace window - let window_size = size(px(1280.0), px(800.0)); - let bounds = Bounds { - origin: point(px(100.0), px(100.0)), - size: window_size, - }; - - // Create a project for the workspace - let project = project::Project::local( - app_state.client.clone(), - app_state.node_runtime.clone(), - app_state.user_store.clone(), - app_state.languages.clone(), - app_state.fs.clone(), - None, - false, - cx, - ); - - let workspace_window: WindowHandle = cx - .open_window( - WindowOptions { - window_bounds: Some(WindowBounds::Windowed(bounds)), - focus: true, - show: true, - ..Default::default() - }, - |window, cx| { - cx.new(|cx| { - Workspace::new(None, project.clone(), app_state.clone(), window, cx) - }) - }, - ) - .expect("Failed to open workspace window"); - - println!("Workspace window opened, adding project folder..."); - - // Add the test project as a worktree - let add_folder_task = workspace_window - .update(cx, |workspace, window, cx| { - workspace.open_paths( - vec![project_path_clone.clone()], - workspace::OpenOptions::default(), - None, - window, - cx, + let test_result = std::panic::catch_unwind(|| { + Application::new().run(move |cx| { + // Initialize settings store first (required by theme and other subsystems) + let settings_store = SettingsStore::test(cx); + cx.set_global(settings_store); + + // Create AppState using the production-like initialization + let app_state = init_app_state(cx); + + // Initialize all Zed subsystems + gpui_tokio::init(cx); + theme::init(theme::LoadThemes::JustBase, cx); + client::init(&app_state.client, cx); + audio::init(cx); + workspace::init(app_state.clone(), cx); + release_channel::init(semver::Version::new(0, 0, 0), cx); + command_palette::init(cx); + editor::init(cx); + project_panel::init(cx); + outline_panel::init(cx); + terminal_view::init(cx); + image_viewer::init(cx); + search::init(cx); + + // Open a real Zed workspace window + let window_size = size(px(1280.0), px(800.0)); + let bounds = Bounds { + origin: point(px(100.0), px(100.0)), + size: window_size, + }; + + // Create a project for the workspace + let project = project::Project::local( + app_state.client.clone(), + app_state.node_runtime.clone(), + app_state.user_store.clone(), + app_state.languages.clone(), + app_state.fs.clone(), + None, + false, + cx, + ); + + let workspace_window: WindowHandle = cx + .open_window( + WindowOptions { + window_bounds: Some(WindowBounds::Windowed(bounds)), + focus: true, + show: true, + ..Default::default() + }, + |window, cx| { + cx.new(|cx| { + Workspace::new(None, project.clone(), app_state.clone(), window, cx) + }) + }, ) - }) - .expect("Failed to update workspace"); - - // Spawn async task to wait for project to load, then capture screenshot - cx.spawn(async move |mut cx| { - // Wait for the folder to be added - println!("Waiting for project to load..."); - add_folder_task.await; - - // Wait for the UI to fully render - println!("Waiting for UI to stabilize..."); - cx.background_executor() - .timer(std::time::Duration::from_secs(2)) - .await; + .expect("Failed to open workspace window"); - println!("Capturing screenshot..."); - - // Try multiple times in case the first attempt fails - let mut result = Err(anyhow::anyhow!("No capture attempts")); - for attempt in 1..=3 { - println!("Capture attempt {}...", attempt); - result = capture_screenshot(workspace_window.into(), &mut cx).await; - if result.is_ok() { - break; - } - if attempt < 3 { - println!("Attempt {} failed, retrying...", attempt); - cx.background_executor() - .timer(std::time::Duration::from_millis(500)) - .await; + // Add the test project as a worktree directly to the project + let add_worktree_task = workspace_window + .update(cx, |workspace, _window, cx| { + workspace.project().update(cx, |project, cx| { + project.find_or_create_worktree(&project_path_clone, true, cx) + }) + }) + .expect("Failed to update workspace"); + + // Spawn async task to set up the UI and capture screenshot + cx.spawn(async move |mut cx| { + // Wait for the worktree to be added + if let Err(e) = add_worktree_task.await { + eprintln!("Failed to add worktree: {:?}", e); } - } - match result { - Ok(path) => { - println!("\n=== Visual Test PASSED ==="); - println!("Screenshot saved to: {}", path); + // Wait for UI to settle + cx.background_executor() + .timer(std::time::Duration::from_millis(500)) + .await; + + // Open the project panel + cx.update(|cx| { + workspace_window + .update(cx, |workspace, window, cx| { + workspace.open_panel::(window, cx); + }) + .ok(); + }) + .ok(); + + // Wait for project panel to render + cx.background_executor() + .timer(std::time::Duration::from_millis(500)) + .await; + + // Open main.rs in the editor + let open_file_task = cx.update(|cx| { + workspace_window + .update(cx, |workspace, window, cx| { + let worktree = workspace.project().read(cx).worktrees(cx).next(); + if let Some(worktree) = worktree { + let worktree_id = worktree.read(cx).id(); + let rel_path: std::sync::Arc = + util::rel_path::rel_path("src/main.rs").into(); + let project_path: project::ProjectPath = + (worktree_id, rel_path.clone()).into(); + Some(workspace.open_path(project_path, None, true, window, cx)) + } else { + None + } + }) + .ok() + .flatten() + }); + + if let Ok(Some(task)) = open_file_task { + if let Ok(item) = task.await { + // Focus the opened item to dismiss the welcome screen + cx.update(|cx| { + workspace_window + .update(cx, |workspace, window, cx| { + let pane = workspace.active_pane().clone(); + pane.update(cx, |pane, cx| { + if let Some(index) = pane.index_for_item(item.as_ref()) { + pane.activate_item(index, true, true, window, cx); + } + }); + }) + .ok(); + }) + .ok(); + + // Wait for item activation to render + cx.background_executor() + .timer(std::time::Duration::from_millis(500)) + .await; + } } - Err(e) => { - eprintln!("\n=== Visual Test FAILED ==="); - eprintln!("Error: {}", e); - eprintln!(); - eprintln!("If you see 'Screen Recording permission' errors:"); - eprintln!(" 1. Open System Settings > Privacy & Security > Screen Recording"); - eprintln!(" 2. Enable your terminal app (Terminal.app, iTerm2, etc.)"); - eprintln!(" 3. Restart your terminal and try again"); + + // Wait for UI to fully stabilize + cx.background_executor() + .timer(std::time::Duration::from_secs(2)) + .await; + + // Run the visual test + let test_result = run_visual_test( + "workspace_with_editor", + workspace_window.into(), + &mut cx, + update_baseline, + ) + .await; + + match test_result { + Ok(TestResult::Passed) => { + println!("\n=== Visual Test PASSED ==="); + } + Ok(TestResult::BaselineUpdated(path)) => { + println!("\n=== Baseline Updated ==="); + println!("New baseline saved to: {}", path.display()); + } + Err(e) => { + eprintln!("\n=== Visual Test FAILED ==="); + eprintln!("Error: {}", e); + std::process::exit(1); + } } - } - cx.update(|cx| cx.quit()).ok(); - }) - .detach(); + cx.update(|cx| cx.quit()).ok(); + }) + .detach(); + }); }); - // Keep temp_dir alive until we're done - it will be dropped here + // Keep temp_dir alive until we're done drop(temp_dir); + + if test_result.is_err() { + std::process::exit(1); + } +} + +enum TestResult { + Passed, + BaselineUpdated(PathBuf), +} + +async fn run_visual_test( + test_name: &str, + window: gpui::AnyWindowHandle, + cx: &mut gpui::AsyncApp, + update_baseline: bool, +) -> Result { + // Capture the screenshot + let screenshot = capture_screenshot(window, cx).await?; + + // Get paths + let baseline_path = get_baseline_path(test_name); + let output_dir = std::env::var("VISUAL_TEST_OUTPUT_DIR") + .unwrap_or_else(|_| "target/visual_tests".to_string()); + let actual_path = Path::new(&output_dir).join(format!("{}.png", test_name)); + + // Create output directory + if let Some(parent) = actual_path.parent() { + std::fs::create_dir_all(parent)?; + } + + // Save the actual screenshot + screenshot.save(&actual_path)?; + println!("Screenshot saved to: {}", actual_path.display()); + + if update_baseline { + // Update the baseline + if let Some(parent) = baseline_path.parent() { + std::fs::create_dir_all(parent)?; + } + screenshot.save(&baseline_path)?; + return Ok(TestResult::BaselineUpdated(baseline_path)); + } + + // Compare against baseline + if !baseline_path.exists() { + return Err(anyhow::anyhow!( + "Baseline image not found: {}\n\ + Run with UPDATE_BASELINE=1 to create it.", + baseline_path.display() + )); + } + + let baseline = image::open(&baseline_path) + .context("Failed to load baseline image")? + .to_rgba8(); + + let comparison = compare_images(&baseline, &screenshot); + + println!( + "Image comparison: {:.2}% match ({} different pixels out of {})", + comparison.match_percentage * 100.0, + comparison.diff_pixel_count, + comparison.total_pixels + ); + + if comparison.match_percentage >= MATCH_THRESHOLD { + Ok(TestResult::Passed) + } else { + // Save the diff image for debugging + if let Some(diff_image) = comparison.diff_image { + let diff_path = Path::new(&output_dir).join(format!("{}_diff.png", test_name)); + diff_image.save(&diff_path)?; + println!("Diff image saved to: {}", diff_path.display()); + } + + Err(anyhow::anyhow!( + "Screenshot does not match baseline.\n\ + Match: {:.2}% (threshold: {:.2}%)\n\ + Actual: {}\n\ + Baseline: {}\n\ + \n\ + Run with UPDATE_BASELINE=1 to update the baseline if this change is intentional.", + comparison.match_percentage * 100.0, + MATCH_THRESHOLD * 100.0, + actual_path.display(), + baseline_path.display() + )) + } +} + +fn get_baseline_path(test_name: &str) -> PathBuf { + // Find the workspace root by looking for Cargo.toml + let mut path = std::env::current_dir().expect("Failed to get current directory"); + while !path.join("Cargo.toml").exists() || !path.join("crates").exists() { + if !path.pop() { + panic!("Could not find workspace root"); + } + } + path.join(BASELINE_DIR).join(format!("{}.png", test_name)) +} + +struct ImageComparison { + match_percentage: f64, + diff_image: Option, + diff_pixel_count: u64, + total_pixels: u64, +} + +fn compare_images(baseline: &RgbaImage, actual: &RgbaImage) -> ImageComparison { + // Check dimensions + if baseline.dimensions() != actual.dimensions() { + return ImageComparison { + match_percentage: 0.0, + diff_image: None, + diff_pixel_count: baseline.width() as u64 * baseline.height() as u64, + total_pixels: baseline.width() as u64 * baseline.height() as u64, + }; + } + + let (width, height) = baseline.dimensions(); + let total_pixels = width as u64 * height as u64; + let mut diff_count: u64 = 0; + let mut diff_image = RgbaImage::new(width, height); + + for y in 0..height { + for x in 0..width { + let baseline_pixel = baseline.get_pixel(x, y); + let actual_pixel = actual.get_pixel(x, y); + + if pixels_are_similar(baseline_pixel, actual_pixel) { + // Matching pixel - show as dimmed version of actual + diff_image.put_pixel( + x, + y, + image::Rgba([ + actual_pixel[0] / 3, + actual_pixel[1] / 3, + actual_pixel[2] / 3, + 255, + ]), + ); + } else { + diff_count += 1; + // Different pixel - highlight in red + diff_image.put_pixel(x, y, image::Rgba([255, 0, 0, 255])); + } + } + } + + let match_percentage = if total_pixels > 0 { + (total_pixels - diff_count) as f64 / total_pixels as f64 + } else { + 1.0 + }; + + ImageComparison { + match_percentage, + diff_image: Some(diff_image), + diff_pixel_count: diff_count, + total_pixels, + } +} + +fn pixels_are_similar(a: &image::Rgba, b: &image::Rgba) -> bool { + // Allow small differences due to anti-aliasing, font rendering, etc. + const TOLERANCE: i16 = 2; + + (a[0] as i16 - b[0] as i16).abs() <= TOLERANCE + && (a[1] as i16 - b[1] as i16).abs() <= TOLERANCE + && (a[2] as i16 - b[2] as i16).abs() <= TOLERANCE + && (a[3] as i16 - b[3] as i16).abs() <= TOLERANCE +} + +async fn capture_screenshot( + window: gpui::AnyWindowHandle, + cx: &mut gpui::AsyncApp, +) -> Result { + // Get the native window ID + let window_id = cx + .update(|cx| { + cx.update_window(window, |_view, window: &mut Window, _cx| { + window.native_window_id() + }) + })?? + .ok_or_else(|| anyhow::anyhow!("Failed to get native window ID"))?; + + // Capture the screenshot + let screenshot = gpui::capture_window_screenshot(window_id) + .await + .map_err(|_| anyhow::anyhow!("Screenshot capture was cancelled"))??; + + println!( + "Screenshot captured: {}x{} pixels", + screenshot.width(), + screenshot.height() + ); + + Ok(screenshot) } /// Create test files in a real filesystem directory @@ -311,7 +576,6 @@ cargo test } /// Initialize AppState with real filesystem for visual testing. -/// This creates a minimal AppState without FakeFs to avoid test dispatcher issues. fn init_app_state(cx: &mut gpui::App) -> Arc { use client::Client; use clock::FakeSystemClock; @@ -340,49 +604,3 @@ fn init_app_state(cx: &mut gpui::App) -> Arc { session, }) } - -async fn capture_screenshot( - window: gpui::AnyWindowHandle, - cx: &mut gpui::AsyncApp, -) -> Result { - // Get the native window ID - let window_id = cx - .update(|cx| { - cx.update_window(window, |_view, window: &mut Window, _cx| { - window.native_window_id() - }) - })?? - .ok_or_else(|| anyhow::anyhow!("Failed to get native window ID"))?; - - println!("Window ID: {}", window_id); - - // Capture the screenshot - let screenshot = gpui::capture_window_screenshot(window_id) - .await - .map_err(|_| anyhow::anyhow!("Screenshot capture was cancelled"))??; - - println!( - "Screenshot captured: {}x{} pixels", - screenshot.width(), - screenshot.height() - ); - - // Determine output path - let output_dir = - std::env::var("VISUAL_TEST_OUTPUT_DIR").unwrap_or_else(|_| "target/visual_tests".into()); - let output_path = Path::new(&output_dir).join("zed_workspace.png"); - - // Create output directory - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - - // Save the screenshot - screenshot.save(&output_path)?; - - // Return absolute path - let abs_path = output_path - .canonicalize() - .unwrap_or_else(|_| output_path.clone()); - Ok(abs_path.display().to_string()) -} diff --git a/crates/zed/test_fixtures/visual_tests/workspace_with_editor.png b/crates/zed/test_fixtures/visual_tests/workspace_with_editor.png new file mode 100644 index 0000000000000000000000000000000000000000..e5aeb4450e7c480e2660c1ee3b915b4f6fd0e93c Binary files /dev/null and b/crates/zed/test_fixtures/visual_tests/workspace_with_editor.png differ