Try to download wasi-sdk ahead of time (#42377)

Conrad Irwin and Ben Kunkle created

This hopefully resolves the lingering test failures on linux,
but also adds some logging just in case this isn't the problem...

Release Notes:

- N/A

---------

Co-authored-by: Ben Kunkle <ben@zed.dev>

Change summary

.github/workflows/compare_perf.yml                |  3 
.github/workflows/release.yml                     | 17 +++-
.github/workflows/release_nightly.yml             |  6 +
.github/workflows/run_agent_evals.yml             |  3 
.github/workflows/run_bundling.yml                |  6 +
.github/workflows/run_tests.yml                   | 28 +++++--
.github/workflows/run_unit_evals.yml              |  3 
crates/extension_host/src/extension_store_test.rs |  8 +
crates/zlog/src/zlog.rs                           | 13 +-
script/download-wasi-sdk                          | 60 +++++++++++++++++
tooling/xtask/src/tasks/workflows/run_tests.rs    |  8 +-
tooling/xtask/src/tasks/workflows/steps.rs        |  8 +
12 files changed, 138 insertions(+), 25 deletions(-)

Detailed changes

.github/workflows/compare_perf.yml 🔗

@@ -35,6 +35,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: compare_perf::run_perf::install_hyperfine
       run: cargo install hyperfine
       shell: bash -euxo pipefail {0}

.github/workflows/release.yml 🔗

@@ -57,16 +57,19 @@ jobs:
         mkdir -p ./../.cargo
         cp ./.cargo/ci-config.toml ./../.cargo/config.toml
       shell: bash -euxo pipefail {0}
+    - name: steps::cache_rust_dependencies_namespace
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        cache: rust
     - name: steps::setup_linux
       run: ./script/linux
       shell: bash -euxo pipefail {0}
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
-    - name: steps::cache_rust_dependencies_namespace
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        cache: rust
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: steps::setup_node
       uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
       with:
@@ -202,6 +205,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}
@@ -242,6 +248,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}

.github/workflows/release_nightly.yml 🔗

@@ -93,6 +93,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}
@@ -140,6 +143,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}

.github/workflows/run_agent_evals.yml 🔗

@@ -40,6 +40,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: steps::setup_cargo_config
       run: |
         mkdir -p ./../.cargo

.github/workflows/run_bundling.yml 🔗

@@ -34,6 +34,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}
@@ -74,6 +77,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: ./script/bundle-linux
       run: ./script/bundle-linux
       shell: bash -euxo pipefail {0}

.github/workflows/run_tests.yml 🔗

@@ -143,16 +143,19 @@ jobs:
         mkdir -p ./../.cargo
         cp ./.cargo/ci-config.toml ./../.cargo/config.toml
       shell: bash -euxo pipefail {0}
+    - name: steps::cache_rust_dependencies_namespace
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        cache: rust
     - name: steps::setup_linux
       run: ./script/linux
       shell: bash -euxo pipefail {0}
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
-    - name: steps::cache_rust_dependencies_namespace
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        cache: rust
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: steps::setup_node
       uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
       with:
@@ -232,6 +235,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: steps::setup_cargo_config
       run: |
         mkdir -p ./../.cargo
@@ -263,16 +269,19 @@ jobs:
         mkdir -p ./../.cargo
         cp ./.cargo/ci-config.toml ./../.cargo/config.toml
       shell: bash -euxo pipefail {0}
+    - name: steps::cache_rust_dependencies_namespace
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        cache: rust
     - name: steps::setup_linux
       run: ./script/linux
       shell: bash -euxo pipefail {0}
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
-    - name: steps::cache_rust_dependencies_namespace
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        cache: rust
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: cargo build -p collab
       run: cargo build -p collab
       shell: bash -euxo pipefail {0}
@@ -348,6 +357,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: run_tests::check_docs::install_mdbook
       uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08
       with:

.github/workflows/run_unit_evals.yml 🔗

@@ -33,6 +33,9 @@ jobs:
     - name: steps::install_mold
       run: ./script/install-mold
       shell: bash -euxo pipefail {0}
+    - name: steps::download_wasi_sdk
+      run: ./script/download-wasi-sdk
+      shell: bash -euxo pipefail {0}
     - name: steps::cargo_install_nextest
       run: cargo install cargo-nextest --locked
       shell: bash -euxo pipefail {0}

crates/extension_host/src/extension_store_test.rs 🔗

@@ -31,8 +31,7 @@ use util::test::TempTree;
 #[cfg(test)]
 #[ctor::ctor]
 fn init_logger() {
-    // show info logs while we debug the extension_store tests hanging.
-    zlog::init_test_with("info");
+    zlog::init_test();
 }
 
 #[gpui::test]
@@ -532,6 +531,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
 
 #[gpui::test]
 async fn test_extension_store_with_test_extension(cx: &mut TestAppContext) {
+    log::info!("Initializing test");
     init_test(cx);
     cx.executor().allow_parking();
 
@@ -556,6 +556,8 @@ async fn test_extension_store_with_test_extension(cx: &mut TestAppContext) {
     let extensions_dir = extensions_tree.path().canonicalize().unwrap();
     let project_dir = project_dir.path().canonicalize().unwrap();
 
+    log::info!("Setting up test");
+
     let project = Project::test(fs.clone(), [project_dir.as_path()], cx).await;
 
     let proxy = Arc::new(ExtensionHostProxy::new());
@@ -674,6 +676,8 @@ async fn test_extension_store_with_test_extension(cx: &mut TestAppContext) {
         )
     });
 
+    log::info!("Flushing events");
+
     // Ensure that debounces fire.
     let mut events = cx.events(&extension_store);
     let executor = cx.executor();

crates/zlog/src/zlog.rs 🔗

@@ -30,16 +30,17 @@ pub fn init_test() {
     }
 }
 
-pub fn init_test_with(filter: &str) {
-    if try_init(Some(filter.to_owned())).is_ok() {
-        init_output_stdout();
-    }
-}
-
 fn get_env_config() -> Option<String> {
     std::env::var("ZED_LOG")
         .or_else(|_| std::env::var("RUST_LOG"))
         .ok()
+        .or_else(|| {
+            if std::env::var("CI").is_ok() {
+                Some("info".to_owned())
+            } else {
+                None
+            }
+        })
 }
 
 pub fn process_env(filter: Option<String>) {

script/download-wasi-sdk 🔗

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Check if ./target/wasi-sdk exists
+if [ ! -d "./target/wasi-sdk" ]; then
+    echo "WASI SDK not found, downloading v25..."
+
+    # Determine OS and architecture
+    OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+    ARCH=$(uname -m)
+
+    # Map architecture names to WASI SDK format
+    case $ARCH in
+        x86_64)
+            ARCH="x86_64"
+            ;;
+        arm64|aarch64)
+            ARCH="arm64"
+            ;;
+        *)
+            echo "Unsupported architecture: $ARCH"
+            exit 1
+            ;;
+    esac
+
+    # Map OS names to WASI SDK format
+    case $OS in
+        darwin)
+            OS="macos"
+            ;;
+        linux)
+            OS="linux"
+            ;;
+        mingw*|msys*|cygwin*)
+            OS="mingw"
+            ;;
+        *)
+            echo "Unsupported OS: $OS"
+            exit 1
+            ;;
+    esac
+
+    # Construct download URL
+    WASI_SDK_VERSION="25"
+    WASI_SDK_URL="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-${ARCH}-${OS}.tar.gz"
+
+    echo "Downloading from: $WASI_SDK_URL"
+
+    # Create target directory if it doesn't exist
+    mkdir -p ./target
+
+    # Download and extract
+    curl -L "$WASI_SDK_URL" | tar -xz -C ./target
+
+    # Rename the extracted directory to wasi-sdk
+    mv "./target/wasi-sdk-${WASI_SDK_VERSION}.0-${ARCH}-${OS}" "./target/wasi-sdk"
+
+    echo "WASI SDK v25 installed successfully"
+else
+    echo "WASI SDK already exists at ./target/wasi-sdk"
+fi

tooling/xtask/src/tasks/workflows/run_tests.rs 🔗

@@ -292,8 +292,8 @@ fn check_workspace_binaries() -> NamedJob {
             .runs_on(runners::LINUX_LARGE)
             .add_step(steps::checkout_repo())
             .add_step(steps::setup_cargo_config(Platform::Linux))
-            .map(steps::install_linux_dependencies)
             .add_step(steps::cache_rust_dependencies_namespace())
+            .map(steps::install_linux_dependencies)
             .add_step(steps::script("cargo build -p collab"))
             .add_step(steps::script("cargo build --workspace --bins --examples"))
             .add_step(steps::cleanup_cargo_config(Platform::Linux)),
@@ -312,13 +312,13 @@ pub(crate) fn run_platform_tests(platform: Platform) -> NamedJob {
             .runs_on(runner)
             .add_step(steps::checkout_repo())
             .add_step(steps::setup_cargo_config(platform))
+            .when(platform == Platform::Linux, |this| {
+                this.add_step(steps::cache_rust_dependencies_namespace())
+            })
             .when(
                 platform == Platform::Linux,
                 steps::install_linux_dependencies,
             )
-            .when(platform == Platform::Linux, |this| {
-                this.add_step(steps::cache_rust_dependencies_namespace())
-            })
             .add_step(steps::setup_node())
             .add_step(steps::clippy(platform))
             .add_step(steps::cargo_install_nextest(platform))

tooling/xtask/src/tasks/workflows/steps.rs 🔗

@@ -113,8 +113,14 @@ fn install_mold() -> Step<Run> {
     named::bash("./script/install-mold")
 }
 
+fn download_wasi_sdk() -> Step<Run> {
+    named::bash("./script/download-wasi-sdk")
+}
+
 pub(crate) fn install_linux_dependencies(job: Job) -> Job {
-    job.add_step(setup_linux()).add_step(install_mold())
+    job.add_step(setup_linux())
+        .add_step(install_mold())
+        .add_step(download_wasi_sdk())
 }
 
 pub fn script(name: &str) -> Step<Run> {