From 80b45ef93bbafdd483a5704ab47cac88a3197ac0 Mon Sep 17 00:00:00 2001 From: Isaac Clayton Date: Thu, 14 Jul 2022 13:23:04 +0200 Subject: [PATCH 1/3] Precompile plugins depending on target triple --- crates/plugin_runtime/Cargo.toml | 2 +- crates/plugin_runtime/build.rs | 19 ++++++--- crates/zed/build.rs | 4 ++ crates/zed/src/languages.rs | 6 +++ crates/zed/src/languages/language_plugin.rs | 45 +++++++++++---------- styles/package-lock.json | 1 + 6 files changed, 50 insertions(+), 27 deletions(-) diff --git a/crates/plugin_runtime/Cargo.toml b/crates/plugin_runtime/Cargo.toml index 21c1ab973d44132ab59415a50034be13b661dee1..a8c0a063a87c2a2972933409f5e965249f42d8f1 100644 --- a/crates/plugin_runtime/Cargo.toml +++ b/crates/plugin_runtime/Cargo.toml @@ -15,4 +15,4 @@ pollster = "0.2.5" smol = "1.2.5" [build-dependencies] -wasmtime = "0.38" +wasmtime = { version = "0.38", features = ["all-arch"] } diff --git a/crates/plugin_runtime/build.rs b/crates/plugin_runtime/build.rs index 8a0c5c57de75a425d3b9faf80714a630da27f138..49c3f89a808b0db003f8f0522e2e4b1b2985d6de 100644 --- a/crates/plugin_runtime/build.rs +++ b/crates/plugin_runtime/build.rs @@ -27,6 +27,11 @@ fn main() { unknown => panic!("unknown profile `{}`", unknown), }; + // Get the target architecture for pre-cross-compilation of plugins + // and write it to disk to be used when embedding plugins + let target_triple = std::env::var("TARGET").unwrap().to_string(); + println!("cargo:rerun-if-env-changed=TARGET"); + // Invoke cargo to build the plugins let build_successful = std::process::Command::new("cargo") .args([ @@ -43,7 +48,7 @@ fn main() { assert!(build_successful); // Find all compiled binaries - let engine = create_default_engine(); + let engine = create_default_engine(&target_triple); let binaries = std::fs::read_dir(base.join("target/wasm32-wasi").join(profile_target)) .expect("Could not find compiled plugins in target"); @@ -61,7 +66,7 @@ fn main() { if let Some(path) = is_wasm() { let out_path = base.join("bin").join(path.file_name().unwrap()); std::fs::copy(&path, &out_path).expect("Could not copy compiled plugin to bin"); - precompile(&out_path, &engine); + precompile(&out_path, &engine, &target_triple); } } } @@ -69,21 +74,25 @@ fn main() { /// Creates an engine with the default configuration. /// N.B. This must create an engine with the same config as the one /// in `plugin_runtime/src/plugin.rs`. -fn create_default_engine() -> Engine { +fn create_default_engine(target_triple: &str) -> Engine { let mut config = Config::default(); + config + .target(target_triple) + .expect(&format!("Could not set target to `{}`", target_triple)); config.async_support(true); config.consume_fuel(true); Engine::new(&config).expect("Could not create precompilation engine") } -fn precompile(path: &Path, engine: &Engine) { +fn precompile(path: &Path, engine: &Engine, target_triple: &str) { let bytes = std::fs::read(path).expect("Could not read wasm module"); let compiled = engine .precompile_module(&bytes) .expect("Could not precompile module"); let out_path = path.parent().unwrap().join(&format!( - "{}.pre", + "{}.{}", path.file_name().unwrap().to_string_lossy(), + target_triple, )); let mut out_file = std::fs::File::create(out_path) .expect("Could not create output file for precompiled module"); diff --git a/crates/zed/build.rs b/crates/zed/build.rs index 7a0646dda6d7e25f7ef2d258a3bf74941c420d3e..3d1cb3d170c547346d8bd8b659c698fab774cdf5 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -2,6 +2,10 @@ use std::process::Command; fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.14"); + println!( + "cargo:rustc-env=TARGET_TRIPLE={}", + std::env::var("TARGET").unwrap() + ); let output = Command::new("npm") .current_dir("../../styles") diff --git a/crates/zed/src/languages.rs b/crates/zed/src/languages.rs index 8dc20bdbd16dda3709d3a199c43494bb9b6ef892..8dc37a3afe3f5c47a46086305e960fb545bd6847 100644 --- a/crates/zed/src/languages.rs +++ b/crates/zed/src/languages.rs @@ -3,6 +3,7 @@ pub use language::*; use lazy_static::lazy_static; use rust_embed::RustEmbed; use std::{borrow::Cow, str, sync::Arc}; +// use util::ResultExt; mod c; mod go; @@ -54,6 +55,11 @@ pub async fn init(languages: Arc, _executor: Arc) "json", tree_sitter_json::language(), Some(CachedLspAdapter::new(json::JsonLspAdapter).await), + // TODO: switch back to plugin + // match language_plugin::new_json(executor).await.log_err() { + // Some(lang) => Some(CachedLspAdapter::new(lang).await), + // None => None, + // }, ), ( "markdown", diff --git a/crates/zed/src/languages/language_plugin.rs b/crates/zed/src/languages/language_plugin.rs index 4fbc63a667f507c087d77da3ddc4d1e6c063b914..c37d61f801c2cf073f2af3defd702ab3a527bc09 100644 --- a/crates/zed/src/languages/language_plugin.rs +++ b/crates/zed/src/languages/language_plugin.rs @@ -5,30 +5,33 @@ use collections::HashMap; use futures::lock::Mutex; use gpui::executor::Background; use language::{LanguageServerName, LspAdapter}; -// use plugin_runtime::{Plugin, PluginBinary, PluginBuilder, WasiFn}; -use plugin_runtime::{Plugin, WasiFn}; +use plugin_runtime::{Plugin, PluginBinary, PluginBuilder, WasiFn}; use std::{any::Any, path::PathBuf, sync::Arc}; use util::ResultExt; -// pub async fn new_json(executor: Arc) -> Result { -// let plugin = PluginBuilder::new_default()? -// .host_function_async("command", |command: String| async move { -// let mut args = command.split(' '); -// let command = args.next().unwrap(); -// smol::process::Command::new(command) -// .args(args) -// .output() -// .await -// .log_err() -// .map(|output| output.stdout) -// })? -// .init(PluginBinary::Precompiled(include_bytes!( -// "../../../../plugins/bin/json_language.wasm.pre" -// ))) -// .await?; -// -// PluginLspAdapter::new(plugin, executor).await -// } +#[allow(dead_code)] +pub async fn new_json(executor: Arc) -> Result { + let bytes = include_bytes!(concat!( + "../../../../plugins/bin/json_language.wasm.", + env!("TARGET_TRIPLE"), + )); + + let plugin = PluginBuilder::new_default()? + .host_function_async("command", |command: String| async move { + let mut args = command.split(' '); + let command = args.next().unwrap(); + smol::process::Command::new(command) + .args(args) + .output() + .await + .log_err() + .map(|output| output.stdout) + })? + .init(PluginBinary::Precompiled(bytes)) + .await?; + + PluginLspAdapter::new(plugin, executor).await +} pub struct PluginLspAdapter { name: WasiFn<(), String>, diff --git a/styles/package-lock.json b/styles/package-lock.json index 2eb6d3a1bfaeaa206f0cc8a0a03efa0387d144c2..49304dc2fa98dfec942bf6687be56cefc13cdf80 100644 --- a/styles/package-lock.json +++ b/styles/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "styles", "version": "1.0.0", "license": "ISC", "dependencies": { From af1ad474e3a9cb9e9e6feddb446e19b556f732f7 Mon Sep 17 00:00:00 2001 From: Isaac Clayton Date: Thu, 14 Jul 2022 13:46:41 +0200 Subject: [PATCH 2/3] Update docs --- crates/plugin_runtime/README.md | 34 +++++++++++++++++++++++++-------- crates/plugin_runtime/build.rs | 2 -- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/crates/plugin_runtime/README.md b/crates/plugin_runtime/README.md index 8ac843cb02f0491dbdc09250e51734440abe561a..012f1563625135f5ca088f4e0272b0cb6820f65d 100644 --- a/crates/plugin_runtime/README.md +++ b/crates/plugin_runtime/README.md @@ -152,7 +152,7 @@ Plugins in the `plugins` directory are automatically recompiled and serialized t - `plugin.wasm` is the plugin compiled to Wasm. As a baseline, this should be about 4MB for debug builds and 2MB for release builds, but it depends on the specific plugin being built. -- `plugin.wasm.pre` is the plugin compiled to Wasm *and additionally* precompiled to host-platform-agnostic cranelift-specific IR. This should be about 700KB for debug builds and 500KB in release builds. Each plugin takes about 1 or 2 seconds to compile to native code using cranelift, so precompiling plugins drastically reduces the startup time required to begin to run a plugin. +- `plugin.wasm.the-target-triple` is the plugin compiled to Wasm *and additionally* precompiled to host-platform-specific native code, determined by the `TARGET` cargo exposes at compile-time. This should be about 700KB for debug builds and 500KB in release builds. Each plugin takes about 1 or 2 seconds to compile to native code using cranelift, so precompiling plugins drastically reduces the startup time required to begin to run a plugin. For all intents and purposes, it is *highly recommended* that you use precompiled plugins where possible, as they are much more lightweight and take much less time to instantiate. @@ -246,18 +246,36 @@ Once all imports are marked, we can instantiate the plugin. To instantiate the p ```rust let plugin = builder .init( - true, - include_bytes!("../../../plugins/bin/cool_plugin.wasm.pre"), + PluginBinary::Precompiled(bytes), ) .await .unwrap(); ``` -The `.init` method currently takes two arguments: - -1. First, the 'precompiled' flag, indicating whether the plugin is *normal* (`.wasm`) or precompiled (`.wasm.pre`). When using a precompiled plugin, set this flag to `true`. - -2. Second, the raw plugin Wasm itself, as an array of bytes. When not precompiled, this can be either the Wasm binary format (`.wasm`) or the Wasm textual format (`.wat`). When precompiled, this must be the precompiled plugin (`.wasm.pre`). +The `.init` method takes a single argument containing the plugin binary. + +1. If not precompiled, use `PluginBinary::Wasm(bytes)`. This supports both the WebAssembly Textual format (`.wat`) and the WebAssembly Binary format (`.wasm`). + +2. If precompiled, use `PluginBinary::Precompiled(bytes)`. This supports precompiled plugins ending in `.wasm.the-target-triple`. You need to be extra-careful when using precompiled plugins to ensure that the plugin target matches the target of the binary you are compiling: + + a. To tell the current crate what the current `target` triple is, add the following line to the crate's `build.rs`: + + ```rust + println!( + "cargo:rustc-env=TARGET_TRIPLE={}", + std::env::var("TARGET").unwrap() + ); + ``` + + b. To get the correct precompiled binary in the crate itself, read this exposed environment variable and include the plugin: + + ```rust + let bytes = include_bytes!(concat!( + "../../../plugins/bin/plugin.wasm.", + env!("TARGET_TRIPLE"), + )); + let precompiled = PluginBinary::Precompiled(bytes); + ``` The `.init` method is asynchronous, and must be `.await`ed upon. If the plugin is malformed or doesn't import the right functions, an error will be raised. diff --git a/crates/plugin_runtime/build.rs b/crates/plugin_runtime/build.rs index 49c3f89a808b0db003f8f0522e2e4b1b2985d6de..dbfdebf9a88bcced99daa1d3373b872e2c6623ad 100644 --- a/crates/plugin_runtime/build.rs +++ b/crates/plugin_runtime/build.rs @@ -28,9 +28,7 @@ fn main() { }; // Get the target architecture for pre-cross-compilation of plugins - // and write it to disk to be used when embedding plugins let target_triple = std::env::var("TARGET").unwrap().to_string(); - println!("cargo:rerun-if-env-changed=TARGET"); // Invoke cargo to build the plugins let build_successful = std::process::Command::new("cargo") From b3ac63b7b50f42711e6950e3f4509f326ff77dbe Mon Sep 17 00:00:00 2001 From: Isaac Clayton Date: Thu, 14 Jul 2022 15:05:54 +0200 Subject: [PATCH 3/3] Remove triple-based suffix --- crates/plugin_runtime/README.md | 23 ++------------------- crates/plugin_runtime/build.rs | 18 ++++++++-------- crates/zed/build.rs | 4 ---- crates/zed/src/languages/language_plugin.rs | 9 +++----- 4 files changed, 14 insertions(+), 40 deletions(-) diff --git a/crates/plugin_runtime/README.md b/crates/plugin_runtime/README.md index 012f1563625135f5ca088f4e0272b0cb6820f65d..38d1c0bb5d1d2bc44f0c150c8667774a998befaf 100644 --- a/crates/plugin_runtime/README.md +++ b/crates/plugin_runtime/README.md @@ -152,7 +152,7 @@ Plugins in the `plugins` directory are automatically recompiled and serialized t - `plugin.wasm` is the plugin compiled to Wasm. As a baseline, this should be about 4MB for debug builds and 2MB for release builds, but it depends on the specific plugin being built. -- `plugin.wasm.the-target-triple` is the plugin compiled to Wasm *and additionally* precompiled to host-platform-specific native code, determined by the `TARGET` cargo exposes at compile-time. This should be about 700KB for debug builds and 500KB in release builds. Each plugin takes about 1 or 2 seconds to compile to native code using cranelift, so precompiling plugins drastically reduces the startup time required to begin to run a plugin. +- `plugin.wasm.pre` is the plugin compiled to Wasm *and additionally* precompiled to host-platform-specific native code, determined by the `TARGET` cargo exposes at compile-time. This should be about 700KB for debug builds and 500KB in release builds. Each plugin takes about 1 or 2 seconds to compile to native code using cranelift, so precompiling plugins drastically reduces the startup time required to begin to run a plugin. For all intents and purposes, it is *highly recommended* that you use precompiled plugins where possible, as they are much more lightweight and take much less time to instantiate. @@ -256,26 +256,7 @@ The `.init` method takes a single argument containing the plugin binary. 1. If not precompiled, use `PluginBinary::Wasm(bytes)`. This supports both the WebAssembly Textual format (`.wat`) and the WebAssembly Binary format (`.wasm`). -2. If precompiled, use `PluginBinary::Precompiled(bytes)`. This supports precompiled plugins ending in `.wasm.the-target-triple`. You need to be extra-careful when using precompiled plugins to ensure that the plugin target matches the target of the binary you are compiling: - - a. To tell the current crate what the current `target` triple is, add the following line to the crate's `build.rs`: - - ```rust - println!( - "cargo:rustc-env=TARGET_TRIPLE={}", - std::env::var("TARGET").unwrap() - ); - ``` - - b. To get the correct precompiled binary in the crate itself, read this exposed environment variable and include the plugin: - - ```rust - let bytes = include_bytes!(concat!( - "../../../plugins/bin/plugin.wasm.", - env!("TARGET_TRIPLE"), - )); - let precompiled = PluginBinary::Precompiled(bytes); - ``` +2. If precompiled, use `PluginBinary::Precompiled(bytes)`. This supports precompiled plugins ending in `.wasm.pre`. You need to be extra-careful when using precompiled plugins to ensure that the plugin target matches the target of the binary you are compiling. The `.init` method is asynchronous, and must be `.await`ed upon. If the plugin is malformed or doesn't import the right functions, an error will be raised. diff --git a/crates/plugin_runtime/build.rs b/crates/plugin_runtime/build.rs index dbfdebf9a88bcced99daa1d3373b872e2c6623ad..0614659b5ed19b629df947ff10c92494ab65fdaa 100644 --- a/crates/plugin_runtime/build.rs +++ b/crates/plugin_runtime/build.rs @@ -26,10 +26,6 @@ fn main() { "release" => (&["--release"][..], "release"), unknown => panic!("unknown profile `{}`", unknown), }; - - // Get the target architecture for pre-cross-compilation of plugins - let target_triple = std::env::var("TARGET").unwrap().to_string(); - // Invoke cargo to build the plugins let build_successful = std::process::Command::new("cargo") .args([ @@ -45,8 +41,13 @@ fn main() { .success(); assert!(build_successful); - // Find all compiled binaries + // Get the target architecture for pre-cross-compilation of plugins + // and create and engine with the appropriate config + let target_triple = std::env::var("TARGET").unwrap().to_string(); + println!("cargo:rerun-if-env-changed=TARGET"); let engine = create_default_engine(&target_triple); + + // Find all compiled binaries let binaries = std::fs::read_dir(base.join("target/wasm32-wasi").join(profile_target)) .expect("Could not find compiled plugins in target"); @@ -64,7 +65,7 @@ fn main() { if let Some(path) = is_wasm() { let out_path = base.join("bin").join(path.file_name().unwrap()); std::fs::copy(&path, &out_path).expect("Could not copy compiled plugin to bin"); - precompile(&out_path, &engine, &target_triple); + precompile(&out_path, &engine); } } } @@ -82,15 +83,14 @@ fn create_default_engine(target_triple: &str) -> Engine { Engine::new(&config).expect("Could not create precompilation engine") } -fn precompile(path: &Path, engine: &Engine, target_triple: &str) { +fn precompile(path: &Path, engine: &Engine) { let bytes = std::fs::read(path).expect("Could not read wasm module"); let compiled = engine .precompile_module(&bytes) .expect("Could not precompile module"); let out_path = path.parent().unwrap().join(&format!( - "{}.{}", + "{}.pre", path.file_name().unwrap().to_string_lossy(), - target_triple, )); let mut out_file = std::fs::File::create(out_path) .expect("Could not create output file for precompiled module"); diff --git a/crates/zed/build.rs b/crates/zed/build.rs index 3d1cb3d170c547346d8bd8b659c698fab774cdf5..7a0646dda6d7e25f7ef2d258a3bf74941c420d3e 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -2,10 +2,6 @@ use std::process::Command; fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.14"); - println!( - "cargo:rustc-env=TARGET_TRIPLE={}", - std::env::var("TARGET").unwrap() - ); let output = Command::new("npm") .current_dir("../../styles") diff --git a/crates/zed/src/languages/language_plugin.rs b/crates/zed/src/languages/language_plugin.rs index c37d61f801c2cf073f2af3defd702ab3a527bc09..bd9a9d005fd7643ab880786f92de2055a69a9380 100644 --- a/crates/zed/src/languages/language_plugin.rs +++ b/crates/zed/src/languages/language_plugin.rs @@ -11,11 +11,6 @@ use util::ResultExt; #[allow(dead_code)] pub async fn new_json(executor: Arc) -> Result { - let bytes = include_bytes!(concat!( - "../../../../plugins/bin/json_language.wasm.", - env!("TARGET_TRIPLE"), - )); - let plugin = PluginBuilder::new_default()? .host_function_async("command", |command: String| async move { let mut args = command.split(' '); @@ -27,7 +22,9 @@ pub async fn new_json(executor: Arc) -> Result { .log_err() .map(|output| output.stdout) })? - .init(PluginBinary::Precompiled(bytes)) + .init(PluginBinary::Precompiled(include_bytes!( + "../../../../plugins/bin/json_language.wasm.pre", + ))) .await?; PluginLspAdapter::new(plugin, executor).await