Get basic Wasm runtime running

Isaac Clayton created

Change summary

Cargo.lock                                  |   2 
crates/runner/Cargo.toml                    |   3 
crates/runner/README.md                     |   6 +
crates/runner/build.rs                      |  24 ++++
crates/runner/plugin/Cargo.lock             |  25 ++++
crates/runner/plugin/Cargo.toml             |   2 
crates/runner/plugin/cargo_test.lua         |   6 -
crates/runner/plugin/cargo_test/Cargo.lock  |   7 +
crates/runner/plugin/cargo_test/Cargo.toml  |   7 +
crates/runner/plugin/cargo_test/src/main.rs |  63 ++++++++++
crates/runner/src/lib.rs                    |   3 
crates/runner/src/lua.rs                    |   2 
crates/runner/src/main.rs                   |  67 ++++++++---
crates/runner/src/runtime.rs                |   6 
crates/runner/src/wasm.rs                   | 135 +++++++++++++++++++++++
15 files changed, 329 insertions(+), 29 deletions(-)

Detailed changes

Cargo.lock 🔗 

@@ -4268,6 +4268,8 @@ dependencies = [
 name = "runner"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
+ "bincode",
  "map-macro",
  "mlua",
  "serde",

crates/runner/Cargo.toml 🔗 

@@ -8,4 +8,5 @@ mlua = { version = "0.8.0-beta.5", features = ["lua54", "vendored", "serialize"]
 serde = "1.0"
 map-macro = "0.2"
 wasmtime = "0.37.0"
-# bincode = "1.3"
+anyhow = { version = "1.0", features = ["std"] }
+bincode = "1.3"

crates/runner/README.md 🔗 

@@ -0,0 +1,6 @@
+# Zed's Plugin Runner
+This crate contains a fairly generic interface through which plugins may be added to extend the editor. Currently the intention of this plugin runtime is language server definitions.
+
+Anything that implements the `Runtime` trait may be used as a plugin. Plugin interfaces are declared by implementing the `Interface` trait.
+
+Wasm plugins can be run through `wasmtime`. We plan to add wasi support eventually. We also plan to add macros to generate bindings between Rust plugins compiled to Wasm and the host runtime.

crates/runner/build.rs 🔗 

@@ -0,0 +1,24 @@
+use std::env;
+use std::process::Command;
+
+fn main() {
+    let cwd = std::env::current_dir().unwrap();
+    let plugin_workspace = cwd.join("plugin").join("Cargo.toml");
+    Command::new("cargo")
+        .args(&["clean", "--manifest-path"])
+        .arg(&plugin_workspace)
+        .status()
+        .unwrap();
+    Command::new("cargo")
+        .args(&[
+            "build",
+            "--release",
+            "--target",
+            "wasm32-unknown-unknown",
+            "--manifest-path",
+        ])
+        .arg(&plugin_workspace)
+        .status()
+        .unwrap();
+    println!("cargo:warning=recompiling plugins")
+}

crates/runner/plugin/Cargo.lock 🔗 

@@ -0,0 +1,25 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_test"
+version = "0.1.0"
+dependencies = [
+ "bincode",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"

crates/runner/plugin/cargo_test.lua 🔗 

@@ -1,5 +1,3 @@
-print("initializing plugin...")
-
 query = [[(
     (attribute_item
         (meta_item
@@ -10,13 +8,9 @@ query = [[(
 )]]
 
 function run_test(name)
-    print('running test `' .. name .. '`:')
     local command = 'cargo test -- ' .. name
     local openPop = assert(io.popen(command, 'r'))
     local output = openPop:read('*all')
     openPop:close()
-    print('done running test')
     return output
 end
-
-print("done initializing plugin.")

crates/runner/plugin/cargo_test/src/main.rs 🔗 

@@ -0,0 +1,63 @@
+use core::slice;
+
+#[repr(C)]
+pub struct Buffer {
+    ptr: *const u8,
+    len: usize,
+}
+
+/// Allocates a buffer with an exact size.
+/// We don't return the size because it has to be passed in anyway.
+#[no_mangle]
+pub extern "C" fn __alloc_buffer(len: usize) -> *const u8 {
+    let vec = vec![0; len];
+    let buffer = unsafe { Buffer::from_vec(vec) };
+    return buffer.ptr;
+}
+
+/// Frees a given buffer, requires the size.
+#[no_mangle]
+pub extern "C" fn __free_buffer(ptr: *const u8, len: usize) {
+    let buffer = Buffer { ptr, len };
+    let vec = unsafe { buffer.to_vec() };
+    std::mem::drop(vec);
+}
+
+impl Buffer {
+    pub unsafe fn to_vec(&self) -> Vec<u8> {
+        slice::from_raw_parts(self.ptr, self.len).to_vec()
+    }
+
+    pub unsafe fn from_vec(mut vec: Vec<u8>) -> Buffer {
+        vec.shrink_to(0);
+        let ptr = vec.as_ptr();
+        let len = vec.len();
+        std::mem::forget(vec);
+        Buffer { ptr, len }
+    }
+
+    pub fn leak_to_heap(self) -> *const Buffer {
+        let boxed = Box::new(self);
+        let ptr = Box::<Buffer>::into_raw(boxed) as *const Buffer;
+        return ptr;
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn banana(ptr: *const u8, len: usize) -> *const Buffer {
+    // setup
+    let buffer = Buffer { ptr, len };
+    let data = unsafe { buffer.to_vec() };
+    // operation
+    // let reversed: Vec<u8> = data.into_iter().rev().collect();
+    let number: f64 = bincode::deserialize(&data).unwrap();
+    let new_number = number * 2.0;
+    let new_data = bincode::serialize(&new_number).unwrap();
+    // teardown
+    let new_buffer = unsafe { Buffer::from_vec(new_data) };
+    return new_buffer.leak_to_heap();
+}
+
+pub fn main() -> () {
+    ()
+}

crates/runner/src/lua.rs 🔗 

@@ -22,7 +22,7 @@ impl Runtime for Lua {
         Ok(self.from_value(val)?)
     }
 
-    fn call<T: Serialize + DeserializeOwned>(&mut self, handle: &Handle, arg: T) -> Result<T> {
+    fn call<A: Serialize, R: DeserializeOwned>(&mut self, handle: &Handle, arg: A) -> Result<R> {
         let fun: Function = self.globals().get(handle.inner())?;
         let arg: Value = self.to_value(&arg)?;
         let result = fun.call(arg)?;

crates/runner/src/main.rs 🔗 

@@ -2,34 +2,65 @@ use mlua::Lua;
 
 use runner::*;
 
-pub fn main() -> Result<(), mlua::Error> {
-    let source = include_str!("../plugin/cargo_test.lua").to_string();
+// pub fn main() -> Result<(), mlua::Error> {
+//     let source = include_str!("../plugin/cargo_test.lua").to_string();
 
-    let module = LuaPlugin::new(source);
-    // .setup(|runtime| {
-    //     let greet = runtime
-    //         .create_function(|_, name: String| {
-    //             println!("Hello, {}!", name);
-    //             Ok(())
-    //         })
-    //         .map_err(|_| ())?;
+//     let module = LuaPlugin::new(source);
+//     let mut lua: Lua = Runtime::init(module)?;
+//     let runner: TestRunner = lua.as_interface::<TestRunner>().unwrap();
 
-    //     runtime.globals().set("greet", greet).map_err(|_| ())?;
-    //     Ok(())
-    // });
+//     println!("extracted interface: {:#?}", &runner);
 
-    let mut lua: Lua = Runtime::init(module)?;
-    let runner: TestRunner = lua.as_interface::<TestRunner>().unwrap();
+//     let contents = runner.run_test(&mut lua, "it_works".into());
 
-    println!("extracted interface: {:#?}", &runner);
+//     println!("test results:{}", contents.unwrap());
 
-    let contents = runner.run_test(&mut lua, "it_works".into());
+//     Ok(())
+// }
 
-    println!("test results:{}", contents.unwrap());
+// pub fn main() -> mlua::Result<()> {
+//     let module = LuaPlugin::new(include_str!("../plugin/cargo_test.lua").to_string());
+//     let mut lua: Lua = Runtime::init(module)?;
+//     let runner = lua.as_interface::<TestRunner>().unwrap();
+//     let test_results = runner.run_test(&mut lua, "it_works".into());
+//     Ok(())
+// }
+
+pub fn main() -> anyhow::Result<()> {
+    let plugin = WasmPlugin {
+        source_bytes: include_bytes!(
+            "../plugin/target/wasm32-unknown-unknown/release/cargo_test.wasm"
+        )
+        .to_vec(),
+        store_data: (),
+    };
+
+    let mut wasm: Wasm<()> = Runtime::init(plugin)?;
+    let banana = wasm.as_interface::<Banana>().unwrap();
+    let result = banana.banana(&mut wasm, 420.69);
+
+    dbg!("{}", result);
 
     Ok(())
 }
 
+struct Banana {
+    banana: Handle,
+}
+
+impl Interface for Banana {
+    fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self> {
+        let banana = runtime.handle_for("banana")?;
+        Some(Banana { banana })
+    }
+}
+
+impl Banana {
+    fn banana<T: Runtime>(&self, runtime: &mut T, number: f64) -> Option<f64> {
+        runtime.call(&self.banana, number).ok()
+    }
+}
+
 #[allow(dead_code)]
 #[derive(Debug)]
 struct TestRunner {

crates/runner/src/runtime.rs 🔗 

@@ -45,11 +45,11 @@ where
     fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error>;
 
     /// Call a function defined in the module.
-    fn call<T: Serialize + DeserializeOwned>(
+    fn call<A: Serialize, R: DeserializeOwned>(
         &mut self,
         handle: &Handle,
-        arg: T,
-    ) -> Result<T, Self::Error>;
+        arg: A,
+    ) -> Result<R, Self::Error>;
 
     /// Registers a handle with the runtime.
     /// This is a mutable item if needed, but generally

crates/runner/src/wasm.rs 🔗 

@@ -0,0 +1,135 @@
+use std::collections::HashMap;
+
+use anyhow::anyhow;
+
+use wasmtime::{Engine, Func, Instance, Memory, MemoryType, Module, Store, TypedFunc};
+
+use crate::*;
+
+pub struct Wasm<T> {
+    engine: Engine,
+    module: Module,
+    store: Store<T>,
+    instance: Instance,
+    alloc_buffer: TypedFunc<i32, i32>,
+    free_buffer: TypedFunc<(i32, i32), ()>,
+}
+
+pub struct WasmPlugin<T> {
+    pub source_bytes: Vec<u8>,
+    pub store_data: T,
+}
+
+impl<T> Wasm<T> {
+    pub fn dump_memory(data: &[u8]) {
+        for (i, byte) in data.iter().enumerate() {
+            if i % 32 == 0 {
+                println!();
+            }
+            if i % 4 == 0 {
+                print!("|");
+            }
+            if *byte == 0 {
+                print!("__")
+            } else {
+                print!("{:02x}", byte);
+            }
+        }
+        println!();
+    }
+}
+
+impl<S> Runtime for Wasm<S> {
+    type Plugin = WasmPlugin<S>;
+    type Error = anyhow::Error;
+
+    fn init(plugin: WasmPlugin<S>) -> Result<Self, Self::Error> {
+        let engine = Engine::default();
+        let module = Module::new(&engine, plugin.source_bytes)?;
+        let mut store: Store<S> = Store::new(&engine, plugin.store_data);
+        let instance = Instance::new(&mut store, &module, &[])?;
+
+        let alloc_buffer = instance.get_typed_func(&mut store, "__alloc_buffer")?;
+        let free_buffer = instance.get_typed_func(&mut store, "__free_buffer")?;
+
+        Ok(Wasm {
+            engine,
+            module,
+            store,
+            instance,
+            alloc_buffer,
+            free_buffer,
+        })
+    }
+
+    fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error> {
+        let export = self
+            .instance
+            .get_export(&mut self.store, handle.inner())
+            .ok_or_else(|| anyhow!("Could not get export"))?;
+
+        todo!()
+    }
+
+    // TODO: dont' use as for conversions
+    fn call<A: Serialize, R: DeserializeOwned>(
+        &mut self,
+        handle: &Handle,
+        arg: A,
+    ) -> Result<R, Self::Error> {
+        // serialize the argument using bincode
+        let arg = bincode::serialize(&arg)?;
+        let arg_buffer_len = arg.len();
+
+        // allocate a buffer and write the argument to that buffer
+        let arg_buffer_ptr = self
+            .alloc_buffer
+            .call(&mut self.store, arg_buffer_len as i32)?;
+        let plugin_memory = self
+            .instance
+            .get_memory(&mut self.store, "memory")
+            .ok_or_else(|| anyhow!("Could not grab slice of plugin memory"))?;
+        plugin_memory.write(&mut self.store, arg_buffer_ptr as usize, &arg)?;
+
+        // get the webassembly function we want to actually call
+        let fun = self
+            .instance
+            .get_typed_func::<(i32, i32), i32, _>(&mut self.store, handle.inner())?;
+
+        // call the function, passing in the buffer and its length
+        // this should return a pointer to a (ptr, lentgh) pair
+        let result_buffer = fun.call(&mut self.store, (arg_buffer_ptr, arg_buffer_len as i32))?;
+        dbg!(result_buffer);
+
+        // panic!();
+        // dbg!()
+
+        // create a buffer to read the (ptr, length) pair into
+        // this is a total of 4 + 4 = 8 bytes.
+        let buffer = &mut [0; 8];
+        plugin_memory.read(&mut self.store, result_buffer as usize, buffer)?;
+
+        // use these bytes (wasm stores things little-endian)
+        // to get a pointer to the buffer and its length
+        let b = buffer;
+        let result_buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
+        let result_buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]) as usize;
+        let result_buffer_end = result_buffer_ptr + result_buffer_len;
+
+        dbg!(result_buffer_ptr);
+        dbg!(result_buffer_len);
+
+        // read the buffer at this point into a byte array
+        let result = &plugin_memory.data(&mut self.store)[result_buffer_ptr..result_buffer_end];
+
+        // deserialize the byte array into the provided serde type
+        let result = bincode::deserialize(result)?;
+        return Ok(result);
+    }
+
+    fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
+        self.instance
+            .get_export(&mut self.store, name.as_ref())
+            .is_some()
+    }
+}