Change ABI from pair of u32s to single u64

Isaac Clayton created

Change summary

crates/plugin/src/lib.rs                    |  49 ++++---
crates/plugin_macros/src/lib.rs             |  47 +++----
crates/plugin_runtime/src/wasi.rs           | 147 ++++++++++------------
crates/zed/src/languages/language_plugin.rs |  17 +
plugins/json_language/src/lib.rs            |  51 ++++---
5 files changed, 156 insertions(+), 155 deletions(-)

Detailed changes

crates/plugin/src/lib.rs 🔗

@@ -1,50 +1,55 @@
 pub use bincode;
 pub use serde;
 
-#[repr(C)]
+// TODO: move the implementation to one place?
 pub struct __Buffer {
-    pub ptr: *const u8,
-    pub len: usize,
+    pub ptr: u32, // *const u8,
+    pub len: u32, // usize,
+}
+
+impl __Buffer {
+    pub fn into_u64(self) -> u64 {
+        ((self.ptr as u64) << 32) | (self.len as u64)
+    }
+
+    pub fn from_u64(packed: u64) -> Self {
+        __Buffer {
+            ptr: (packed >> 32) as u32,
+            len: packed as u32,
+        }
+    }
 }
 
 /// Allocates a buffer with an exact size.
 /// We don't return the size because it has to be passed in anyway.
 #[no_mangle]
-pub extern "C" fn __alloc_buffer(len: usize) -> *const u8 {
-    let vec = vec![0; len];
+pub extern "C" fn __alloc_buffer(len: u32) -> u32 {
+    let vec = vec![0; len as usize];
     let buffer = unsafe { __Buffer::from_vec(vec) };
     return buffer.ptr;
 }
 
-// /// Frees a given buffer, requires the size.
-// #[no_mangle]
-// pub extern "C" fn __free_buffer(ptr: *const u8, len: usize) {
-//     let buffer = Buffer { ptr, len };
-//     let vec = unsafe { buffer.to_vec() };
-//     std::mem::drop(vec);
-// }
+/// Frees a given buffer, requires the size.
+#[no_mangle]
+pub extern "C" fn __free_buffer(buffer: u64) {
+    let vec = unsafe { __Buffer::from_u64(buffer).to_vec() };
+    std::mem::drop(vec);
+}
 
 impl __Buffer {
     #[inline(always)]
     pub unsafe fn to_vec(&self) -> Vec<u8> {
-        core::slice::from_raw_parts(self.ptr, self.len).to_vec()
+        core::slice::from_raw_parts(self.ptr as *const u8, self.len as usize).to_vec()
     }
 
     #[inline(always)]
     pub unsafe fn from_vec(mut vec: Vec<u8>) -> __Buffer {
         vec.shrink_to(0);
-        let ptr = vec.as_ptr();
-        let len = vec.len();
+        let ptr = vec.as_ptr() as u32;
+        let len = vec.len() as u32;
         std::mem::forget(vec);
         __Buffer { ptr, len }
     }
-
-    #[inline(always)]
-    pub fn leak_to_heap(self) -> *const __Buffer {
-        let boxed = Box::new(self);
-        let ptr = Box::<__Buffer>::into_raw(boxed) as *const __Buffer;
-        return ptr;
-    }
 }
 
 pub mod prelude {

crates/plugin_macros/src/lib.rs 🔗

@@ -59,10 +59,9 @@ pub fn export(args: TokenStream, function: TokenStream) -> TokenStream {
 
         #[no_mangle]
         // TODO: switch len from usize to u32?
-        pub extern "C" fn #outer_fn_name(ptr: *const u8, len: usize) -> *const ::plugin::__Buffer {
+        pub extern "C" fn #outer_fn_name(packed_buffer: u64) -> u64 {
             // setup
-            let buffer = ::plugin::__Buffer { ptr, len };
-            let data = unsafe { buffer.to_vec() };
+            let data = unsafe { ::plugin::__Buffer::from_u64(packed_buffer).to_vec() };
 
             // operation
             let data: #ty = match ::plugin::bincode::deserialize(&data) {
@@ -74,8 +73,8 @@ pub fn export(args: TokenStream, function: TokenStream) -> TokenStream {
             let new_data = new_data.unwrap();
 
             // teardown
-            let new_buffer = unsafe { ::plugin::__Buffer::from_vec(new_data) };
-            return new_buffer.leak_to_heap();
+            let new_buffer = unsafe { ::plugin::__Buffer::from_vec(new_data) }.into_u64();
+            return new_buffer;
         }
     })
 }
@@ -101,8 +100,8 @@ pub fn import(args: TokenStream, function: TokenStream) -> TokenStream {
     //     block: todo!(),
     // };
 
-    // let inner_fn_name = format_ident!("{}", inner_fn.sig.ident);
-    // let outer_fn_name = format_ident!("__{}", inner_fn_name);
+    let outer_fn_name = format_ident!("{}", fn_declare.sig.ident);
+    let inner_fn_name = format_ident!("__{}", outer_fn_name);
 
     //     let variadic = inner_fn.sig.inputs.len();
     //     let i = (0..variadic).map(syn::Index::from);
@@ -135,32 +134,26 @@ pub fn import(args: TokenStream, function: TokenStream) -> TokenStream {
 
     // TokenStream::from(quote! {
     //     extern "C" {
-    //         #[no_mangle]
-    //         fn #outer_fn_name(ptr: *const u8, len: usize) -> *const ::plugin::__Buffer;
+    //         fn #inner_fn_name(buffer: u64) -> u64;
     //     }
 
     //     #[no_mangle]
-    //     fn #inner_fn_name #params -> #output {
-    //         println!("executing command: {}", string);
-    //         // serialize data
-    //         let data = #collect_params;
+    //     fn #outer_fn_name #args /* (string: &str) */ -> #return_type /* Option<Vec<u8>> */ {
+    //         dbg!("executing command: {}", string);
+    //         // setup
+    //         let data = #args_collect;
     //         let data = ::plugin::bincode::serialize(&data).unwrap();
     //         let buffer = unsafe { ::plugin::__Buffer::from_vec(data) };
-    //         let ptr = buffer.ptr;
-    //         let len = buffer.len;
-    //         // leak data to heap
-    //         buffer.leak_to_heap();
-    //         // call extern function
-    //         let result = unsafe { __command(ptr, len) };
-    //         // get result
-    //         let result = todo!(); // convert into box
-
-    //         // deserialize data
-    //         let data: Option<String> = match ::plugin::bincode::deserialize(&data) {
+
+    //         // operation
+    //         let new_buffer = unsafe { #inner_fn_name(buffer.into_u64()) };
+    //         let new_data = unsafe { ::plugin::__Buffer::from_u64(new_buffer).to_vec() };
+
+    //         // teardown
+    //         match ::plugin::bincode::deserialize(&new_data) {
     //             Ok(d) => d,
-    //             Err(e) => panic!("Data passed to function not deserializable."),
-    //         };
-    //         return data;
+    //             Err(e) => panic!("Data returned from function not deserializable."),
+    //         }
     //     }
     // })
     todo!()

crates/plugin_runtime/src/wasi.rs 🔗

@@ -8,15 +8,34 @@ use serde::{de::DeserializeOwned, Serialize};
 use wasi_common::{dir, file};
 use wasmtime::{
     AsContext, AsContextMut, Caller, Config, Engine, Extern, Instance, Linker, Module, Store,
-    StoreContext, StoreContextMut, Trap, TypedFunc,
+    StoreContext, StoreContextMut, Trap, TypedFunc, WasmParams,
 };
 use wasmtime::{IntoFunc, Memory};
 use wasmtime_wasi::{Dir, WasiCtx, WasiCtxBuilder};
 
 pub struct WasiResource(u32);
 
+#[repr(C)]
+struct WasiBuffer {
+    ptr: u32,
+    len: u32,
+}
+
+impl WasiBuffer {
+    pub fn into_u64(self) -> u64 {
+        ((self.ptr as u64) << 32) | (self.len as u64)
+    }
+
+    pub fn from_u64(packed: u64) -> Self {
+        WasiBuffer {
+            ptr: (packed >> 32) as u32,
+            len: packed as u32,
+        }
+    }
+}
+
 pub struct WasiFn<A: Serialize, R: DeserializeOwned> {
-    function: TypedFunc<(u32, u32), u32>,
+    function: TypedFunc<u64, u64>,
     _function_type: PhantomData<fn(A) -> R>,
 }
 
@@ -31,37 +50,6 @@ impl<A: Serialize, R: DeserializeOwned> Clone for WasiFn<A, R> {
     }
 }
 
-// impl<A: Serialize, R: DeserializeOwned> WasiFn<A, R> {
-//     #[inline(always)]
-//     pub async fn call(&self, runtime: &mut Wasi, arg: A) -> Result<R, Error> {
-//         runtime.call(self, arg).await
-//     }
-// }
-
-// type signature derived from:
-// https://docs.rs/wasmtime/latest/wasmtime/struct.Linker.html#method.func_wrap2_async
-// macro_rules! dynHostFunction {
-//     () => {
-//         Box<
-//             dyn for<'a> Fn(Caller<'a, WasiCtx>, u32, u32)
-//                 -> Box<dyn Future<Output = u32> + Send + 'a>
-//                     + Send
-//                     + Sync
-//                     + 'static
-//         >
-//     };
-// }
-
-// macro_rules! implHostFunction {
-//     () => {
-//         impl for<'a> Fn(Caller<'a, WasiCtx>, u32, u32)
-//             -> Box<dyn Future<Output = u32> + Send + 'a>
-//                 + Send
-//                 + Sync
-//                 + 'static
-//     };
-// }
-
 pub struct WasiPluginBuilder {
     wasi_ctx: WasiCtx,
     engine: Engine,
@@ -73,7 +61,7 @@ impl WasiPluginBuilder {
         let mut config = Config::default();
         config.async_support(true);
         let engine = Engine::new(&config)?;
-        let mut linker = Linker::new(&engine);
+        let linker = Linker::new(&engine);
 
         Ok(WasiPluginBuilder {
             // host_functions: HashMap::new(),
@@ -96,10 +84,10 @@ impl WasiPluginBuilder {
         name: &str,
         function: impl Fn(A) -> R + Send + Sync + 'static,
     ) -> Result<Self, Error> {
-        self.linker.func_wrap2_async(
+        self.linker.func_wrap1_async(
             "env",
-            name,
-            move |mut caller: Caller<'_, WasiCtxAlloc>, ptr: u32, len: u32| {
+            &format!("__{}", name),
+            move |mut caller: Caller<'_, WasiCtxAlloc>, packed_buffer: u64| {
                 // TODO: use try block once avaliable
                 let result: Result<(Memory, Vec<u8>), Trap> = (|| {
                     // grab a handle to the memory
@@ -109,8 +97,11 @@ impl WasiPluginBuilder {
                     };
 
                     // get the args passed from Guest
-                    let args =
-                        Wasi::deserialize_from_buffer(&mut plugin_memory, &caller, ptr, len)?;
+                    let args = Wasi::deserialize_from_buffer(
+                        &mut plugin_memory,
+                        &caller,
+                        WasiBuffer::from_u64(packed_buffer),
+                    )?;
 
                     // Call the Host-side function
                     let result: R = function(args);
@@ -125,8 +116,7 @@ impl WasiPluginBuilder {
                 Box::new(async move {
                     let (mut plugin_memory, result) = result?;
 
-                    // todo!();
-                    let (ptr, len) = Wasi::serialize_to_buffer(
+                    let buffer = Wasi::serialize_to_buffer(
                         caller.data().alloc_buffer(),
                         &mut plugin_memory,
                         &mut caller,
@@ -134,7 +124,7 @@ impl WasiPluginBuilder {
                     )
                     .await?;
 
-                    Ok(7u32)
+                    Ok(buffer.into_u64())
                 })
             },
         )?;
@@ -159,7 +149,7 @@ impl WasiPluginBuilder {
 #[derive(Copy, Clone)]
 struct WasiAlloc {
     alloc_buffer: TypedFunc<u32, u32>,
-    free_buffer: TypedFunc<u32, u32>,
+    free_buffer: TypedFunc<u64, ()>,
 }
 
 struct WasiCtxAlloc {
@@ -174,7 +164,7 @@ impl WasiCtxAlloc {
             .alloc_buffer
     }
 
-    fn free_buffer(&self) -> TypedFunc<u32, u32> {
+    fn free_buffer(&self) -> TypedFunc<u64, ()> {
         self.alloc
             .expect("allocator has been not initialized, cannot free buffer!")
             .free_buffer
@@ -351,48 +341,46 @@ impl Wasi {
         plugin_memory: &mut Memory,
         mut store: impl AsContextMut<Data = WasiCtxAlloc>,
         item: Vec<u8>,
-    ) -> Result<(u32, u32), Error> {
+    ) -> Result<WasiBuffer, Error> {
         // allocate a buffer and write the argument to that buffer
-        let buffer_len = item.len() as u32;
-        let buffer_ptr = alloc_buffer.call_async(&mut store, buffer_len).await?;
-        plugin_memory.write(&mut store, buffer_ptr as usize, &item)?;
-        Ok((buffer_ptr, buffer_len))
+        let len = item.len() as u32;
+        let ptr = alloc_buffer.call_async(&mut store, len).await?;
+        plugin_memory.write(&mut store, ptr as usize, &item)?;
+        Ok(WasiBuffer { ptr, len })
     }
 
-    /// Takes `ptr to a `(ptr, len)` pair, and returns `(ptr, len)`.
-    fn deref_buffer(
-        plugin_memory: &mut Memory,
-        store: impl AsContext<Data = WasiCtxAlloc>,
-        buffer: u32,
-    ) -> Result<(u32, u32), Error> {
-        // create a buffer to read the (ptr, length) pair into
-        // this is a total of 4 + 4 = 8 bytes.
-        let raw_buffer = &mut [0; 8];
-        plugin_memory.read(store, buffer as usize, raw_buffer)?;
-
-        // use these bytes (wasm stores things little-endian)
-        // to get a pointer to the buffer and its length
-        let b = raw_buffer;
-        let buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
-        let buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
-
-        return Ok((buffer_ptr, buffer_len));
-    }
+    // /// Takes `ptr to a `(ptr, len)` pair, and returns `(ptr, len)`.
+    // fn deref_buffer(
+    //     plugin_memory: &mut Memory,
+    //     store: impl AsContext<Data = WasiCtxAlloc>,
+    //     buffer: u32,
+    // ) -> Result<(u32, u32), Error> {
+    //     // create a buffer to read the (ptr, length) pair into
+    //     // this is a total of 4 + 4 = 8 bytes.
+    //     let raw_buffer = &mut [0; 8];
+    //     plugin_memory.read(store, buffer as usize, raw_buffer)?;
+
+    //     // use these bytes (wasm stores things little-endian)
+    //     // to get a pointer to the buffer and its length
+    //     let b = raw_buffer;
+    //     let buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
+    //     let buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
+
+    //     return Ok((buffer_ptr, buffer_len));
+    // }
 
     /// Takes a `(ptr, len)` pair and returns the corresponding deserialized buffer.
     fn deserialize_from_buffer<R: DeserializeOwned>(
         plugin_memory: &mut Memory,
         store: impl AsContext<Data = WasiCtxAlloc>,
-        buffer_ptr: u32,
-        buffer_len: u32,
+        buffer: WasiBuffer,
     ) -> Result<R, Error> {
-        let buffer_ptr = buffer_ptr as usize;
-        let buffer_len = buffer_len as usize;
-        let buffer_end = buffer_ptr + buffer_len;
+        let buffer_start = buffer.ptr as usize;
+        let buffer_end = buffer_start + buffer.len as usize;
 
         // read the buffer at this point into a byte array
         // deserialize the byte array into the provided serde type
-        let result = &plugin_memory.data(store.as_context())[buffer_ptr..buffer_end];
+        let result = &plugin_memory.data(store.as_context())[buffer_start..buffer_end];
         let result = bincode::deserialize(result)?;
 
         // TODO: this is handled wasm-side
@@ -402,6 +390,7 @@ impl Wasi {
         Ok(result)
     }
 
+    /// Retrieves the handle to a function of a given type.
     pub fn function<A: Serialize, R: DeserializeOwned, T: AsRef<str>>(
         &mut self,
         name: T,
@@ -409,7 +398,7 @@ impl Wasi {
         let fun_name = format!("__{}", name.as_ref());
         let fun = self
             .instance
-            .get_typed_func::<(u32, u32), u32, _>(&mut self.store, &fun_name)?;
+            .get_typed_func::<u64, u64, _>(&mut self.store, &fun_name)?;
         Ok(WasiFn {
             function: fun,
             _function_type: PhantomData,
@@ -417,6 +406,7 @@ impl Wasi {
     }
 
     // TODO: dont' use as for conversions
+    /// Asynchronously calls a function defined Guest-side.
     pub async fn call<A: Serialize, R: DeserializeOwned>(
         &mut self,
         handle: &WasiFn<A, R>,
@@ -444,16 +434,13 @@ impl Wasi {
         // this returns a ptr to a (ptr, lentgh) pair
         let result_buffer = handle
             .function
-            .call_async(&mut self.store, arg_buffer)
+            .call_async(&mut self.store, arg_buffer.into_u64())
             .await?;
-        let (result_buffer_ptr, result_buffer_len) =
-            Self::deref_buffer(&mut plugin_memory, &mut self.store, result_buffer)?;
 
         Self::deserialize_from_buffer(
             &mut plugin_memory,
             &mut self.store,
-            result_buffer_ptr,
-            result_buffer_len,
+            WasiBuffer::from_u64(result_buffer),
         )
     }
 }

crates/zed/src/languages/language_plugin.rs 🔗

@@ -12,8 +12,14 @@ pub async fn new_json(executor: Arc<Background>) -> Result<PluginLspAdapter> {
     let plugin = WasiPluginBuilder::new_with_default_ctx()?
         .host_function("command", |command: String| {
             // TODO: actual thing
-            std::process::Command::new(command).output().unwrap();
-            Some("Hello".to_string())
+            dbg!(&command);
+            let mut args = command.split(' ');
+            let command = args.next().unwrap();
+            std::process::Command::new(command)
+                .args(args)
+                .output()
+                .log_err()
+                .map(|output| dbg!(output.stdout))
         })?
         .init(include_bytes!("../../../../plugins/bin/json_language.wasm"))
         .await?;
@@ -24,7 +30,7 @@ pub struct PluginLspAdapter {
     name: WasiFn<(), String>,
     server_args: WasiFn<(), Vec<String>>,
     fetch_latest_server_version: WasiFn<(), Option<String>>,
-    fetch_server_binary: WasiFn<(PathBuf, String), Option<PathBuf>>,
+    fetch_server_binary: WasiFn<(PathBuf, String), Result<PathBuf, String>>,
     cached_server_binary: WasiFn<PathBuf, Option<PathBuf>>,
     label_for_completion: WasiFn<String, Option<String>>,
     initialization_options: WasiFn<(), String>,
@@ -102,9 +108,10 @@ impl LspAdapter for PluginLspAdapter {
         async move {
             let mut runtime = runtime.lock().await;
             let handle = runtime.attach_path(&container_dir)?;
-            let result: Option<PathBuf> = runtime.call(&function, (container_dir, version)).await?;
+            let result: Result<PathBuf, String> =
+                runtime.call(&function, (container_dir, version)).await?;
             runtime.remove_resource(handle)?;
-            result.ok_or_else(|| anyhow!("Could not load cached server binary"))
+            result.map_err(|e| anyhow!("{}", e))
         }
         .boxed()
     }

plugins/json_language/src/lib.rs 🔗

@@ -5,7 +5,7 @@ use std::fs;
 use std::path::PathBuf;
 
 // #[import]
-// fn my_command(string: &str) -> Option<String>;
+// fn command(string: &str) -> Option<String>;
 
 // #[no_mangle]
 // // TODO: switch len from usize to u32?
@@ -29,39 +29,34 @@ use std::path::PathBuf;
 // }
 
 extern "C" {
-    fn __command(ptr: *const u8, len: usize) -> *mut ::plugin::__Buffer;
+    fn __command(buffer: u64) -> u64;
 }
 
 #[no_mangle]
-fn command(string: &str) -> Option<String> {
+fn command(string: &str) -> Option<Vec<u8>> {
     dbg!("executing command: {}", string);
-    // serialize data
+    // setup
     let data = string;
     let data = ::plugin::bincode::serialize(&data).unwrap();
     let buffer = unsafe { ::plugin::__Buffer::from_vec(data) };
-    let ptr = buffer.ptr;
-    let len = buffer.len;
-    // leak data to heap
-    buffer.leak_to_heap();
-    // call extern function
-    let result = unsafe { __command(ptr, len) };
-    // get result
-    let new_buffer = unsafe { Box::from_raw(result) }; // convert into box
-    let new_data = unsafe { new_buffer.to_vec() };
-
-    // deserialize data
-    let new_data: Option<String> = match ::plugin::bincode::deserialize(&new_data) {
+
+    // operation
+    let new_buffer = unsafe { __command(buffer.into_u64()) };
+    let new_data = unsafe { ::plugin::__Buffer::from_u64(new_buffer).to_vec() };
+    let new_data: Option<Vec<u8>> = match ::plugin::bincode::deserialize(&new_data) {
         Ok(d) => d,
         Err(e) => panic!("Data returned from function not deserializable."),
     };
+
+    // teardown
     return new_data;
 }
 
 // TODO: some sort of macro to generate ABI bindings
-extern "C" {
-    pub fn hello(item: u32) -> u32;
-    pub fn bye(item: u32) -> u32;
-}
+// extern "C" {
+//     pub fn hello(item: u32) -> u32;
+//     pub fn bye(item: u32) -> u32;
+// }
 
 // #[bind]
 // pub async fn name(u32) -> u32 {
@@ -99,11 +94,15 @@ pub fn fetch_latest_server_version() -> Option<String> {
     }
 
     // TODO: command returns error code
-    let output = command("npm info vscode-json-languageserver --json")?;
+    let output =
+        command("npm info vscode-json-languageserver --json").expect("could not run command");
     // if !output.is_ok() {
     //     return None;
     // }
 
+    let output = String::from_utf8(output).unwrap();
+    dbg!(&output);
+
     let mut info: NpmInfo = serde_json::from_str(&output).ok()?;
     info.versions.pop()
 }
@@ -120,6 +119,8 @@ pub fn fetch_server_binary(container_dir: PathBuf, version: String) -> Result<Pa
             "npm install vscode-json-languageserver@{}",
             version
         ));
+        let output = output.map(String::from_utf8);
+        dbg!(&output);
         if output.is_none() {
             return Err("failed to install vscode-json-languageserver".to_string());
         }
@@ -142,20 +143,28 @@ pub fn fetch_server_binary(container_dir: PathBuf, version: String) -> Result<Pa
 #[export]
 pub fn cached_server_binary(container_dir: PathBuf) -> Option<PathBuf> {
     let mut last_version_dir = None;
+    println!("reading directory");
     let mut entries = fs::read_dir(&container_dir).ok()?;
 
     while let Some(entry) = entries.next() {
+        println!("looking at entries");
         let entry = entry.ok()?;
+        println!("some more stuff");
         if entry.file_type().ok()?.is_dir() {
+            println!("this is it!");
+
             last_version_dir = Some(entry.path());
         }
     }
 
     let last_version_dir = last_version_dir?;
+    println!("here we go");
     let bin_path = last_version_dir.join(BIN_PATH);
     if bin_path.exists() {
+        dbg!(&bin_path);
         Some(bin_path)
     } else {
+        println!("no binary found");
         None
     }
 }