Lay some groundwork for language plugins (#3837)

Max Brunsfeld created 

This PR adds undocumented functionality for loading custom language
plugins at runtime. I don't intend to expose the functionality to end
users yet, but this will allow the team to test the capability
internally.

### Implementation

There isn't much new code in Zed. Most of the work here is within
Tree-sitter, in PRs https://github.com/tree-sitter/tree-sitter/pull/1864
and https://github.com/tree-sitter/tree-sitter/pull/2840, which allow
Tree-sitter to load languages from WASM blobs. I've tested the
functionality in Tree-sitter's test suite and via its CLI, but having it
wired into Zed allows us to test the functionality more fully.

### Details

Now, on startup, Zed will look for subdirectories inside of
`~/Application Support/plugins`. These subdirectories are expected to
look similar to the per-language subdirectories in
[`crates/zed2/src/languages`](https://github.com/zed-industries/zed/tree/main/crates/zed2/src/languages),
except that they also contain a `.wasm` file for the parser itself.

I'll add more details here as I go.

Change summary

Cargo.lock                             | 436 +++++++++++++++++++++++++--
Cargo.toml                             |   7 
crates/language/src/language.rs        |  18 
crates/language/src/syntax_map.rs      |   2 
crates/language2/src/language2.rs      | 109 +++++-
crates/language2/src/syntax_map.rs     |  11 
crates/semantic_index/src/parsing.rs   |   2 
crates/semantic_index2/src/parsing.rs  |   2 
crates/settings/src/settings_store.rs  |   4 
crates/settings2/src/settings_store.rs |   4 
crates/util/src/paths.rs               |   1 
crates/zed2/src/languages.rs           |  17 +
12 files changed, 518 insertions(+), 95 deletions(-)

Detailed changes

Cargo.lock 🔗 

@@ -327,6 +327,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arbitrary"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
+
 [[package]]
 name = "arrayref"
 version = "0.3.7"
@@ -2370,7 +2376,15 @@ version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "593b398dd0c5b1e2e3a9c3dae8584e287894ea84e361949ad506376e99196265"
 dependencies = [
- "cranelift-entity",
+ "cranelift-entity 0.89.2",
+]
+
+[[package]]
+name = "cranelift-bforest"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-entity 0.103.0",
 ]
 
 [[package]]
@@ -2381,14 +2395,34 @@ checksum = "afc0d8faabd099ea15ab33d49d150e5572c04cfeb95d675fd41286739b754629"
 dependencies = [
  "arrayvec 0.7.4",
  "bumpalo",
- "cranelift-bforest",
- "cranelift-codegen-meta",
- "cranelift-codegen-shared",
- "cranelift-entity",
- "cranelift-isle",
+ "cranelift-bforest 0.89.2",
+ "cranelift-codegen-meta 0.89.2",
+ "cranelift-codegen-shared 0.89.2",
+ "cranelift-entity 0.89.2",
+ "cranelift-isle 0.89.2",
  "gimli 0.26.2",
  "log",
- "regalloc2",
+ "regalloc2 0.4.2",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-codegen"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "bumpalo",
+ "cranelift-bforest 0.103.0",
+ "cranelift-codegen-meta 0.103.0",
+ "cranelift-codegen-shared 0.103.0",
+ "cranelift-control",
+ "cranelift-entity 0.103.0",
+ "cranelift-isle 0.103.0",
+ "gimli 0.28.0",
+ "hashbrown 0.14.0",
+ "log",
+ "regalloc2 0.9.3",
  "smallvec",
  "target-lexicon",
 ]
@@ -2399,7 +2433,15 @@ version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ac1669e42579476f001571d6ba4b825fac686282c97b88b18f8e34242066a81"
 dependencies = [
- "cranelift-codegen-shared",
+ "cranelift-codegen-shared 0.89.2",
+]
+
+[[package]]
+name = "cranelift-codegen-meta"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-codegen-shared 0.103.0",
 ]
 
 [[package]]
@@ -2408,6 +2450,19 @@ version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2a1b1eef9640ab72c1e7b583ac678083855a509da34b4b4378bd99954127c20"
 
+[[package]]
+name = "cranelift-codegen-shared"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+
+[[package]]
+name = "cranelift-control"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "arbitrary",
+]
+
 [[package]]
 name = "cranelift-entity"
 version = "0.89.2"
@@ -2417,13 +2472,33 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "cranelift-entity"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
 [[package]]
 name = "cranelift-frontend"
 version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fca1474b5302348799656d43a40eacd716a3b46169405a3af812832c9edf77b4"
 dependencies = [
- "cranelift-codegen",
+ "cranelift-codegen 0.89.2",
+ "log",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-frontend"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-codegen 0.103.0",
  "log",
  "smallvec",
  "target-lexicon",
@@ -2435,13 +2510,28 @@ version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77aa537f020ea43483100153278e7215d41695bdcef9eea6642d122675f64249"
 
+[[package]]
+name = "cranelift-isle"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+
 [[package]]
 name = "cranelift-native"
 version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bdc6b65241a95b7d8eafbf4e114c082e49b80162a2dcd9c6bcc5989c3310c9e"
 dependencies = [
- "cranelift-codegen",
+ "cranelift-codegen 0.89.2",
+ "libc",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-native"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-codegen 0.103.0",
  "libc",
  "target-lexicon",
 ]
@@ -2452,14 +2542,29 @@ version = "0.89.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4eb6359f606a1c80ccaa04fae9dbbb504615ec7a49b6c212b341080fff7a65dd"
 dependencies = [
- "cranelift-codegen",
- "cranelift-entity",
- "cranelift-frontend",
+ "cranelift-codegen 0.89.2",
+ "cranelift-entity 0.89.2",
+ "cranelift-frontend 0.89.2",
+ "itertools 0.10.5",
+ "log",
+ "smallvec",
+ "wasmparser 0.92.0",
+ "wasmtime-types 2.0.2",
+]
+
+[[package]]
+name = "cranelift-wasm"
+version = "0.103.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-codegen 0.103.0",
+ "cranelift-entity 0.103.0",
+ "cranelift-frontend 0.103.0",
  "itertools 0.10.5",
  "log",
  "smallvec",
- "wasmparser",
- "wasmtime-types",
+ "wasmparser 0.118.1",
+ "wasmtime-types 16.0.0",
 ]
 
 [[package]]
@@ -3202,6 +3307,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
+[[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
+
 [[package]]
 name = "fallible-streaming-iterator"
 version = "0.1.9"
@@ -3840,7 +3951,7 @@ version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
 dependencies = [
- "fallible-iterator",
+ "fallible-iterator 0.2.0",
  "indexmap 1.9.3",
  "stable_deref_trait",
 ]
@@ -3850,6 +3961,11 @@ name = "gimli"
 version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
+dependencies = [
+ "fallible-iterator 0.3.0",
+ "indexmap 2.0.0",
+ "stable_deref_trait",
+]
 
 [[package]]
 name = "git"
@@ -4484,6 +4600,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.0",
+ "serde",
 ]
 
 [[package]]
@@ -6181,6 +6298,9 @@ version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
+ "crc32fast",
+ "hashbrown 0.14.0",
+ "indexmap 2.0.0",
  "memchr",
 ]
 
@@ -6706,7 +6826,7 @@ dependencies = [
  "serde_json",
  "smol",
  "wasi-common",
- "wasmtime",
+ "wasmtime 2.0.2",
  "wasmtime-wasi",
 ]
 
@@ -7541,6 +7661,19 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "regalloc2"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6"
+dependencies = [
+ "hashbrown 0.13.2",
+ "log",
+ "rustc-hash",
+ "slice-group-by",
+ "smallvec",
+]
+
 [[package]]
 name = "regex"
 version = "1.9.5"
@@ -7923,7 +8056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
 dependencies = [
  "bitflags 2.4.1",
- "fallible-iterator",
+ "fallible-iterator 0.2.0",
  "fallible-streaming-iterator",
  "hashlink",
  "libsqlite3-sys",
@@ -8994,6 +9127,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be6c3f39c37a4283ee4b43d1311c828f2e1fb0541e76ea0cb1a2abd9ef2f5b3b"
 
+[[package]]
+name = "sptr"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
+
 [[package]]
 name = "sqlez"
 version = "0.1.0"
@@ -10336,10 +10475,12 @@ dependencies = [
 [[package]]
 name = "tree-sitter"
 version = "0.20.10"
-source = "git+https://github.com/tree-sitter/tree-sitter?rev=b5f461a69bf3df7298b1903574d506179e6390b0#b5f461a69bf3df7298b1903574d506179e6390b0"
+source = "git+https://github.com/tree-sitter/tree-sitter?rev=31c40449749c4263a91a43593831b82229049a4c#31c40449749c4263a91a43593831b82229049a4c"
 dependencies = [
  "cc",
  "regex",
+ "wasmtime 16.0.0",
+ "wasmtime-c-api-impl",
 ]
 
 [[package]]
@@ -11212,6 +11353,16 @@ dependencies = [
  "indexmap 1.9.3",
 ]
 
+[[package]]
+name = "wasmparser"
+version = "0.118.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9"
+dependencies = [
+ "indexmap 2.0.0",
+ "semver",
+]
+
 [[package]]
 name = "wasmtime"
 version = "2.0.2"
@@ -11232,17 +11383,44 @@ dependencies = [
  "rayon",
  "serde",
  "target-lexicon",
- "wasmparser",
+ "wasmparser 0.92.0",
  "wasmtime-cache",
- "wasmtime-cranelift",
- "wasmtime-environ",
+ "wasmtime-cranelift 2.0.2",
+ "wasmtime-environ 2.0.2",
  "wasmtime-fiber",
- "wasmtime-jit",
- "wasmtime-runtime",
+ "wasmtime-jit 2.0.2",
+ "wasmtime-runtime 2.0.2",
  "wat",
  "windows-sys 0.36.1",
 ]
 
+[[package]]
+name = "wasmtime"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "bumpalo",
+ "cfg-if 1.0.0",
+ "indexmap 2.0.0",
+ "libc",
+ "log",
+ "object 0.32.1",
+ "once_cell",
+ "paste",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "target-lexicon",
+ "wasmparser 0.118.1",
+ "wasmtime-cranelift 16.0.0",
+ "wasmtime-environ 16.0.0",
+ "wasmtime-jit 16.0.0",
+ "wasmtime-runtime 16.0.0",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "wasmtime-asm-macros"
 version = "2.0.2"
@@ -11252,6 +11430,36 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "wasmtime-asm-macros"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
+[[package]]
+name = "wasmtime-c-api-impl"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "log",
+ "once_cell",
+ "tracing",
+ "wasmtime 16.0.0",
+ "wasmtime-c-api-macros",
+]
+
+[[package]]
+name = "wasmtime-c-api-macros"
+version = "0.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
 [[package]]
 name = "wasmtime-cache"
 version = "2.0.2"
@@ -11279,18 +11487,57 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "017c3605ccce867b3ba7f71d95e5652acc22b9dc2971ad6a6f9df4a8d7af2648"
 dependencies = [
  "anyhow",
- "cranelift-codegen",
- "cranelift-entity",
- "cranelift-frontend",
- "cranelift-native",
- "cranelift-wasm",
+ "cranelift-codegen 0.89.2",
+ "cranelift-entity 0.89.2",
+ "cranelift-frontend 0.89.2",
+ "cranelift-native 0.89.2",
+ "cranelift-wasm 0.89.2",
  "gimli 0.26.2",
  "log",
  "object 0.29.0",
  "target-lexicon",
  "thiserror",
- "wasmparser",
- "wasmtime-environ",
+ "wasmparser 0.92.0",
+ "wasmtime-environ 2.0.2",
+]
+
+[[package]]
+name = "wasmtime-cranelift"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "cfg-if 1.0.0",
+ "cranelift-codegen 0.103.0",
+ "cranelift-control",
+ "cranelift-entity 0.103.0",
+ "cranelift-frontend 0.103.0",
+ "cranelift-native 0.103.0",
+ "cranelift-wasm 0.103.0",
+ "gimli 0.28.0",
+ "log",
+ "object 0.32.1",
+ "target-lexicon",
+ "thiserror",
+ "wasmparser 0.118.1",
+ "wasmtime-cranelift-shared",
+ "wasmtime-environ 16.0.0",
+ "wasmtime-versioned-export-macros",
+]
+
+[[package]]
+name = "wasmtime-cranelift-shared"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen 0.103.0",
+ "cranelift-control",
+ "cranelift-native 0.103.0",
+ "gimli 0.28.0",
+ "object 0.32.1",
+ "target-lexicon",
+ "wasmtime-environ 16.0.0",
 ]
 
 [[package]]
@@ -11300,7 +11547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6aec5c1f81aab9bb35997113c171b6bb9093afc90e3757c55e0c08dc9ac612e4"
 dependencies = [
  "anyhow",
- "cranelift-entity",
+ "cranelift-entity 0.89.2",
  "gimli 0.26.2",
  "indexmap 1.9.3",
  "log",
@@ -11308,8 +11555,27 @@ dependencies = [
  "serde",
  "target-lexicon",
  "thiserror",
- "wasmparser",
- "wasmtime-types",
+ "wasmparser 0.92.0",
+ "wasmtime-types 2.0.2",
+]
+
+[[package]]
+name = "wasmtime-environ"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "cranelift-entity 0.103.0",
+ "gimli 0.28.0",
+ "indexmap 2.0.0",
+ "log",
+ "object 0.32.1",
+ "serde",
+ "serde_derive",
+ "target-lexicon",
+ "thiserror",
+ "wasmparser 0.118.1",
+ "wasmtime-types 16.0.0",
 ]
 
 [[package]]
@@ -11321,7 +11587,7 @@ dependencies = [
  "cc",
  "cfg-if 1.0.0",
  "rustix 0.35.16",
- "wasmtime-asm-macros",
+ "wasmtime-asm-macros 2.0.2",
  "windows-sys 0.36.1",
 ]
 
@@ -11345,12 +11611,33 @@ dependencies = [
  "serde",
  "target-lexicon",
  "thiserror",
- "wasmtime-environ",
+ "wasmtime-environ 2.0.2",
  "wasmtime-jit-debug",
- "wasmtime-runtime",
+ "wasmtime-runtime 2.0.2",
  "windows-sys 0.36.1",
 ]
 
+[[package]]
+name = "wasmtime-jit"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "cfg-if 1.0.0",
+ "gimli 0.28.0",
+ "log",
+ "object 0.32.1",
+ "rustix 0.38.21",
+ "serde",
+ "serde_derive",
+ "target-lexicon",
+ "wasmtime-environ 16.0.0",
+ "wasmtime-jit-icache-coherence",
+ "wasmtime-runtime 16.0.0",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "wasmtime-jit-debug"
 version = "2.0.2"
@@ -11362,6 +11649,16 @@ dependencies = [
  "rustix 0.35.16",
 ]
 
+[[package]]
+name = "wasmtime-jit-icache-coherence"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "wasmtime-runtime"
 version = "2.0.2"
@@ -11381,23 +11678,71 @@ dependencies = [
  "rand 0.8.5",
  "rustix 0.35.16",
  "thiserror",
- "wasmtime-asm-macros",
- "wasmtime-environ",
+ "wasmtime-asm-macros 2.0.2",
+ "wasmtime-environ 2.0.2",
  "wasmtime-fiber",
  "wasmtime-jit-debug",
  "windows-sys 0.36.1",
 ]
 
+[[package]]
+name = "wasmtime-runtime"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "anyhow",
+ "cc",
+ "cfg-if 1.0.0",
+ "indexmap 2.0.0",
+ "libc",
+ "log",
+ "mach",
+ "memfd",
+ "memoffset 0.9.0",
+ "paste",
+ "psm",
+ "rustix 0.38.21",
+ "sptr",
+ "wasm-encoder",
+ "wasmtime-asm-macros 16.0.0",
+ "wasmtime-environ 16.0.0",
+ "wasmtime-versioned-export-macros",
+ "wasmtime-wmemcheck",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "wasmtime-types"
 version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5dc3dd9521815984b35d6362f79e6b9c72475027cd1c71c44eb8df8fbf33a9fb"
 dependencies = [
- "cranelift-entity",
+ "cranelift-entity 0.89.2",
  "serde",
  "thiserror",
- "wasmparser",
+ "wasmparser 0.92.0",
+]
+
+[[package]]
+name = "wasmtime-types"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "cranelift-entity 0.103.0",
+ "serde",
+ "serde_derive",
+ "thiserror",
+ "wasmparser 0.118.1",
+]
+
+[[package]]
+name = "wasmtime-versioned-export-macros"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
 ]
 
 [[package]]
@@ -11409,10 +11754,15 @@ dependencies = [
  "anyhow",
  "wasi-cap-std-sync",
  "wasi-common",
- "wasmtime",
+ "wasmtime 2.0.2",
  "wiggle",
 ]
 
+[[package]]
+name = "wasmtime-wmemcheck"
+version = "16.0.0"
+source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7"
+
 [[package]]
 name = "wast"
 version = "35.0.2"
@@ -11567,7 +11917,7 @@ dependencies = [
  "bitflags 1.3.2",
  "thiserror",
  "tracing",
- "wasmtime",
+ "wasmtime 2.0.2",
  "wiggle-macro",
 ]

Cargo.toml 🔗 

@@ -83,7 +83,6 @@ members = [
     "crates/picker2",
     "crates/plugin",
     "crates/plugin_macros",
-    "crates/plugin_runtime",
     "crates/prettier",
     "crates/prettier2",
     "crates/project",
@@ -176,11 +175,12 @@ thiserror = { version = "1.0.29" }
 time = { version = "0.3", features = ["serde", "serde-well-known"] }
 toml = { version = "0.5" }
 tiktoken-rs = "0.5.7"
-tree-sitter = "0.20"
+tree-sitter = { version = "0.20", features = ["wasm"] }
 unindent = { version = "0.1.7" }
 pretty_assertions = "1.3.0"
 git2 = { version = "0.15", default-features = false}
 uuid = { version = "1.1.2", features = ["v4"] }
+wasmtime = "16"
 
 tree-sitter-bash = { git = "https://github.com/tree-sitter/tree-sitter-bash", rev = "7331995b19b8f8aba2d5e26deb51d2195c18bc94" }
 tree-sitter-c = "0.20.1"
@@ -212,8 +212,9 @@ tree-sitter-vue = {git = "https://github.com/zed-industries/tree-sitter-vue", re
 tree-sitter-uiua = {git = "https://github.com/shnarazk/tree-sitter-uiua", rev = "9260f11be5900beda4ee6d1a24ab8ddfaf5a19b2"}
 
 [patch.crates-io]
-tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "b5f461a69bf3df7298b1903574d506179e6390b0" }
+tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "31c40449749c4263a91a43593831b82229049a4c" }
 async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" }
+wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", rev = "v16.0.0" }
 
 # TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457
 cocoa = { git = "https://github.com/servo/core-foundation-rs", rev = "079665882507dd5e2ff77db3de5070c1f6c0fb85" }

crates/language/src/language.rs 🔗 

@@ -1169,7 +1169,7 @@ impl Language {
                     indents_config: None,
                     injection_config: None,
                     override_config: None,
-                    error_query: Query::new(ts_language, "(ERROR) @error").unwrap(),
+                    error_query: Query::new(&ts_language, "(ERROR) @error").unwrap(),
                     ts_language,
                     highlight_map: Default::default(),
                 })
@@ -1230,13 +1230,13 @@ impl Language {
 
     pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
+        grammar.highlights_query = Some(Query::new(&grammar.ts_language, source)?);
         Ok(self)
     }
 
     pub fn with_outline_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut item_capture_ix = None;
         let mut name_capture_ix = None;
         let mut context_capture_ix = None;
@@ -1264,7 +1264,7 @@ impl Language {
 
     pub fn with_embedding_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut item_capture_ix = None;
         let mut name_capture_ix = None;
         let mut context_capture_ix = None;
@@ -1295,7 +1295,7 @@ impl Language {
 
     pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut open_capture_ix = None;
         let mut close_capture_ix = None;
         get_capture_indices(
@@ -1317,7 +1317,7 @@ impl Language {
 
     pub fn with_indents_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut indent_capture_ix = None;
         let mut start_capture_ix = None;
         let mut end_capture_ix = None;
@@ -1345,7 +1345,7 @@ impl Language {
 
     pub fn with_injection_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut language_capture_ix = None;
         let mut content_capture_ix = None;
         get_capture_indices(
@@ -1384,7 +1384,7 @@ impl Language {
     }
 
     pub fn with_override_query(mut self, source: &str) -> anyhow::Result<Self> {
-        let query = Query::new(self.grammar_mut().ts_language, source)?;
+        let query = Query::new(&self.grammar_mut().ts_language, source)?;
 
         let mut override_configs_by_id = HashMap::default();
         for (ix, name) in query.capture_names().iter().copied().enumerate() {
@@ -1695,7 +1695,7 @@ impl Grammar {
         PARSER.with(|parser| {
             let mut parser = parser.borrow_mut();
             parser
-                .set_language(self.ts_language)
+                .set_language(&self.ts_language)
                 .expect("incompatible grammar");
             let mut chunks = text.chunks_in_range(0..text.len());
             parser

crates/language/src/syntax_map.rs 🔗 

@@ -1166,7 +1166,7 @@ fn parse_text(
             .set_included_ranges(&ranges)
             .expect("overlapping ranges");
         parser
-            .set_language(grammar.ts_language)
+            .set_language(&grammar.ts_language)
             .expect("incompatible grammar");
         parser
             .parse_with(

crates/language2/src/language2.rs 🔗 

@@ -44,7 +44,7 @@ use std::{
 };
 use syntax_map::SyntaxSnapshot;
 use theme::{SyntaxTheme, Theme};
-use tree_sitter::{self, Query};
+use tree_sitter::{self, wasmtime, Query, WasmStore};
 use unicase::UniCase;
 use util::{http::HttpClient, paths::PathExt};
 use util::{post_inc, ResultExt, TryFutureExt as _, UnwrapFuture};
@@ -84,10 +84,15 @@ impl LspBinaryStatusSender {
 }
 
 thread_local! {
-    static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
+    static PARSER: RefCell<Parser> = {
+        let mut parser = Parser::new();
+        parser.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap()).unwrap();
+        RefCell::new(parser)
+    };
 }
 
 lazy_static! {
+    pub static ref WASM_ENGINE: wasmtime::Engine = wasmtime::Engine::default();
     pub static ref NEXT_GRAMMAR_ID: AtomicUsize = Default::default();
     pub static ref PLAIN_TEXT: Arc<Language> = Arc::new(Language::new(
         LanguageConfig {
@@ -360,6 +365,7 @@ pub struct CodeLabel {
 #[derive(Clone, Deserialize)]
 pub struct LanguageConfig {
     pub name: Arc<str>,
+    pub grammar_name: Option<Arc<str>>,
     pub path_suffixes: Vec<String>,
     pub brackets: BracketPairConfig,
     #[serde(default, deserialize_with = "deserialize_regex")]
@@ -446,6 +452,7 @@ impl Default for LanguageConfig {
     fn default() -> Self {
         Self {
             name: "".into(),
+            grammar_name: None,
             path_suffixes: Default::default(),
             brackets: Default::default(),
             auto_indent_using_last_non_empty_line: auto_indent_using_last_non_empty_line_default(),
@@ -620,14 +627,25 @@ type AvailableLanguageId = usize;
 #[derive(Clone)]
 struct AvailableLanguage {
     id: AvailableLanguageId,
-    path: &'static str,
     config: LanguageConfig,
-    grammar: tree_sitter::Language,
+    grammar: AvailableGrammar,
     lsp_adapters: Vec<Arc<dyn LspAdapter>>,
-    get_queries: fn(&str) -> LanguageQueries,
     loaded: bool,
 }
 
+#[derive(Clone)]
+enum AvailableGrammar {
+    Native {
+        grammar: tree_sitter::Language,
+        asset_dir: &'static str,
+        get_queries: fn(&str) -> LanguageQueries,
+    },
+    Wasm {
+        grammar_name: Arc<str>,
+        path: Arc<Path>,
+    },
+}
+
 pub struct LanguageRegistry {
     state: RwLock<LanguageRegistryState>,
     language_server_download_dir: Option<Arc<Path>>,
@@ -699,7 +717,7 @@ impl LanguageRegistry {
 
     pub fn register(
         &self,
-        path: &'static str,
+        asset_dir: &'static str,
         config: LanguageConfig,
         grammar: tree_sitter::Language,
         lsp_adapters: Vec<Arc<dyn LspAdapter>>,
@@ -708,11 +726,24 @@ impl LanguageRegistry {
         let state = &mut *self.state.write();
         state.available_languages.push(AvailableLanguage {
             id: post_inc(&mut state.next_available_language_id),
-            path,
             config,
-            grammar,
+            grammar: AvailableGrammar::Native {
+                grammar,
+                get_queries,
+                asset_dir,
+            },
             lsp_adapters,
-            get_queries,
+            loaded: false,
+        });
+    }
+
+    pub fn register_wasm(&self, path: Arc<Path>, grammar_name: Arc<str>, config: LanguageConfig) {
+        let state = &mut *self.state.write();
+        state.available_languages.push(AvailableLanguage {
+            id: post_inc(&mut state.next_available_language_id),
+            config,
+            grammar: AvailableGrammar::Wasm { grammar_name, path },
+            lsp_adapters: Vec::new(),
             loaded: false,
         });
     }
@@ -837,13 +868,43 @@ impl LanguageRegistry {
                         executor
                             .spawn(async move {
                                 let id = language.id;
-                                let queries = (language.get_queries)(&language.path);
-                                let language =
-                                    Language::new(language.config, Some(language.grammar))
+                                let name = language.config.name.clone();
+                                let language = async {
+                                    let (grammar, queries) = match language.grammar {
+                                        AvailableGrammar::Native {
+                                            grammar,
+                                            asset_dir,
+                                            get_queries,
+                                        } => (grammar, (get_queries)(asset_dir)),
+                                        AvailableGrammar::Wasm { grammar_name, path } => {
+                                            let mut wasm_path = path.join(grammar_name.as_ref());
+                                            wasm_path.set_extension("wasm");
+                                            let wasm_bytes = std::fs::read(&wasm_path)?;
+                                            let grammar = PARSER.with(|parser| {
+                                                let mut parser = parser.borrow_mut();
+                                                let mut store = parser.take_wasm_store().unwrap();
+                                                let grammar =
+                                                    store.load_language(&grammar_name, &wasm_bytes);
+                                                parser.set_wasm_store(store).unwrap();
+                                                grammar
+                                            })?;
+                                            let mut queries = LanguageQueries::default();
+                                            if let Ok(contents) = std::fs::read_to_string(
+                                                &path.join("highlights.scm"),
+                                            ) {
+                                                queries.highlights = Some(contents.into());
+                                            }
+                                            (grammar, queries)
+                                        }
+                                    };
+                                    Language::new(language.config, Some(grammar))
                                         .with_lsp_adapters(language.lsp_adapters)
-                                        .await;
-                                let name = language.name();
-                                match language.with_queries(queries) {
+                                        .await
+                                        .with_queries(queries)
+                                }
+                                .await;
+
+                                match language {
                                     Ok(language) => {
                                         let language = Arc::new(language);
                                         let mut state = this.state.write();
@@ -1175,7 +1236,7 @@ impl Language {
                     indents_config: None,
                     injection_config: None,
                     override_config: None,
-                    error_query: Query::new(ts_language, "(ERROR) @error").unwrap(),
+                    error_query: Query::new(&ts_language, "(ERROR) @error").unwrap(),
                     ts_language,
                     highlight_map: Default::default(),
                 })
@@ -1236,13 +1297,13 @@ impl Language {
 
     pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
+        grammar.highlights_query = Some(Query::new(&grammar.ts_language, source)?);
         Ok(self)
     }
 
     pub fn with_outline_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut item_capture_ix = None;
         let mut name_capture_ix = None;
         let mut context_capture_ix = None;
@@ -1270,7 +1331,7 @@ impl Language {
 
     pub fn with_embedding_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut item_capture_ix = None;
         let mut name_capture_ix = None;
         let mut context_capture_ix = None;
@@ -1301,7 +1362,7 @@ impl Language {
 
     pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut open_capture_ix = None;
         let mut close_capture_ix = None;
         get_capture_indices(
@@ -1323,7 +1384,7 @@ impl Language {
 
     pub fn with_indents_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut indent_capture_ix = None;
         let mut start_capture_ix = None;
         let mut end_capture_ix = None;
@@ -1351,7 +1412,7 @@ impl Language {
 
     pub fn with_injection_query(mut self, source: &str) -> Result<Self> {
         let grammar = self.grammar_mut();
-        let query = Query::new(grammar.ts_language, source)?;
+        let query = Query::new(&grammar.ts_language, source)?;
         let mut language_capture_ix = None;
         let mut content_capture_ix = None;
         get_capture_indices(
@@ -1390,7 +1451,7 @@ impl Language {
     }
 
     pub fn with_override_query(mut self, source: &str) -> anyhow::Result<Self> {
-        let query = Query::new(self.grammar_mut().ts_language, source)?;
+        let query = Query::new(&self.grammar_mut().ts_language, source)?;
 
         let mut override_configs_by_id = HashMap::default();
         for (ix, name) in query.capture_names().iter().enumerate() {
@@ -1701,7 +1762,7 @@ impl Grammar {
         PARSER.with(|parser| {
             let mut parser = parser.borrow_mut();
             parser
-                .set_language(self.ts_language)
+                .set_language(&self.ts_language)
                 .expect("incompatible grammar");
             let mut chunks = text.chunks_in_range(0..text.len());
             parser

crates/language2/src/syntax_map.rs 🔗 

@@ -7,7 +7,6 @@ use futures::FutureExt;
 use parking_lot::Mutex;
 use std::{
     borrow::Cow,
-    cell::RefCell,
     cmp::{self, Ordering, Reverse},
     collections::BinaryHeap,
     fmt, iter,
@@ -16,13 +15,9 @@ use std::{
 };
 use sum_tree::{Bias, SeekTarget, SumTree};
 use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
-use tree_sitter::{
-    Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
-};
+use tree_sitter::{Node, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree};
 
-thread_local! {
-    static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
-}
+use super::PARSER;
 
 static QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Mutex::new(vec![]);
 
@@ -1166,7 +1161,7 @@ fn parse_text(
             .set_included_ranges(&ranges)
             .expect("overlapping ranges");
         parser
-            .set_language(grammar.ts_language)
+            .set_language(&grammar.ts_language)
             .expect("incompatible grammar");
         parser
             .parse_with(

crates/semantic_index/src/parsing.rs 🔗 

@@ -169,7 +169,7 @@ impl CodeContextRetriever {
             .embedding_config
             .as_ref()
             .ok_or_else(|| anyhow!("no embedding queries"))?;
-        self.parser.set_language(grammar.ts_language).unwrap();
+        self.parser.set_language(&grammar.ts_language).unwrap();
 
         let tree = self
             .parser

crates/semantic_index2/src/parsing.rs 🔗 

@@ -169,7 +169,7 @@ impl CodeContextRetriever {
             .embedding_config
             .as_ref()
             .ok_or_else(|| anyhow!("no embedding queries"))?;
-        self.parser.set_language(grammar.ts_language).unwrap();
+        self.parser.set_language(&grammar.ts_language).unwrap();
 
         let tree = self
             .parser

crates/settings/src/settings_store.rs 🔗 

@@ -679,14 +679,14 @@ fn replace_value_in_json_text(
 
     lazy_static! {
         static ref PAIR_QUERY: tree_sitter::Query = tree_sitter::Query::new(
-            tree_sitter_json::language(),
+            &tree_sitter_json::language(),
             "(pair key: (string) @key value: (_) @value)",
         )
         .unwrap();
     }
 
     let mut parser = tree_sitter::Parser::new();
-    parser.set_language(tree_sitter_json::language()).unwrap();
+    parser.set_language(&tree_sitter_json::language()).unwrap();
     let syntax_tree = parser.parse(text, None).unwrap();
 
     let mut cursor = tree_sitter::QueryCursor::new();

crates/settings2/src/settings_store.rs 🔗 

@@ -715,14 +715,14 @@ fn replace_value_in_json_text(
 
     lazy_static! {
         static ref PAIR_QUERY: tree_sitter::Query = tree_sitter::Query::new(
-            tree_sitter_json::language(),
+            &tree_sitter_json::language(),
             "(pair key: (string) @key value: (_) @value)",
         )
         .unwrap();
     }
 
     let mut parser = tree_sitter::Parser::new();
-    parser.set_language(tree_sitter_json::language()).unwrap();
+    parser.set_language(&tree_sitter_json::language()).unwrap();
     let syntax_tree = parser.parse(text, None).unwrap();
 
     let mut cursor = tree_sitter::QueryCursor::new();

crates/util/src/paths.rs 🔗 

@@ -10,6 +10,7 @@ lazy_static::lazy_static! {
     pub static ref EMBEDDINGS_DIR: PathBuf = HOME.join(".config/zed/embeddings");
     pub static ref LOGS_DIR: PathBuf = HOME.join("Library/Logs/Zed");
     pub static ref SUPPORT_DIR: PathBuf = HOME.join("Library/Application Support/Zed");
+    pub static ref PLUGINS_DIR: PathBuf = HOME.join("Library/Application Support/Zed/plugins");
     pub static ref LANGUAGES_DIR: PathBuf = HOME.join("Library/Application Support/Zed/languages");
     pub static ref COPILOT_DIR: PathBuf = HOME.join("Library/Application Support/Zed/copilot");
     pub static ref DEFAULT_PRETTIER_DIR: PathBuf = HOME.join("Library/Application Support/Zed/prettier");

crates/zed2/src/languages.rs 🔗 

@@ -5,7 +5,7 @@ use node_runtime::NodeRuntime;
 use rust_embed::RustEmbed;
 use settings::Settings;
 use std::{borrow::Cow, str, sync::Arc};
-use util::asset_str;
+use util::{asset_str, paths::PLUGINS_DIR};
 
 use self::elixir::ElixirSettings;
 
@@ -228,6 +228,21 @@ pub fn init(
         tree_sitter_uiua::language(),
         vec![Arc::new(uiua::UiuaLanguageServer {})],
     );
+
+    if let Ok(children) = std::fs::read_dir(&*PLUGINS_DIR) {
+        for child in children {
+            if let Ok(child) = child {
+                let path = child.path();
+                let config_path = path.join("config.toml");
+                if let Ok(config) = std::fs::read(&config_path) {
+                    let config: LanguageConfig = toml::from_slice(&config).unwrap();
+                    if let Some(grammar_name) = config.grammar_name.clone() {
+                        languages.register_wasm(path.into(), grammar_name, config);
+                    }
+                }
+            }
+        }
+    }
 }
 
 #[cfg(any(test, feature = "test-support"))]