Expanded Semantic language support (#2818)

Kyle Caverly created

Expand Language Support within Semantic Search

Release Notes (Preview-only)

- Added semantic search support for lua, ruby, php, svelte, erb, heex,
html and markdown.
- Full details are included here:
https://linear.app/zed-industries/issue/Z-2611/allow-semantic-search-for-all-supported-languages

Change summary

Cargo.lock                                        | 232 +++----
crates/semantic_index/Cargo.toml                  |  15 
crates/semantic_index/src/parsing.rs              |  28 
crates/semantic_index/src/semantic_index.rs       |   1 
crates/semantic_index/src/semantic_index_tests.rs | 538 +++++++++++++++++
crates/zed/src/languages/lua/config.toml          |   1 
crates/zed/src/languages/lua/embedding.scm        |  10 
crates/zed/src/languages/php/config.toml          |   1 
crates/zed/src/languages/php/embedding.scm        |  36 +
crates/zed/src/languages/php/outline.scm          |   7 
crates/zed/src/languages/ruby/config.toml         |   1 
crates/zed/src/languages/ruby/embedding.scm       |  22 
12 files changed, 751 insertions(+), 141 deletions(-)

Detailed changes

Cargo.lock 🔗

@@ -141,7 +141,7 @@ source = "git+https://github.com/alacritty/alacritty?rev=7b9f32300ee0a249c087230
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -187,9 +187,9 @@ checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
 
 [[package]]
 name = "alsa"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8512c9117059663fb5606788fbca3619e2a91dac0e3fe516242eab1fa6be5e44"
+checksum = "e2562ad8dcf0f789f65c6fdaad8a8a9708ed6b488e649da28c01656ad66b8b47"
 dependencies = [
  "alsa-sys",
  "bitflags 1.3.2",
@@ -215,9 +215,9 @@ checksum = "ec8ad6edb4840b78c5c3d88de606b22252d552b55f3a4699fbb10fc070ec3049"
 
 [[package]]
 name = "android-activity"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40bc1575e653f158cbdc6ebcd917b9564e66321c5325c232c3591269c257be69"
+checksum = "64529721f27c2314ced0890ce45e469574a73e5e6fdd6e9da1860eb29285f5e0"
 dependencies = [
  "android-properties",
  "bitflags 1.3.2",
@@ -507,7 +507,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -555,7 +555,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -598,7 +598,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -856,7 +856,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.27",
+ "syn 2.0.28",
  "which",
 ]
 
@@ -1040,7 +1040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
 dependencies = [
  "memchr",
- "regex-automata 0.3.3",
+ "regex-automata 0.3.4",
  "serde",
 ]
 
@@ -1358,7 +1358,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1425,7 +1425,7 @@ dependencies = [
  "sum_tree",
  "tempfile",
  "thiserror",
- "time 0.3.23",
+ "time 0.3.24",
  "tiny_http",
  "url",
  "util",
@@ -1527,7 +1527,7 @@ dependencies = [
  "sha-1 0.9.8",
  "sqlx",
  "theme",
- "time 0.3.23",
+ "time 0.3.24",
  "tokio",
  "tokio-tungstenite",
  "toml 0.5.11",
@@ -2042,9 +2042,9 @@ dependencies = [
 
 [[package]]
 name = "curl-sys"
-version = "0.4.64+curl-8.2.0"
+version = "0.4.65+curl-8.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f96069f0b1cb1241c838740659a771ef143363f52772a9ce1bd9c04c75eee0dc"
+checksum = "961ba061c9ef2fe34bbd12b807152d96f0badd2bebe7b90ce6c8c8b7572a0986"
 dependencies = [
  "cc",
  "libc",
@@ -2124,6 +2124,15 @@ dependencies = [
  "byteorder",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "dhat"
 version = "0.3.2"
@@ -2341,7 +2350,7 @@ dependencies = [
  "tree-sitter",
  "tree-sitter-html",
  "tree-sitter-rust",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-typescript",
  "unindent",
  "util",
  "workspace",
@@ -2425,9 +2434,9 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
 dependencies = [
  "errno-dragonfly",
  "libc",
@@ -2731,7 +2740,7 @@ dependencies = [
  "smol",
  "sum_tree",
  "tempfile",
- "time 0.3.23",
+ "time 0.3.24",
  "util",
 ]
 
@@ -2881,7 +2890,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -3033,9 +3042,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "globset"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df"
+checksum = "aca8bbd8e0707c1887a8bbb7e6b40e228f251ff5d62c8220a4a7a53c73aff006"
 dependencies = [
  "aho-corasick 1.0.2",
  "bstr",
@@ -3121,7 +3130,7 @@ dependencies = [
  "smol",
  "sqlez",
  "sum_tree",
- "time 0.3.23",
+ "time 0.3.24",
  "tiny-skia",
  "usvg",
  "util",
@@ -3851,7 +3860,7 @@ dependencies = [
  "text",
  "theme",
  "tree-sitter",
- "tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=a2861e88a730287a60c11ea9299c033c7d076e30)",
+ "tree-sitter-elixir",
  "tree-sitter-embedded-template",
  "tree-sitter-heex",
  "tree-sitter-html",
@@ -3860,7 +3869,7 @@ dependencies = [
  "tree-sitter-python",
  "tree-sitter-ruby",
  "tree-sitter-rust",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-typescript",
  "unicase",
  "unindent",
  "util",
@@ -4034,9 +4043,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.3"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
 
 [[package]]
 name = "lipsum"
@@ -4744,7 +4753,7 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -4895,7 +4904,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -5134,9 +5143,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "pest"
-version = "2.7.1"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d2d1d55045829d65aad9d389139882ad623b33b904e7c9f1b10c5b8927298e5"
+checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a"
 dependencies = [
  "thiserror",
  "ucd-trie",
@@ -5192,7 +5201,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -5230,7 +5239,7 @@ dependencies = [
  "line-wrap",
  "quick-xml",
  "serde",
- "time 0.3.23",
+ "time 0.3.24",
 ]
 
 [[package]]
@@ -5345,7 +5354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
 dependencies = [
  "proc-macro2",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -5905,7 +5914,7 @@ checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
 dependencies = [
  "aho-corasick 1.0.2",
  "memchr",
- "regex-automata 0.3.3",
+ "regex-automata 0.3.4",
  "regex-syntax 0.7.4",
 ]
 
@@ -5920,9 +5929,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
+checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294"
 dependencies = [
  "aho-corasick 1.0.2",
  "memchr",
@@ -6217,7 +6226,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.27",
+ "syn 2.0.28",
  "walkdir",
 ]
 
@@ -6234,13 +6243,12 @@ dependencies = [
 
 [[package]]
 name = "rust_decimal"
-version = "1.30.0"
+version = "1.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0446843641c69436765a35a5a77088e28c2e6a12da93e84aa3ab1cd4aa5a042"
+checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea"
 dependencies = [
  "arrayvec 0.7.4",
  "borsh",
- "bytecheck",
  "byteorder",
  "bytes 1.4.0",
  "num-traits",
@@ -6294,7 +6302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06"
 dependencies = [
  "bitflags 1.3.2",
- "errno 0.3.1",
+ "errno 0.3.2",
  "io-lifetimes 1.0.11",
  "libc",
  "linux-raw-sys 0.3.8",
@@ -6308,9 +6316,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
 dependencies = [
  "bitflags 2.3.3",
- "errno 0.3.1",
+ "errno 0.3.2",
  "libc",
- "linux-raw-sys 0.4.3",
+ "linux-raw-sys 0.4.5",
  "windows-sys",
 ]
 
@@ -6509,7 +6517,7 @@ dependencies = [
  "serde_json",
  "sqlx",
  "thiserror",
- "time 0.3.23",
+ "time 0.3.24",
  "tracing",
  "url",
  "uuid 1.4.1",
@@ -6537,7 +6545,7 @@ dependencies = [
  "rust_decimal",
  "sea-query-derive",
  "serde_json",
- "time 0.3.23",
+ "time 0.3.24",
  "uuid 1.4.1",
 ]
 
@@ -6552,7 +6560,7 @@ dependencies = [
  "sea-query",
  "serde_json",
  "sqlx",
- "time 0.3.23",
+ "time 0.3.24",
  "uuid 1.4.1",
 ]
 
@@ -6685,12 +6693,15 @@ dependencies = [
  "theme",
  "tiktoken-rs 0.5.0",
  "tree-sitter",
- "tree-sitter-cpp 0.20.2",
- "tree-sitter-elixir 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tree-sitter-json 0.19.0",
+ "tree-sitter-cpp",
+ "tree-sitter-elixir",
+ "tree-sitter-json 0.20.0",
+ "tree-sitter-lua",
+ "tree-sitter-php",
+ "tree-sitter-ruby",
  "tree-sitter-rust",
- "tree-sitter-toml 0.20.0",
- "tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tree-sitter-toml",
+ "tree-sitter-typescript",
  "unindent",
  "util",
  "workspace",
@@ -6722,22 +6733,22 @@ checksum = "5a9f47faea3cad316faa914d013d24f471cd90bfca1a0c70f05a3f42c6441e99"
 
 [[package]]
 name = "serde"
-version = "1.0.175"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
+checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.175"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
+checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -6762,9 +6773,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
 dependencies = [
  "indexmap 2.0.0",
  "itoa 1.0.9",
@@ -6786,13 +6797,13 @@ dependencies = [
 
 [[package]]
 name = "serde_repr"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e168eaaf71e8f9bd6037feb05190485708e019f4fd87d161b3c0a0d37daf85e5"
+checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -7238,7 +7249,7 @@ dependencies = [
  "sqlx-rt",
  "stringprep",
  "thiserror",
- "time 0.3.23",
+ "time 0.3.24",
  "tokio-stream",
  "url",
  "uuid 1.4.1",
@@ -7487,9 +7498,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.27"
+version = "2.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
+checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7557,9 +7568,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.10"
+version = "0.12.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e"
+checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
 
 [[package]]
 name = "tempdir"
@@ -7742,7 +7753,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -7815,10 +7826,11 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.23"
+version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446"
+checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b"
 dependencies = [
+ "deranged",
  "itoa 1.0.9",
  "serde",
  "time-core",
@@ -7833,9 +7845,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4"
+checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd"
 dependencies = [
  "time-core",
 ]
@@ -7931,7 +7943,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -8153,7 +8165,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -8255,16 +8267,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-cpp"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c88fd925d0333e63ac64e521f5bd79c53019e569ffbbccfeef346a326f459e9"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-css"
 version = "0.19.0"
@@ -8274,16 +8276,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-elixir"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a9916f3e1c80b3c8aab8582604e97e8720cb9b893489b347cf999f80f9d469e"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-elixir"
 version = "0.1.0"
@@ -8471,26 +8463,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-toml"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca517f578a98b23d20780247cc2688407fa81effad5b627a5a364ec3339b53e8"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
-[[package]]
-name = "tree-sitter-typescript"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "079c695c32d39ad089101c66393aeaca30e967fba3486a91f573d2f0e12d290a"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-typescript"
 version = "0.20.2"
@@ -8993,7 +8965,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
  "wasm-bindgen-shared",
 ]
 
@@ -9027,7 +8999,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -9040,9 +9012,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-encoder"
-version = "0.31.0"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06a3d1b4a575ffb873679402b2aedb3117555eb65c27b1b86c8a91e574bc2a2a"
+checksum = "41763f20eafed1399fff1afb466496d3a959f58241436cfdc17e3f5ca954de16"
 dependencies = [
  "leb128",
 ]
@@ -9264,9 +9236,9 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "62.0.0"
+version = "62.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7f7ee878019d69436895f019b65f62c33da63595d8e857cbdc87c13ecb29a32"
+checksum = "b8ae06f09dbe377b889fbd620ff8fa21e1d49d1d9d364983c0cdbf9870cb9f1f"
 dependencies = [
  "leb128",
  "memchr",
@@ -9276,11 +9248,11 @@ dependencies = [
 
 [[package]]
 name = "wat"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "295572bf24aa5b685a971a83ad3e8b6e684aaad8a9be24bc7bf59bed84cc1c08"
+checksum = "842e15861d203fb4a96d314b0751cdeaf0f6f8b35e8d81d2953af2af5e44e637"
 dependencies = [
- "wast 62.0.0",
+ "wast 62.0.1",
 ]
 
 [[package]]
@@ -9657,9 +9629,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25b5872fa2e10bd067ae946f927e726d7d603eaeb6e02fa6a350e0722d2b8c11"
+checksum = "8bd122eb777186e60c3fdf765a58ac76e41c582f1f535fbf3314434c6b58f3f7"
 dependencies = [
  "memchr",
 ]
@@ -9930,9 +9902,9 @@ dependencies = [
  "tree-sitter",
  "tree-sitter-bash",
  "tree-sitter-c",
- "tree-sitter-cpp 0.20.0",
+ "tree-sitter-cpp",
  "tree-sitter-css",
- "tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=a2861e88a730287a60c11ea9299c033c7d076e30)",
+ "tree-sitter-elixir",
  "tree-sitter-elm",
  "tree-sitter-embedded-template",
  "tree-sitter-glsl",
@@ -9950,8 +9922,8 @@ dependencies = [
  "tree-sitter-rust",
  "tree-sitter-scheme",
  "tree-sitter-svelte",
- "tree-sitter-toml 0.5.1",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-toml",
+ "tree-sitter-typescript",
  "tree-sitter-yaml",
  "unindent",
  "url",
@@ -9988,7 +9960,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]

crates/semantic_index/Cargo.toml 🔗

@@ -54,9 +54,12 @@ tempdir.workspace = true
 ctor.workspace = true
 env_logger.workspace = true
 
-tree-sitter-typescript = "*"
-tree-sitter-json = "*"
-tree-sitter-rust = "*"
-tree-sitter-toml = "*"
-tree-sitter-cpp = "*"
-tree-sitter-elixir = "*"
+tree-sitter-typescript.workspace = true
+tree-sitter-json.workspace = true
+tree-sitter-rust.workspace = true
+tree-sitter-toml.workspace = true
+tree-sitter-cpp.workspace = true
+tree-sitter-elixir.workspace = true
+tree-sitter-lua.workspace = true
+tree-sitter-ruby.workspace = true
+tree-sitter-php.workspace = true

crates/semantic_index/src/parsing.rs 🔗

@@ -21,7 +21,9 @@ const CODE_CONTEXT_TEMPLATE: &str =
     "The below code snippet is from file '<path>'\n\n```<language>\n<item>\n```";
 const ENTIRE_FILE_TEMPLATE: &str =
     "The below snippet is from file '<path>'\n\n```<language>\n<item>\n```";
-pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] = &["TOML", "YAML", "CSS"];
+const MARKDOWN_CONTEXT_TEMPLATE: &str = "The below file contents is from file '<path>'\n\n<item>";
+pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] =
+    &["TOML", "YAML", "CSS", "HEEX", "ERB", "SVELTE", "HTML"];
 
 pub struct CodeContextRetriever {
     pub parser: Parser,
@@ -59,7 +61,7 @@ impl CodeContextRetriever {
         let document_span = ENTIRE_FILE_TEMPLATE
             .replace("<path>", relative_path.to_string_lossy().as_ref())
             .replace("<language>", language_name.as_ref())
-            .replace("item", &content);
+            .replace("<item>", &content);
 
         Ok(vec![Document {
             range: 0..content.len(),
@@ -69,6 +71,19 @@ impl CodeContextRetriever {
         }])
     }
 
+    fn parse_markdown_file(&self, relative_path: &Path, content: &str) -> Result<Vec<Document>> {
+        let document_span = MARKDOWN_CONTEXT_TEMPLATE
+            .replace("<path>", relative_path.to_string_lossy().as_ref())
+            .replace("<item>", &content);
+
+        Ok(vec![Document {
+            range: 0..content.len(),
+            content: document_span,
+            embedding: Vec::new(),
+            name: "Markdown".to_string(),
+        }])
+    }
+
     fn get_matches_in_file(
         &mut self,
         content: &str,
@@ -135,6 +150,8 @@ impl CodeContextRetriever {
 
         if PARSEABLE_ENTIRE_FILE_TYPES.contains(&language_name.as_ref()) {
             return self.parse_entire_file(relative_path, language_name, &content);
+        } else if &language_name.to_string() == &"Markdown".to_string() {
+            return self.parse_markdown_file(relative_path, &content);
         }
 
         let mut documents = self.parse_file(content, language)?;
@@ -200,7 +217,12 @@ impl CodeContextRetriever {
 
             let mut document_content = String::new();
             for context_range in &context_match.context_ranges {
-                document_content.push_str(&content[context_range.clone()]);
+                add_content_from_range(
+                    &mut document_content,
+                    content,
+                    context_range.clone(),
+                    context_match.start_col,
+                );
                 document_content.push_str("\n");
             }
 

crates/semantic_index/src/semantic_index.rs 🔗

@@ -612,6 +612,7 @@ impl SemanticIndex {
                                 .await
                             {
                                 if !PARSEABLE_ENTIRE_FILE_TYPES.contains(&language.name().as_ref())
+                                    && &language.name().as_ref() != &"Markdown"
                                     && language
                                         .grammar()
                                         .and_then(|grammar| grammar.embedding_config.as_ref())

crates/semantic_index/src/semantic_index_tests.rs 🔗

@@ -485,6 +485,79 @@ async fn test_code_context_retrieval_javascript() {
     )
 }
 
+#[gpui::test]
+async fn test_code_context_retrieval_lua() {
+    let language = lua_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        -- Creates a new class
+        -- @param baseclass The Baseclass of this class, or nil.
+        -- @return A new class reference.
+        function classes.class(baseclass)
+            -- Create the class definition and metatable.
+            local classdef = {}
+            -- Find the super class, either Object or user-defined.
+            baseclass = baseclass or classes.Object
+            -- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
+            setmetatable(classdef, { __index = baseclass })
+            -- All class instances have a reference to the class object.
+            classdef.class = classdef
+            --- Recursivly allocates the inheritance tree of the instance.
+            -- @param mastertable The 'root' of the inheritance tree.
+            -- @return Returns the instance with the allocated inheritance tree.
+            function classdef.alloc(mastertable)
+                -- All class instances have a reference to a superclass object.
+                local instance = { super = baseclass.alloc(mastertable) }
+                -- Any functions this instance does not know of will 'look up' to the superclass definition.
+                setmetatable(instance, { __index = classdef, __newindex = mastertable })
+                return instance
+            end
+        end
+        "#.unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (r#"
+                -- Creates a new class
+                -- @param baseclass The Baseclass of this class, or nil.
+                -- @return A new class reference.
+                function classes.class(baseclass)
+                    -- Create the class definition and metatable.
+                    local classdef = {}
+                    -- Find the super class, either Object or user-defined.
+                    baseclass = baseclass or classes.Object
+                    -- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
+                    setmetatable(classdef, { __index = baseclass })
+                    -- All class instances have a reference to the class object.
+                    classdef.class = classdef
+                    --- Recursivly allocates the inheritance tree of the instance.
+                    -- @param mastertable The 'root' of the inheritance tree.
+                    -- @return Returns the instance with the allocated inheritance tree.
+                    function classdef.alloc(mastertable)
+                        --[ ... ]--
+                        --[ ... ]--
+                    end
+                end"#.unindent(),
+            114),
+            (r#"
+            --- Recursivly allocates the inheritance tree of the instance.
+            -- @param mastertable The 'root' of the inheritance tree.
+            -- @return Returns the instance with the allocated inheritance tree.
+            function classdef.alloc(mastertable)
+                -- All class instances have a reference to a superclass object.
+                local instance = { super = baseclass.alloc(mastertable) }
+                -- Any functions this instance does not know of will 'look up' to the superclass definition.
+                setmetatable(instance, { __index = classdef, __newindex = mastertable })
+                return instance
+            end"#.unindent(), 809),
+        ]
+    );
+}
+
 #[gpui::test]
 async fn test_code_context_retrieval_elixir() {
     let language = elixir_lang();
@@ -753,6 +826,346 @@ async fn test_code_context_retrieval_cpp() {
     );
 }
 
+#[gpui::test]
+async fn test_code_context_retrieval_ruby() {
+    let language = ruby_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        # This concern is inspired by "sudo mode" on GitHub. It
+        # is a way to re-authenticate a user before allowing them
+        # to see or perform an action.
+        #
+        # Add `before_action :require_challenge!` to actions you
+        # want to protect.
+        #
+        # The user will be shown a page to enter the challenge (which
+        # is either the password, or just the username when no
+        # password exists). Upon passing, there is a grace period
+        # during which no challenge will be asked from the user.
+        #
+        # Accessing challenge-protected resources during the grace
+        # period will refresh the grace period.
+        module ChallengableConcern
+            extend ActiveSupport::Concern
+
+            CHALLENGE_TIMEOUT = 1.hour.freeze
+
+            def require_challenge!
+                return if skip_challenge?
+
+                if challenge_passed_recently?
+                    session[:challenge_passed_at] = Time.now.utc
+                    return
+                end
+
+                @challenge = Form::Challenge.new(return_to: request.url)
+
+                if params.key?(:form_challenge)
+                    if challenge_passed?
+                        session[:challenge_passed_at] = Time.now.utc
+                    else
+                        flash.now[:alert] = I18n.t('challenge.invalid_password')
+                        render_challenge
+                    end
+                else
+                    render_challenge
+                end
+            end
+
+            def challenge_passed?
+                current_user.valid_password?(challenge_params[:current_password])
+            end
+        end
+
+        class Animal
+            include Comparable
+
+            attr_reader :legs
+
+            def initialize(name, legs)
+                @name, @legs = name, legs
+            end
+
+            def <=>(other)
+                legs <=> other.legs
+            end
+        end
+
+        # Singleton method for car object
+        def car.wheels
+            puts "There are four wheels"
+        end"#
+        .unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (
+                r#"
+        # This concern is inspired by "sudo mode" on GitHub. It
+        # is a way to re-authenticate a user before allowing them
+        # to see or perform an action.
+        #
+        # Add `before_action :require_challenge!` to actions you
+        # want to protect.
+        #
+        # The user will be shown a page to enter the challenge (which
+        # is either the password, or just the username when no
+        # password exists). Upon passing, there is a grace period
+        # during which no challenge will be asked from the user.
+        #
+        # Accessing challenge-protected resources during the grace
+        # period will refresh the grace period.
+        module ChallengableConcern
+            extend ActiveSupport::Concern
+
+            CHALLENGE_TIMEOUT = 1.hour.freeze
+
+            def require_challenge!
+                # ...
+            end
+
+            def challenge_passed?
+                # ...
+            end
+        end"#
+                    .unindent(),
+                558,
+            ),
+            (
+                r#"
+            def require_challenge!
+                return if skip_challenge?
+
+                if challenge_passed_recently?
+                    session[:challenge_passed_at] = Time.now.utc
+                    return
+                end
+
+                @challenge = Form::Challenge.new(return_to: request.url)
+
+                if params.key?(:form_challenge)
+                    if challenge_passed?
+                        session[:challenge_passed_at] = Time.now.utc
+                    else
+                        flash.now[:alert] = I18n.t('challenge.invalid_password')
+                        render_challenge
+                    end
+                else
+                    render_challenge
+                end
+            end"#
+                    .unindent(),
+                663,
+            ),
+            (
+                r#"
+                def challenge_passed?
+                    current_user.valid_password?(challenge_params[:current_password])
+                end"#
+                    .unindent(),
+                1254,
+            ),
+            (
+                r#"
+                class Animal
+                    include Comparable
+
+                    attr_reader :legs
+
+                    def initialize(name, legs)
+                        # ...
+                    end
+
+                    def <=>(other)
+                        # ...
+                    end
+                end"#
+                    .unindent(),
+                1363,
+            ),
+            (
+                r#"
+                def initialize(name, legs)
+                    @name, @legs = name, legs
+                end"#
+                    .unindent(),
+                1427,
+            ),
+            (
+                r#"
+                def <=>(other)
+                    legs <=> other.legs
+                end"#
+                    .unindent(),
+                1501,
+            ),
+            (
+                r#"
+                # Singleton method for car object
+                def car.wheels
+                    puts "There are four wheels"
+                end"#
+                    .unindent(),
+                1591,
+            ),
+        ],
+    );
+}
+
+#[gpui::test]
+async fn test_code_context_retrieval_php() {
+    let language = php_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        <?php
+
+        namespace LevelUp\Experience\Concerns;
+
+        /*
+        This is a multiple-lines comment block
+        that spans over multiple
+        lines
+        */
+        function functionName() {
+            echo "Hello world!";
+        }
+
+        trait HasAchievements
+        {
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {
+                if ($progress > 100) {
+                    throw new Exception(message: 'Progress cannot be greater than 100');
+                }
+
+                if ($this->achievements()->find($achievement->id)) {
+                    throw new Exception(message: 'User already has this Achievement');
+                }
+
+                $this->achievements()->attach($achievement, [
+                    'progress' => $progress ?? null,
+                ]);
+
+                $this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
+            }
+
+            public function achievements(): BelongsToMany
+            {
+                return $this->belongsToMany(related: Achievement::class)
+                ->withPivot(columns: 'progress')
+                ->where('is_secret', false)
+                ->using(AchievementUser::class);
+            }
+        }
+
+        interface Multiplier
+        {
+            public function qualifies(array $data): bool;
+
+            public function setMultiplier(): int;
+        }
+
+        enum AuditType: string
+        {
+            case Add = 'add';
+            case Remove = 'remove';
+            case Reset = 'reset';
+            case LevelUp = 'level_up';
+        }
+
+        ?>"#
+    .unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (
+                r#"
+        /*
+        This is a multiple-lines comment block
+        that spans over multiple
+        lines
+        */
+        function functionName() {
+            echo "Hello world!";
+        }"#
+                .unindent(),
+                123,
+            ),
+            (
+                r#"
+        trait HasAchievements
+        {
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {/* ... */}
+
+            public function achievements(): BelongsToMany
+            {/* ... */}
+        }"#
+                .unindent(),
+                177,
+            ),
+            (r#"
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {
+                if ($progress > 100) {
+                    throw new Exception(message: 'Progress cannot be greater than 100');
+                }
+
+                if ($this->achievements()->find($achievement->id)) {
+                    throw new Exception(message: 'User already has this Achievement');
+                }
+
+                $this->achievements()->attach($achievement, [
+                    'progress' => $progress ?? null,
+                ]);
+
+                $this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
+            }"#.unindent(), 245),
+            (r#"
+                public function achievements(): BelongsToMany
+                {
+                    return $this->belongsToMany(related: Achievement::class)
+                    ->withPivot(columns: 'progress')
+                    ->where('is_secret', false)
+                    ->using(AchievementUser::class);
+                }"#.unindent(), 902),
+            (r#"
+                interface Multiplier
+                {
+                    public function qualifies(array $data): bool;
+
+                    public function setMultiplier(): int;
+                }"#.unindent(),
+                1146),
+            (r#"
+                enum AuditType: string
+                {
+                    case Add = 'add';
+                    case Remove = 'remove';
+                    case Reset = 'reset';
+                    case LevelUp = 'level_up';
+                }"#.unindent(), 1265)
+        ],
+    );
+}
+
 #[gpui::test]
 fn test_dot_product(mut rng: StdRng) {
     assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.);
@@ -1083,6 +1496,131 @@ fn cpp_lang() -> Arc<Language> {
     )
 }
 
+fn lua_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "Lua".into(),
+                path_suffixes: vec!["lua".into()],
+                collapsed_placeholder: "--[ ... ]--".to_string(),
+                ..Default::default()
+            },
+            Some(tree_sitter_lua::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                (function_declaration
+                    "function" @name
+                    name: (_) @name
+                    (comment)* @collapse
+                    body: (block) @collapse
+                ) @item
+            )
+        "#,
+        )
+        .unwrap(),
+    )
+}
+
+fn php_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "PHP".into(),
+                path_suffixes: vec!["php".into()],
+                collapsed_placeholder: "/* ... */".into(),
+                ..Default::default()
+            },
+            Some(tree_sitter_php::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                [
+                    (function_definition
+                        "function" @name
+                        name: (_) @name
+                        body: (_
+                            "{" @keep
+                            "}" @keep) @collapse
+                        )
+
+                    (trait_declaration
+                        "trait" @name
+                        name: (_) @name)
+
+                    (method_declaration
+                        "function" @name
+                        name: (_) @name
+                        body: (_
+                            "{" @keep
+                            "}" @keep) @collapse
+                        )
+
+                    (interface_declaration
+                        "interface" @name
+                        name: (_) @name
+                        )
+
+                    (enum_declaration
+                        "enum" @name
+                        name: (_) @name
+                        )
+
+                ] @item
+            )
+            "#,
+        )
+        .unwrap(),
+    )
+}
+
+fn ruby_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "Ruby".into(),
+                path_suffixes: vec!["rb".into()],
+                collapsed_placeholder: "# ...".to_string(),
+                ..Default::default()
+            },
+            Some(tree_sitter_ruby::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                [
+                (module
+                    "module" @name
+                    name: (_) @name)
+                (method
+                    "def" @name
+                    name: (_) @name
+                    body: (body_statement) @collapse)
+                (class
+                    "class" @name
+                    name: (_) @name)
+                (singleton_method
+                    "def" @name
+                    object: (_) @name
+                    "." @name
+                    name: (_) @name
+                    body: (body_statement) @collapse)
+                ] @item
+            )
+            "#,
+        )
+        .unwrap(),
+    )
+}
+
 fn elixir_lang() -> Arc<Language> {
     Arc::new(
         Language::new(

crates/zed/src/languages/lua/config.toml 🔗

@@ -7,3 +7,4 @@ brackets = [
     { start = "[", end = "]", close = true, newline = true },
     { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
 ]
+collapsed_placeholder = "--[ ... ]--"

crates/zed/src/languages/php/config.toml 🔗

@@ -9,3 +9,4 @@ brackets = [
     { start = "(", end = ")", close = true, newline = true },
     { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
 ]
+collapsed_placeholder = "/* ... */"

crates/zed/src/languages/php/embedding.scm 🔗

@@ -0,0 +1,36 @@
+(
+    (comment)* @context
+    .
+    [
+        (function_definition
+            "function" @name
+            name: (_) @name
+            body: (_
+                "{" @keep
+                "}" @keep) @collapse
+            )
+
+        (trait_declaration
+            "trait" @name
+            name: (_) @name)
+
+        (method_declaration
+            "function" @name
+            name: (_) @name
+            body: (_
+                "{" @keep
+                "}" @keep) @collapse
+            )
+
+        (interface_declaration
+            "interface" @name
+            name: (_) @name
+            )
+
+        (enum_declaration
+            "enum" @name
+            name: (_) @name
+            )
+
+        ] @item
+    )

crates/zed/src/languages/php/outline.scm 🔗

@@ -8,8 +8,6 @@
     name: (_) @name
     ) @item
 
-
-
 (method_declaration
     "function" @context
     name: (_) @name
@@ -24,3 +22,8 @@
     "enum" @context
     name: (_) @name
     ) @item
+
+(trait_declaration
+    "trait" @context
+    name: (_) @name
+    ) @item

crates/zed/src/languages/ruby/config.toml 🔗

@@ -10,3 +10,4 @@ brackets = [
   { start = "\"", end = "\"", close = true, newline = false, not_in = ["comment", "string"] },
   { start = "'", end = "'", close = true, newline = false, not_in = ["comment", "string"] },
 ]
+collapsed_placeholder = "# ..."

crates/zed/src/languages/ruby/embedding.scm 🔗

@@ -0,0 +1,22 @@
+(
+    (comment)* @context
+    .
+    [
+        (module
+            "module" @name
+            name: (_) @name)
+        (method
+            "def" @name
+            name: (_) @name
+            body: (body_statement) @collapse)
+        (class
+            "class" @name
+            name: (_) @name)
+        (singleton_method
+            "def" @name
+            object: (_) @name
+            "." @name
+            name: (_) @name
+            body: (body_statement) @collapse)
+        ] @item
+    )