Detailed changes
@@ -36,11 +36,11 @@ dependencies = [
[[package]]
name = "addr2line"
-version = "0.19.0"
+version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
dependencies = [
- "gimli 0.27.2",
+ "gimli 0.27.3",
]
[[package]]
@@ -61,7 +61,7 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
- "getrandom 0.2.9",
+ "getrandom 0.2.10",
"once_cell",
"version_check",
]
@@ -88,9 +88,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
dependencies = [
"memchr",
]
@@ -118,7 +118,7 @@ dependencies = [
"settings",
"smol",
"theme",
- "tiktoken-rs",
+ "tiktoken-rs 0.4.5",
"util",
"workspace",
]
@@ -151,7 +151,7 @@ dependencies = [
"alacritty_config",
"alacritty_config_derive",
"base64 0.13.1",
- "bitflags",
+ "bitflags 1.3.2",
"dirs 4.0.0",
"libc",
"log",
@@ -161,7 +161,7 @@ dependencies = [
"miow 0.3.7",
"nix",
"parking_lot 0.12.1",
- "regex-automata",
+ "regex-automata 0.1.10",
"serde",
"serde_yaml",
"signal-hook",
@@ -177,6 +177,12 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
+[[package]]
+name = "allocator-api2"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9"
+
[[package]]
name = "alsa"
version = "0.7.0"
@@ -184,7 +190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8512c9117059663fb5606788fbca3619e2a91dac0e3fe516242eab1fa6be5e44"
dependencies = [
"alsa-sys",
- "bitflags",
+ "bitflags 1.3.2",
"libc",
"nix",
]
@@ -205,6 +211,12 @@ version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec8ad6edb4840b78c5c3d88de606b22252d552b55f3a4699fbb10fc070ec3049"
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -225,7 +237,7 @@ dependencies = [
"anstyle-query",
"anstyle-wincon",
"colorchoice",
- "is-terminal 0.4.7",
+ "is-terminal 0.4.9",
"utf8parse",
]
@@ -250,7 +262,7 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys",
]
[[package]]
@@ -260,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
dependencies = [
"anstyle",
- "windows-sys 0.48.0",
+ "windows-sys",
]
[[package]]
@@ -283,9 +295,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "arrayvec"
-version = "0.7.2"
+version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "ascii"
@@ -306,9 +318,9 @@ dependencies = [
[[package]]
name = "async-channel"
-version = "1.8.0"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
+checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
dependencies = [
"concurrent-queue",
"event-listener",
@@ -324,7 +336,7 @@ dependencies = [
"futures-core",
"futures-io",
"once_cell",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"tokio",
]
@@ -338,7 +350,7 @@ dependencies = [
"futures-core",
"futures-io",
"memchr",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
]
[[package]]
@@ -362,7 +374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06"
dependencies = [
"async-lock",
- "autocfg 1.1.0",
+ "autocfg",
"blocking",
"futures-lite",
]
@@ -389,14 +401,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af"
dependencies = [
"async-lock",
- "autocfg 1.1.0",
+ "autocfg",
"cfg-if 1.0.0",
"concurrent-queue",
"futures-lite",
"log",
"parking",
"polling",
- "rustix 0.37.19",
+ "rustix 0.37.23",
"slab",
"socket2",
"waker-fn",
@@ -418,7 +430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4051e67316bc7eff608fe723df5d32ed639946adcd69e07df41fd42a7b411f1f"
dependencies = [
"async-io",
- "autocfg 1.1.0",
+ "autocfg",
"blocking",
"futures-lite",
]
@@ -440,14 +452,14 @@ checksum = "7a9d28b1d97e08915212e2e45310d47854eafa69600756fc735fb788f75199c9"
dependencies = [
"async-io",
"async-lock",
- "autocfg 1.1.0",
+ "autocfg",
"blocking",
"cfg-if 1.0.0",
"event-listener",
"futures-lite",
- "rustix 0.37.19",
+ "rustix 0.37.23",
"signal-hook",
- "windows-sys 0.48.0",
+ "windows-sys",
]
[[package]]
@@ -469,7 +481,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.25",
]
[[package]]
@@ -492,7 +504,7 @@ dependencies = [
"log",
"memchr",
"once_cell",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"pin-utils",
"slab",
"wasm-bindgen-futures",
@@ -506,7 +518,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
]
[[package]]
@@ -517,7 +529,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.25",
]
[[package]]
@@ -554,13 +566,13 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.68"
+version = "0.1.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.25",
]
[[package]]
@@ -573,7 +585,7 @@ dependencies = [
"futures-io",
"futures-util",
"log",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"tungstenite 0.16.0",
]
@@ -588,12 +600,9 @@ dependencies = [
[[package]]
name = "atomic"
-version = "0.5.1"
+version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c"
-dependencies = [
- "autocfg 1.1.0",
-]
+checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
[[package]]
name = "atomic-waker"
@@ -649,15 +658,6 @@ dependencies = [
"workspace",
]
-[[package]]
-name = "autocfg"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
-dependencies = [
- "autocfg 1.1.0",
-]
-
[[package]]
name = "autocfg"
version = "1.1.0"
@@ -673,19 +673,19 @@ dependencies = [
"async-trait",
"axum-core",
"base64 0.13.1",
- "bitflags",
+ "bitflags 1.3.2",
"bytes 1.4.0",
"futures-util",
"headers",
"http",
"http-body",
"hyper",
- "itoa 1.0.6",
+ "itoa 1.0.8",
"matchit",
"memchr",
"mime",
"percent-encoding",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"serde",
"serde_json",
"serde_urlencoded",
@@ -726,7 +726,7 @@ dependencies = [
"futures-util",
"http",
"mime",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"serde",
"serde_json",
"tokio",
@@ -738,16 +738,16 @@ dependencies = [
[[package]]
name = "backtrace"
-version = "0.3.67"
+version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
dependencies = [
- "addr2line 0.19.0",
+ "addr2line 0.20.0",
"cc",
"cfg-if 1.0.0",
"libc",
- "miniz_oxide 0.6.2",
- "object 0.30.3",
+ "miniz_oxide 0.7.1",
+ "object 0.31.1",
"rustc-demangle",
]
@@ -772,9 +772,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
-version = "0.21.0"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
[[package]]
name = "base64ct"
@@ -797,7 +797,7 @@ version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
@@ -817,7 +817,7 @@ version = "0.65.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
@@ -830,7 +830,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.18",
+ "syn 2.0.25",
"which",
]
@@ -855,6 +855,24 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
+[[package]]
+name = "bitvec"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
[[package]]
name = "block"
version = "0.1.6"
@@ -969,27 +987,26 @@ dependencies = [
[[package]]
name = "bstr"
-version = "1.4.0"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09"
+checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
dependencies = [
"memchr",
- "once_cell",
- "regex-automata",
+ "regex-automata 0.3.3",
"serde",
]
[[package]]
name = "bumpalo"
-version = "3.12.2"
+version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c6ed94e98ecff0c12dd1b04c15ec0d7d9458ca8fe806cea6f12954efe74c63b"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "bytecheck"
-version = "0.6.10"
+version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f"
+checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627"
dependencies = [
"bytecheck_derive",
"ptr_meta",
@@ -998,9 +1015,9 @@ dependencies = [
[[package]]
name = "bytecheck_derive"
-version = "0.6.10"
+version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5"
+checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61"
dependencies = [
"proc-macro2",
"quote",
@@ -1167,13 +1184,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
-version = "0.4.24"
+version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
+checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
dependencies = [
+ "android-tzdata",
"iana-time-zone",
"js-sys",
- "num-integer",
"num-traits",
"serde",
"time 0.1.45",
@@ -1204,7 +1221,7 @@ checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
dependencies = [
"glob",
"libc",
- "libloading",
+ "libloading 0.7.4",
]
[[package]]
@@ -1214,7 +1231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"atty",
- "bitflags",
+ "bitflags 1.3.2",
"clap_derive 3.2.25",
"clap_lex 0.2.4",
"indexmap 1.9.3",
@@ -1226,9 +1243,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.3.5"
+version = "4.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2686c4115cb0810d9a984776e197823d08ec94f176549a89a9efded477c456dc"
+checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d"
dependencies = [
"clap_builder",
"clap_derive 4.3.2",
@@ -1237,13 +1254,12 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.3.5"
+version = "4.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e53afce1efce6ed1f633cf0e57612fe51db54a1ee4fd8f8503d078fe02d69ae"
+checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b"
dependencies = [
"anstream",
"anstyle",
- "bitflags",
"clap_lex 0.5.0",
"strsim",
]
@@ -1270,7 +1286,7 @@ dependencies = [
"heck 0.4.1",
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.25",
]
[[package]]
@@ -1337,11 +1353,11 @@ dependencies = [
"sum_tree",
"tempfile",
"thiserror",
- "time 0.3.21",
+ "time 0.3.23",
"tiny_http",
"url",
"util",
- "uuid 1.3.2",
+ "uuid 1.4.0",
]
[[package]]
@@ -1365,7 +1381,7 @@ name = "cocoa"
version = "0.24.0"
source = "git+https://github.com/servo/core-foundation-rs?rev=079665882507dd5e2ff77db3de5070c1f6c0fb85#079665882507dd5e2ff77db3de5070c1f6c0fb85"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"block",
"cocoa-foundation",
"core-foundation",
@@ -1380,7 +1396,7 @@ name = "cocoa-foundation"
version = "0.1.1"
source = "git+https://github.com/servo/core-foundation-rs?rev=079665882507dd5e2ff77db3de5070c1f6c0fb85#079665882507dd5e2ff77db3de5070c1f6c0fb85"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"block",
"core-foundation",
"core-graphics-types",
@@ -1389,16 +1405,6 @@ dependencies = [
"objc",
]
-[[package]]
-name = "codespan-reporting"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
-dependencies = [
- "termcolor",
- "unicode-width",
-]
-
[[package]]
name = "collab"
version = "0.16.0"
@@ -1449,7 +1455,7 @@ dependencies = [
"sha-1 0.9.8",
"sqlx",
"theme",
- "time 0.3.21",
+ "time 0.3.23",
"tokio",
"tokio-tungstenite",
"toml",
@@ -1642,7 +1648,7 @@ name = "core-graphics"
version = "0.22.3"
source = "git+https://github.com/servo/core-foundation-rs?rev=079665882507dd5e2ff77db3de5070c1f6c0fb85#079665882507dd5e2ff77db3de5070c1f6c0fb85"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"core-foundation",
"core-graphics-types",
"foreign-types",
@@ -1654,7 +1660,7 @@ name = "core-graphics-types"
version = "0.1.1"
source = "git+https://github.com/servo/core-foundation-rs?rev=079665882507dd5e2ff77db3de5070c1f6c0fb85#079665882507dd5e2ff77db3de5070c1f6c0fb85"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"core-foundation",
"foreign-types",
"libc",
@@ -1687,7 +1693,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb17e2d1795b1996419648915df94bc7103c28f7b48062d7acf4652fc371b2ff"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"core-foundation-sys 0.6.2",
"coreaudio-sys",
]
@@ -1737,9 +1743,9 @@ dependencies = [
[[package]]
name = "cpufeatures"
-version = "0.2.7"
+version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
dependencies = [
"libc",
]
@@ -1887,14 +1893,14 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
-version = "0.9.14"
+version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
"cfg-if 1.0.0",
"crossbeam-utils",
- "memoffset 0.8.0",
+ "memoffset 0.9.0",
"scopeguard",
]
@@ -1910,9 +1916,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
-version = "0.8.15"
+version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if 1.0.0",
]
@@ -1964,9 +1970,9 @@ dependencies = [
[[package]]
name = "curl-sys"
-version = "0.4.61+curl-8.0.1"
+version = "0.4.63+curl-8.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14d05c10f541ae6f3bc5b3d923c20001f47db7d5f0b2bc6ad16490133842db79"
+checksum = "aeb0fef7046022a1e2ad67a004978f0e3cacb9e3123dc62ce768f92197b771dc"
dependencies = [
"cc",
"libc",
@@ -1977,61 +1983,17 @@ dependencies = [
"winapi 0.3.9",
]
-[[package]]
-name = "cxx"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93"
-dependencies = [
- "cc",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b"
-dependencies = [
- "cc",
- "codespan-reporting",
- "once_cell",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 2.0.18",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.18",
-]
-
[[package]]
name = "dashmap"
-version = "5.4.0"
+version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d"
dependencies = [
"cfg-if 1.0.0",
- "hashbrown 0.12.3",
+ "hashbrown 0.14.0",
"lock_api",
"once_cell",
- "parking_lot_core 0.9.7",
+ "parking_lot_core 0.9.8",
]
[[package]]
@@ -2136,9 +2098,9 @@ dependencies = [
[[package]]
name = "digest"
-version = "0.10.6"
+version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"crypto-common",
@@ -2207,11 +2169,11 @@ dependencies = [
[[package]]
name = "dlib"
-version = "0.5.0"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac1b7517328c04c2aa68422fc60a41b92208182142ed04a25879c26c8f878794"
+checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412"
dependencies = [
- "libloading",
+ "libloading 0.8.0",
]
[[package]]
@@ -2334,7 +2296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
dependencies = [
"humantime",
- "is-terminal 0.4.7",
+ "is-terminal 0.4.9",
"log",
"regex",
"termcolor",
@@ -2351,15 +2313,15 @@ dependencies = [
[[package]]
name = "equivalent"
-version = "1.0.0"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "erased-serde"
-version = "0.3.25"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569"
+checksum = "f94c0e13118e7d7533271f754a168ae8400e6a1cc043f2bfd53cc7290f1a1de3"
dependencies = [
"serde",
]
@@ -2383,7 +2345,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
dependencies = [
"errno-dragonfly",
"libc",
- "windows-sys 0.48.0",
+ "windows-sys",
]
[[package]]
@@ -2398,9 +2360,9 @@ dependencies = [
[[package]]
name = "etagere"
-version = "0.2.7"
+version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6301151a318f367f392c31395beb1cfba5ccd9abc44d1db0db3a4b27b9601c89"
+checksum = "fcf22f748754352918e082e0039335ee92454a5d62bcaf69b5e8daf5907d9644"
dependencies = [
"euclid",
"svg_fmt",
@@ -2427,6 +2389,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
[[package]]
name = "fancy-regex"
version = "0.11.0"
@@ -2517,7 +2485,7 @@ dependencies = [
"cfg-if 1.0.0",
"libc",
"redox_syscall 0.2.16",
- "windows-sys 0.48.0",
+ "windows-sys",
]
[[package]]
@@ -2571,7 +2539,7 @@ name = "font-kit"
version = "0.11.0"
source = "git+https://github.com/zed-industries/font-kit?rev=b2f77d56f450338aa4f7dd2f0197d8c9acb0cf18#b2f77d56f450338aa4f7dd2f0197d8c9acb0cf18"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"byteorder",
"core-foundation",
"core-graphics",
@@ -2618,9 +2586,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
-version = "1.1.0"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
dependencies = [
"percent-encoding",
]
@@ -2671,7 +2639,7 @@ dependencies = [
"smol",
"sum_tree",
"tempfile",
- "time 0.3.21",
+ "time 0.3.23",
"util",
]
@@ -2690,7 +2658,7 @@ dependencies = [
name = "fsevent"
version = "2.0.2"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"fsevent-sys",
"parking_lot 0.11.2",
"tempdir",
@@ -2717,7 +2685,7 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"fuchsia-zircon-sys",
]
@@ -2727,6 +2695,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
+[[package]]
+name = "funty"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
+
[[package]]
name = "futures"
version = "0.1.31"
@@ -2803,7 +2777,7 @@ dependencies = [
"futures-io",
"memchr",
"parking",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"waker-fn",
]
@@ -2815,7 +2789,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.25",
]
[[package]]
@@ -2844,7 +2818,7 @@ dependencies = [
"futures-sink",
"futures-task",
"memchr",
- "pin-project-lite 0.2.9",
+ "pin-project-lite 0.2.10",
"pin-utils",
"slab",
"tokio-io",
@@ -2890,9 +2864,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.2.9"
+version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if 1.0.0",
"libc",
@@ -2922,9 +2896,9 @@ dependencies = [
[[package]]
name = "gimli"
-version = "0.27.2"
+version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
[[package]]
name = "git"
@@ -2952,7 +2926,7 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2994bee4a3a6a51eb90c218523be382fd7ea09b16380b9312e9dbe955ff7c7d1"
dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
"libc",
"libgit2-sys",
"log",
@@ -2967,11 +2941,11 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "globset"
-version = "0.4.10"
+version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc"
+checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df"
dependencies = [
- "aho-corasick 0.7.20",
+ "aho-corasick 1.0.2",
"bstr",
"fnv",
"log",
@@ -3055,11 +3029,11 @@ dependencies = [
"smol",
"sqlez",
"sum_tree",
- "time 0.3.21",
+ "time 0.3.23",
"tiny-skia",
"usvg",
"util",
- "uuid 1.3.2",
+ "uuid 1.4.0",
"waker-fn",
]
@@ -63,6 +63,7 @@ members = [
"crates/theme",
"crates/theme_selector",
"crates/util",
+ "crates/vector_store",
"crates/vim",
"crates/vcs_menu",
"crates/workspace",
@@ -405,6 +405,7 @@
"cmd-k cmd-t": "theme_selector::Toggle",
"cmd-k cmd-s": "zed::OpenKeymap",
"cmd-t": "project_symbols::Toggle",
+ "cmd-ctrl-t": "semantic_search::Toggle",
"cmd-p": "file_finder::Toggle",
"cmd-shift-p": "command_palette::Toggle",
"cmd-shift-m": "diagnostics::Deploy",
@@ -291,6 +291,11 @@
// the terminal will default to matching the buffer's font family.
// "font_family": "Zed Mono"
},
+ // Difference settings for vector_store
+ "vector_store": {
+ "enabled": false,
+ "reindexing_delay_seconds": 600
+ },
// Different settings for specific languages.
"languages": {
"Plain Text": {
@@ -350,6 +350,7 @@ pub struct LanguageQueries {
pub brackets: Option<Cow<'static, str>>,
pub indents: Option<Cow<'static, str>>,
pub outline: Option<Cow<'static, str>>,
+ pub embedding: Option<Cow<'static, str>>,
pub injections: Option<Cow<'static, str>>,
pub overrides: Option<Cow<'static, str>>,
}
@@ -490,12 +491,13 @@ pub struct Language {
pub struct Grammar {
id: usize,
- pub(crate) ts_language: tree_sitter::Language,
+ pub ts_language: tree_sitter::Language,
pub(crate) error_query: Query,
pub(crate) highlights_query: Option<Query>,
pub(crate) brackets_config: Option<BracketConfig>,
pub(crate) indents_config: Option<IndentConfig>,
- pub(crate) outline_config: Option<OutlineConfig>,
+ pub outline_config: Option<OutlineConfig>,
+ pub embedding_config: Option<EmbeddingConfig>,
pub(crate) injection_config: Option<InjectionConfig>,
pub(crate) override_config: Option<OverrideConfig>,
pub(crate) highlight_map: Mutex<HighlightMap>,
@@ -509,12 +511,21 @@ struct IndentConfig {
outdent_capture_ix: Option<u32>,
}
-struct OutlineConfig {
- query: Query,
- item_capture_ix: u32,
- name_capture_ix: u32,
- context_capture_ix: Option<u32>,
- extra_context_capture_ix: Option<u32>,
+pub struct OutlineConfig {
+ pub query: Query,
+ pub item_capture_ix: u32,
+ pub name_capture_ix: u32,
+ pub context_capture_ix: Option<u32>,
+ pub extra_context_capture_ix: Option<u32>,
+}
+
+#[derive(Debug)]
+pub struct EmbeddingConfig {
+ pub query: Query,
+ pub item_capture_ix: u32,
+ pub name_capture_ix: u32,
+ pub context_capture_ix: Option<u32>,
+ pub extra_context_capture_ix: Option<u32>,
}
struct InjectionConfig {
@@ -1146,6 +1157,7 @@ impl Language {
highlights_query: None,
brackets_config: None,
outline_config: None,
+ embedding_config: None,
indents_config: None,
injection_config: None,
override_config: None,
@@ -1182,6 +1194,9 @@ impl Language {
if let Some(query) = queries.outline {
self = self.with_outline_query(query.as_ref())?;
}
+ if let Some(query) = queries.embedding {
+ self = self.with_embedding_query(query.as_ref())?;
+ }
if let Some(query) = queries.injections {
self = self.with_injection_query(query.as_ref())?;
}
@@ -1190,6 +1205,7 @@ impl Language {
}
Ok(self)
}
+
pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
@@ -1224,6 +1240,34 @@ impl Language {
Ok(self)
}
+ pub fn with_embedding_query(mut self, source: &str) -> Result<Self> {
+ let grammar = self.grammar_mut();
+ let query = Query::new(grammar.ts_language, source)?;
+ let mut item_capture_ix = None;
+ let mut name_capture_ix = None;
+ let mut context_capture_ix = None;
+ let mut extra_context_capture_ix = None;
+ get_capture_indices(
+ &query,
+ &mut [
+ ("item", &mut item_capture_ix),
+ ("name", &mut name_capture_ix),
+ ("context", &mut context_capture_ix),
+ ("context.extra", &mut extra_context_capture_ix),
+ ],
+ );
+ if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
+ grammar.embedding_config = Some(EmbeddingConfig {
+ query,
+ item_capture_ix,
+ name_capture_ix,
+ context_capture_ix,
+ extra_context_capture_ix,
+ });
+ }
+ Ok(self)
+ }
+
pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
let query = Query::new(grammar.ts_language, source)?;
@@ -261,6 +261,7 @@ pub enum Event {
ActiveEntryChanged(Option<ProjectEntryId>),
WorktreeAdded,
WorktreeRemoved(WorktreeId),
+ WorktreeUpdatedEntries(WorktreeId, UpdatedEntriesSet),
DiskBasedDiagnosticsStarted {
language_server_id: LanguageServerId,
},
@@ -5403,6 +5404,10 @@ impl Project {
this.update_local_worktree_buffers(&worktree, changes, cx);
this.update_local_worktree_language_servers(&worktree, changes, cx);
this.update_local_worktree_settings(&worktree, changes, cx);
+ cx.emit(Event::WorktreeUpdatedEntries(
+ worktree.read(cx).id(),
+ changes.clone(),
+ ));
}
worktree::Event::UpdatedGitRepositories(updated_repos) => {
this.update_local_worktree_buffers_git_repos(worktree, updated_repos, cx)
@@ -6,6 +6,7 @@ lazy_static::lazy_static! {
pub static ref HOME: PathBuf = dirs::home_dir().expect("failed to determine home directory");
pub static ref CONFIG_DIR: PathBuf = HOME.join(".config").join("zed");
pub static ref CONVERSATIONS_DIR: PathBuf = HOME.join(".config/zed/conversations");
+ pub static ref EMBEDDINGS_DIR: PathBuf = HOME.join(".config/zed/embeddings");
pub static ref LOGS_DIR: PathBuf = HOME.join("Library/Logs/Zed");
pub static ref SUPPORT_DIR: PathBuf = HOME.join("Library/Application Support/Zed");
pub static ref LANGUAGES_DIR: PathBuf = HOME.join("Library/Application Support/Zed/languages");
@@ -0,0 +1,49 @@
+[package]
+name = "vector_store"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+path = "src/vector_store.rs"
+doctest = false
+
+[dependencies]
+gpui = { path = "../gpui" }
+language = { path = "../language" }
+project = { path = "../project" }
+workspace = { path = "../workspace" }
+util = { path = "../util" }
+picker = { path = "../picker" }
+theme = { path = "../theme" }
+editor = { path = "../editor" }
+rpc = { path = "../rpc" }
+settings = { path = "../settings" }
+anyhow.workspace = true
+futures.workspace = true
+smol.workspace = true
+rusqlite = { version = "0.27.0", features = ["blob", "array", "modern_sqlite"] }
+isahc.workspace = true
+log.workspace = true
+tree-sitter.workspace = true
+lazy_static.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+async-trait.workspace = true
+bincode = "1.3.3"
+matrixmultiply = "0.3.7"
+tiktoken-rs = "0.5.0"
+rand.workspace = true
+schemars.workspace = true
+
+[dev-dependencies]
+gpui = { path = "../gpui", features = ["test-support"] }
+language = { path = "../language", features = ["test-support"] }
+project = { path = "../project", features = ["test-support"] }
+rpc = { path = "../rpc", features = ["test-support"] }
+workspace = { path = "../workspace", features = ["test-support"] }
+settings = { path = "../settings", features = ["test-support"]}
+tree-sitter-rust = "*"
+rand.workspace = true
+unindent.workspace = true
+tempdir.workspace = true
@@ -0,0 +1,31 @@
+
+WIP: Sample SQL Queries
+/*
+
+create table "files" (
+"id" INTEGER PRIMARY KEY,
+"path" VARCHAR,
+"sha1" VARCHAR,
+);
+
+create table symbols (
+"file_id" INTEGER REFERENCES("files", "id") ON CASCADE DELETE,
+"offset" INTEGER,
+"embedding" VECTOR,
+);
+
+insert into "files" ("path", "sha1") values ("src/main.rs", "sha1") return id;
+insert into symbols (
+"file_id",
+"start",
+"end",
+"embedding"
+) values (
+(id,),
+(id,),
+(id,),
+(id,),
+)
+
+
+*/
@@ -0,0 +1,325 @@
+use std::{
+ cmp::Ordering,
+ collections::HashMap,
+ path::{Path, PathBuf},
+ rc::Rc,
+ time::SystemTime,
+};
+
+use anyhow::{anyhow, Result};
+
+use crate::parsing::ParsedFile;
+use crate::VECTOR_STORE_VERSION;
+use rpc::proto::Timestamp;
+use rusqlite::{
+ params,
+ types::{FromSql, FromSqlResult, ValueRef},
+};
+
+#[derive(Debug)]
+pub struct FileRecord {
+ pub id: usize,
+ pub relative_path: String,
+ pub mtime: Timestamp,
+}
+
+#[derive(Debug)]
+struct Embedding(pub Vec<f32>);
+
+impl FromSql for Embedding {
+ fn column_result(value: ValueRef) -> FromSqlResult<Self> {
+ let bytes = value.as_blob()?;
+ let embedding: Result<Vec<f32>, Box<bincode::ErrorKind>> = bincode::deserialize(bytes);
+ if embedding.is_err() {
+ return Err(rusqlite::types::FromSqlError::Other(embedding.unwrap_err()));
+ }
+ return Ok(Embedding(embedding.unwrap()));
+ }
+}
+
+pub struct VectorDatabase {
+ db: rusqlite::Connection,
+}
+
+impl VectorDatabase {
+ pub fn new(path: String) -> Result<Self> {
+ let this = Self {
+ db: rusqlite::Connection::open(path)?,
+ };
+ this.initialize_database()?;
+ Ok(this)
+ }
+
+ fn initialize_database(&self) -> Result<()> {
+ rusqlite::vtab::array::load_module(&self.db)?;
+
+ // This will create the database if it doesnt exist
+
+ // Initialize Vector Databasing Tables
+ self.db.execute(
+ "CREATE TABLE IF NOT EXISTS worktrees (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ absolute_path VARCHAR NOT NULL
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS worktrees_absolute_path ON worktrees (absolute_path);
+ ",
+ [],
+ )?;
+
+ self.db.execute(
+ "CREATE TABLE IF NOT EXISTS files (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ worktree_id INTEGER NOT NULL,
+ relative_path VARCHAR NOT NULL,
+ mtime_seconds INTEGER NOT NULL,
+ mtime_nanos INTEGER NOT NULL,
+ vector_store_version INTEGER NOT NULL,
+ FOREIGN KEY(worktree_id) REFERENCES worktrees(id) ON DELETE CASCADE
+ )",
+ [],
+ )?;
+
+ self.db.execute(
+ "CREATE TABLE IF NOT EXISTS documents (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ file_id INTEGER NOT NULL,
+ offset INTEGER NOT NULL,
+ name VARCHAR NOT NULL,
+ embedding BLOB NOT NULL,
+ FOREIGN KEY(file_id) REFERENCES files(id) ON DELETE CASCADE
+ )",
+ [],
+ )?;
+
+ Ok(())
+ }
+
+ pub fn delete_file(&self, worktree_id: i64, delete_path: PathBuf) -> Result<()> {
+ self.db.execute(
+ "DELETE FROM files WHERE worktree_id = ?1 AND relative_path = ?2",
+ params![worktree_id, delete_path.to_str()],
+ )?;
+ Ok(())
+ }
+
+ pub fn insert_file(&self, worktree_id: i64, indexed_file: ParsedFile) -> Result<()> {
+ // Write to files table, and return generated id.
+ self.db.execute(
+ "
+ DELETE FROM files WHERE worktree_id = ?1 AND relative_path = ?2;
+ ",
+ params![worktree_id, indexed_file.path.to_str()],
+ )?;
+ let mtime = Timestamp::from(indexed_file.mtime);
+ self.db.execute(
+ "
+ INSERT INTO files
+ (worktree_id, relative_path, mtime_seconds, mtime_nanos, vector_store_version)
+ VALUES
+ (?1, ?2, $3, $4, $5);
+ ",
+ params![
+ worktree_id,
+ indexed_file.path.to_str(),
+ mtime.seconds,
+ mtime.nanos,
+ VECTOR_STORE_VERSION
+ ],
+ )?;
+
+ let file_id = self.db.last_insert_rowid();
+
+ // Currently inserting at approximately 3400 documents a second
+ // I imagine we can speed this up with a bulk insert of some kind.
+ for document in indexed_file.documents {
+ let embedding_blob = bincode::serialize(&document.embedding)?;
+
+ self.db.execute(
+ "INSERT INTO documents (file_id, offset, name, embedding) VALUES (?1, ?2, ?3, ?4)",
+ params![
+ file_id,
+ document.offset.to_string(),
+ document.name,
+ embedding_blob
+ ],
+ )?;
+ }
+
+ Ok(())
+ }
+
+ pub fn find_or_create_worktree(&self, worktree_root_path: &Path) -> Result<i64> {
+ // Check that the absolute path doesnt exist
+ let mut worktree_query = self
+ .db
+ .prepare("SELECT id FROM worktrees WHERE absolute_path = ?1")?;
+
+ let worktree_id = worktree_query
+ .query_row(params![worktree_root_path.to_string_lossy()], |row| {
+ Ok(row.get::<_, i64>(0)?)
+ })
+ .map_err(|err| anyhow!(err));
+
+ if worktree_id.is_ok() {
+ return worktree_id;
+ }
+
+ // If worktree_id is Err, insert new worktree
+ self.db.execute(
+ "
+ INSERT into worktrees (absolute_path) VALUES (?1)
+ ",
+ params![worktree_root_path.to_string_lossy()],
+ )?;
+ Ok(self.db.last_insert_rowid())
+ }
+
+ pub fn get_file_mtimes(&self, worktree_id: i64) -> Result<HashMap<PathBuf, SystemTime>> {
+ let mut statement = self.db.prepare(
+ "
+ SELECT relative_path, mtime_seconds, mtime_nanos
+ FROM files
+ WHERE worktree_id = ?1
+ ORDER BY relative_path",
+ )?;
+ let mut result: HashMap<PathBuf, SystemTime> = HashMap::new();
+ for row in statement.query_map(params![worktree_id], |row| {
+ Ok((
+ row.get::<_, String>(0)?.into(),
+ Timestamp {
+ seconds: row.get(1)?,
+ nanos: row.get(2)?,
+ }
+ .into(),
+ ))
+ })? {
+ let row = row?;
+ result.insert(row.0, row.1);
+ }
+ Ok(result)
+ }
+
+ pub fn top_k_search(
+ &self,
+ worktree_ids: &[i64],
+ query_embedding: &Vec<f32>,
+ limit: usize,
+ ) -> Result<Vec<(i64, PathBuf, usize, String)>> {
+ let mut results = Vec::<(i64, f32)>::with_capacity(limit + 1);
+ self.for_each_document(&worktree_ids, |id, embedding| {
+ let similarity = dot(&embedding, &query_embedding);
+ let ix = match results
+ .binary_search_by(|(_, s)| similarity.partial_cmp(&s).unwrap_or(Ordering::Equal))
+ {
+ Ok(ix) => ix,
+ Err(ix) => ix,
+ };
+ results.insert(ix, (id, similarity));
+ results.truncate(limit);
+ })?;
+
+ let ids = results.into_iter().map(|(id, _)| id).collect::<Vec<_>>();
+ self.get_documents_by_ids(&ids)
+ }
+
+ fn for_each_document(
+ &self,
+ worktree_ids: &[i64],
+ mut f: impl FnMut(i64, Vec<f32>),
+ ) -> Result<()> {
+ let mut query_statement = self.db.prepare(
+ "
+ SELECT
+ documents.id, documents.embedding
+ FROM
+ documents, files
+ WHERE
+ documents.file_id = files.id AND
+ files.worktree_id IN rarray(?)
+ ",
+ )?;
+
+ query_statement
+ .query_map(params![ids_to_sql(worktree_ids)], |row| {
+ Ok((row.get(0)?, row.get::<_, Embedding>(1)?))
+ })?
+ .filter_map(|row| row.ok())
+ .for_each(|(id, embedding)| f(id, embedding.0));
+ Ok(())
+ }
+
+ fn get_documents_by_ids(&self, ids: &[i64]) -> Result<Vec<(i64, PathBuf, usize, String)>> {
+ let mut statement = self.db.prepare(
+ "
+ SELECT
+ documents.id, files.worktree_id, files.relative_path, documents.offset, documents.name
+ FROM
+ documents, files
+ WHERE
+ documents.file_id = files.id AND
+ documents.id in rarray(?)
+ ",
+ )?;
+
+ let result_iter = statement.query_map(params![ids_to_sql(ids)], |row| {
+ Ok((
+ row.get::<_, i64>(0)?,
+ row.get::<_, i64>(1)?,
+ row.get::<_, String>(2)?.into(),
+ row.get(3)?,
+ row.get(4)?,
+ ))
+ })?;
+
+ let mut values_by_id = HashMap::<i64, (i64, PathBuf, usize, String)>::default();
+ for row in result_iter {
+ let (id, worktree_id, path, offset, name) = row?;
+ values_by_id.insert(id, (worktree_id, path, offset, name));
+ }
+
+ let mut results = Vec::with_capacity(ids.len());
+ for id in ids {
+ let value = values_by_id
+ .remove(id)
+ .ok_or(anyhow!("missing document id {}", id))?;
+ results.push(value);
+ }
+
+ Ok(results)
+ }
+}
+
+fn ids_to_sql(ids: &[i64]) -> Rc<Vec<rusqlite::types::Value>> {
+ Rc::new(
+ ids.iter()
+ .copied()
+ .map(|v| rusqlite::types::Value::from(v))
+ .collect::<Vec<_>>(),
+ )
+}
+
+pub(crate) fn dot(vec_a: &[f32], vec_b: &[f32]) -> f32 {
+ let len = vec_a.len();
+ assert_eq!(len, vec_b.len());
+
+ let mut result = 0.0;
+ unsafe {
+ matrixmultiply::sgemm(
+ 1,
+ len,
+ 1,
+ 1.0,
+ vec_a.as_ptr(),
+ len as isize,
+ 1,
+ vec_b.as_ptr(),
+ 1,
+ len as isize,
+ 0.0,
+ &mut result as *mut f32,
+ 1,
+ 1,
+ );
+ }
+ result
+}
@@ -0,0 +1,166 @@
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use futures::AsyncReadExt;
+use gpui::executor::Background;
+use gpui::serde_json;
+use isahc::http::StatusCode;
+use isahc::prelude::Configurable;
+use isahc::{AsyncBody, Response};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use std::env;
+use std::sync::Arc;
+use std::time::Duration;
+use tiktoken_rs::{cl100k_base, CoreBPE};
+use util::http::{HttpClient, Request};
+
+lazy_static! {
+ static ref OPENAI_API_KEY: Option<String> = env::var("OPENAI_API_KEY").ok();
+ static ref OPENAI_BPE_TOKENIZER: CoreBPE = cl100k_base().unwrap();
+}
+
+#[derive(Clone)]
+pub struct OpenAIEmbeddings {
+ pub client: Arc<dyn HttpClient>,
+ pub executor: Arc<Background>,
+}
+
+#[derive(Serialize)]
+struct OpenAIEmbeddingRequest<'a> {
+ model: &'static str,
+ input: Vec<&'a str>,
+}
+
+#[derive(Deserialize)]
+struct OpenAIEmbeddingResponse {
+ data: Vec<OpenAIEmbedding>,
+ usage: OpenAIEmbeddingUsage,
+}
+
+#[derive(Debug, Deserialize)]
+struct OpenAIEmbedding {
+ embedding: Vec<f32>,
+ index: usize,
+ object: String,
+}
+
+#[derive(Deserialize)]
+struct OpenAIEmbeddingUsage {
+ prompt_tokens: usize,
+ total_tokens: usize,
+}
+
+#[async_trait]
+pub trait EmbeddingProvider: Sync + Send {
+ async fn embed_batch(&self, spans: Vec<&str>) -> Result<Vec<Vec<f32>>>;
+}
+
+pub struct DummyEmbeddings {}
+
+#[async_trait]
+impl EmbeddingProvider for DummyEmbeddings {
+ async fn embed_batch(&self, spans: Vec<&str>) -> Result<Vec<Vec<f32>>> {
+ // 1024 is the OpenAI Embeddings size for ada models.
+ // the model we will likely be starting with.
+ let dummy_vec = vec![0.32 as f32; 1536];
+ return Ok(vec![dummy_vec; spans.len()]);
+ }
+}
+
+impl OpenAIEmbeddings {
+ async fn truncate(span: String) -> String {
+ let mut tokens = OPENAI_BPE_TOKENIZER.encode_with_special_tokens(span.as_ref());
+ if tokens.len() > 8190 {
+ tokens.truncate(8190);
+ let result = OPENAI_BPE_TOKENIZER.decode(tokens.clone());
+ if result.is_ok() {
+ let transformed = result.unwrap();
+ // assert_ne!(transformed, span);
+ return transformed;
+ }
+ }
+
+ return span.to_string();
+ }
+
+ async fn send_request(&self, api_key: &str, spans: Vec<&str>) -> Result<Response<AsyncBody>> {
+ let request = Request::post("https://api.openai.com/v1/embeddings")
+ .redirect_policy(isahc::config::RedirectPolicy::Follow)
+ .header("Content-Type", "application/json")
+ .header("Authorization", format!("Bearer {}", api_key))
+ .body(
+ serde_json::to_string(&OpenAIEmbeddingRequest {
+ input: spans.clone(),
+ model: "text-embedding-ada-002",
+ })
+ .unwrap()
+ .into(),
+ )?;
+
+ Ok(self.client.send(request).await?)
+ }
+}
+
+#[async_trait]
+impl EmbeddingProvider for OpenAIEmbeddings {
+ async fn embed_batch(&self, spans: Vec<&str>) -> Result<Vec<Vec<f32>>> {
+ const BACKOFF_SECONDS: [usize; 3] = [65, 180, 360];
+ const MAX_RETRIES: usize = 3;
+
+ let api_key = OPENAI_API_KEY
+ .as_ref()
+ .ok_or_else(|| anyhow!("no api key"))?;
+
+ let mut request_number = 0;
+ let mut response: Response<AsyncBody>;
+ let mut spans: Vec<String> = spans.iter().map(|x| x.to_string()).collect();
+ while request_number < MAX_RETRIES {
+ response = self
+ .send_request(api_key, spans.iter().map(|x| &**x).collect())
+ .await?;
+ request_number += 1;
+
+ if request_number + 1 == MAX_RETRIES && response.status() != StatusCode::OK {
+ return Err(anyhow!(
+ "openai max retries, error: {:?}",
+ &response.status()
+ ));
+ }
+
+ match response.status() {
+ StatusCode::TOO_MANY_REQUESTS => {
+ let delay = Duration::from_secs(BACKOFF_SECONDS[request_number - 1] as u64);
+ self.executor.timer(delay).await;
+ }
+ StatusCode::BAD_REQUEST => {
+ log::info!("BAD REQUEST: {:?}", &response.status());
+ // Don't worry about delaying bad request, as we can assume
+ // we haven't been rate limited yet.
+ for span in spans.iter_mut() {
+ *span = Self::truncate(span.to_string()).await;
+ }
+ }
+ StatusCode::OK => {
+ let mut body = String::new();
+ response.body_mut().read_to_string(&mut body).await?;
+ let response: OpenAIEmbeddingResponse = serde_json::from_str(&body)?;
+
+ log::info!(
+ "openai embedding completed. tokens: {:?}",
+ response.usage.total_tokens
+ );
+ return Ok(response
+ .data
+ .into_iter()
+ .map(|embedding| embedding.embedding)
+ .collect());
+ }
+ _ => {
+ return Err(anyhow!("openai embedding failed {}", response.status()));
+ }
+ }
+ }
+
+ Err(anyhow!("openai embedding failed"))
+ }
+}
@@ -0,0 +1,172 @@
+use crate::{SearchResult, VectorStore};
+use editor::{scroll::autoscroll::Autoscroll, Editor};
+use gpui::{
+ actions, elements::*, AnyElement, AppContext, ModelHandle, MouseState, Task, ViewContext,
+ WeakViewHandle,
+};
+use picker::{Picker, PickerDelegate, PickerEvent};
+use project::{Project, ProjectPath};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+use util::ResultExt;
+use workspace::Workspace;
+
+const MIN_QUERY_LEN: usize = 5;
+const EMBEDDING_DEBOUNCE_INTERVAL: Duration = Duration::from_millis(500);
+
+actions!(semantic_search, [Toggle]);
+
+pub type SemanticSearch = Picker<SemanticSearchDelegate>;
+
+pub struct SemanticSearchDelegate {
+ workspace: WeakViewHandle<Workspace>,
+ project: ModelHandle<Project>,
+ vector_store: ModelHandle<VectorStore>,
+ selected_match_index: usize,
+ matches: Vec<SearchResult>,
+ history: HashMap<String, Vec<SearchResult>>,
+}
+
+impl SemanticSearchDelegate {
+ // This is currently searching on every keystroke,
+ // This is wildly overkill, and has the potential to get expensive
+ // We will need to update this to throttle searching
+ pub fn new(
+ workspace: WeakViewHandle<Workspace>,
+ project: ModelHandle<Project>,
+ vector_store: ModelHandle<VectorStore>,
+ ) -> Self {
+ Self {
+ workspace,
+ project,
+ vector_store,
+ selected_match_index: 0,
+ matches: vec![],
+ history: HashMap::new(),
+ }
+ }
+}
+
+impl PickerDelegate for SemanticSearchDelegate {
+ fn placeholder_text(&self) -> Arc<str> {
+ "Search repository in natural language...".into()
+ }
+
+ fn confirm(&mut self, cx: &mut ViewContext<SemanticSearch>) {
+ if let Some(search_result) = self.matches.get(self.selected_match_index) {
+ // Open Buffer
+ let search_result = search_result.clone();
+ let buffer = self.project.update(cx, |project, cx| {
+ project.open_buffer(
+ ProjectPath {
+ worktree_id: search_result.worktree_id,
+ path: search_result.file_path.clone().into(),
+ },
+ cx,
+ )
+ });
+
+ let workspace = self.workspace.clone();
+ let position = search_result.clone().offset;
+ cx.spawn(|_, mut cx| async move {
+ let buffer = buffer.await?;
+ workspace.update(&mut cx, |workspace, cx| {
+ let editor = workspace.open_project_item::<Editor>(buffer, cx);
+ editor.update(cx, |editor, cx| {
+ editor.change_selections(Some(Autoscroll::center()), cx, |s| {
+ s.select_ranges([position..position])
+ });
+ });
+ })?;
+ Ok::<_, anyhow::Error>(())
+ })
+ .detach_and_log_err(cx);
+ cx.emit(PickerEvent::Dismiss);
+ }
+ }
+
+ fn dismissed(&mut self, _cx: &mut ViewContext<SemanticSearch>) {}
+
+ fn match_count(&self) -> usize {
+ self.matches.len()
+ }
+
+ fn selected_index(&self) -> usize {
+ self.selected_match_index
+ }
+
+ fn set_selected_index(&mut self, ix: usize, _cx: &mut ViewContext<SemanticSearch>) {
+ self.selected_match_index = ix;
+ }
+
+ fn update_matches(&mut self, query: String, cx: &mut ViewContext<SemanticSearch>) -> Task<()> {
+ log::info!("Searching for {:?}...", query);
+ if query.len() < MIN_QUERY_LEN {
+ log::info!("Query below minimum length");
+ return Task::ready(());
+ }
+
+ let vector_store = self.vector_store.clone();
+ let project = self.project.clone();
+ cx.spawn(|this, mut cx| async move {
+ cx.background().timer(EMBEDDING_DEBOUNCE_INTERVAL).await;
+
+ let retrieved_cached = this.update(&mut cx, |this, _| {
+ let delegate = this.delegate_mut();
+ if delegate.history.contains_key(&query) {
+ let historic_results = delegate.history.get(&query).unwrap().to_owned();
+ delegate.matches = historic_results.clone();
+ true
+ } else {
+ false
+ }
+ });
+
+ if let Some(retrieved) = retrieved_cached.log_err() {
+ if !retrieved {
+ let task = vector_store.update(&mut cx, |store, cx| {
+ store.search(project.clone(), query.to_string(), 10, cx)
+ });
+
+ if let Some(results) = task.await.log_err() {
+ log::info!("Not queried previously, searching...");
+ this.update(&mut cx, |this, _| {
+ let delegate = this.delegate_mut();
+ delegate.matches = results.clone();
+ delegate.history.insert(query, results);
+ })
+ .ok();
+ }
+ } else {
+ log::info!("Already queried, retrieved directly from cached history");
+ }
+ }
+ })
+ }
+
+ fn render_match(
+ &self,
+ ix: usize,
+ mouse_state: &mut MouseState,
+ selected: bool,
+ cx: &AppContext,
+ ) -> AnyElement<Picker<Self>> {
+ let theme = theme::current(cx);
+ let style = &theme.picker.item;
+ let current_style = style.in_state(selected).style_for(mouse_state);
+
+ let search_result = &self.matches[ix];
+
+ let path = search_result.file_path.to_string_lossy();
+ let name = search_result.name.clone();
+
+ Flex::column()
+ .with_child(Text::new(name, current_style.label.text.clone()).with_soft_wrap(false))
+ .with_child(Label::new(
+ path.to_string(),
+ style.inactive_state().default.label.clone(),
+ ))
+ .contained()
+ .with_style(current_style.container)
+ .into_any()
+ }
+}
@@ -0,0 +1,118 @@
+use std::{path::PathBuf, sync::Arc, time::SystemTime};
+
+use anyhow::{anyhow, Ok, Result};
+use project::Fs;
+use tree_sitter::{Parser, QueryCursor};
+
+use crate::PendingFile;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct Document {
+ pub offset: usize,
+ pub name: String,
+ pub embedding: Vec<f32>,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct ParsedFile {
+ pub path: PathBuf,
+ pub mtime: SystemTime,
+ pub documents: Vec<Document>,
+}
+
+const CODE_CONTEXT_TEMPLATE: &str =
+ "The below code snippet is from file '<path>'\n\n```<language>\n<item>\n```";
+
+pub struct CodeContextRetriever {
+ pub parser: Parser,
+ pub cursor: QueryCursor,
+ pub fs: Arc<dyn Fs>,
+}
+
+impl CodeContextRetriever {
+ pub async fn parse_file(
+ &mut self,
+ pending_file: PendingFile,
+ ) -> Result<(ParsedFile, Vec<String>)> {
+ let grammar = pending_file
+ .language
+ .grammar()
+ .ok_or_else(|| anyhow!("no grammar for language"))?;
+ let embedding_config = grammar
+ .embedding_config
+ .as_ref()
+ .ok_or_else(|| anyhow!("no embedding queries"))?;
+
+ let content = self.fs.load(&pending_file.absolute_path).await?;
+
+ self.parser.set_language(grammar.ts_language).unwrap();
+
+ let tree = self
+ .parser
+ .parse(&content, None)
+ .ok_or_else(|| anyhow!("parsing failed"))?;
+
+ let mut documents = Vec::new();
+ let mut context_spans = Vec::new();
+
+ // Iterate through query matches
+ for mat in self.cursor.matches(
+ &embedding_config.query,
+ tree.root_node(),
+ content.as_bytes(),
+ ) {
+ // log::info!("-----MATCH-----");
+
+ let mut name: Vec<&str> = vec![];
+ let mut item: Option<&str> = None;
+ let mut offset: Option<usize> = None;
+ for capture in mat.captures {
+ if capture.index == embedding_config.item_capture_ix {
+ offset = Some(capture.node.byte_range().start);
+ item = content.get(capture.node.byte_range());
+ } else if capture.index == embedding_config.name_capture_ix {
+ if let Some(name_content) = content.get(capture.node.byte_range()) {
+ name.push(name_content);
+ }
+ }
+
+ if let Some(context_capture_ix) = embedding_config.context_capture_ix {
+ if capture.index == context_capture_ix {
+ if let Some(context) = content.get(capture.node.byte_range()) {
+ name.push(context);
+ }
+ }
+ }
+ }
+
+ if item.is_some() && offset.is_some() && name.len() > 0 {
+ let context_span = CODE_CONTEXT_TEMPLATE
+ .replace("<path>", pending_file.relative_path.to_str().unwrap())
+ .replace("<language>", &pending_file.language.name().to_lowercase())
+ .replace("<item>", item.unwrap());
+
+ let mut truncated_span = context_span.clone();
+ truncated_span.truncate(100);
+
+ // log::info!("Name: {:?}", name);
+ // log::info!("Span: {:?}", truncated_span);
+
+ context_spans.push(context_span);
+ documents.push(Document {
+ name: name.join(" "),
+ offset: offset.unwrap(),
+ embedding: Vec::new(),
+ })
+ }
+ }
+
+ return Ok((
+ ParsedFile {
+ path: pending_file.relative_path,
+ mtime: pending_file.modified_time,
+ documents,
+ },
+ context_spans,
+ ));
+ }
+}
@@ -0,0 +1,770 @@
+mod db;
+mod embedding;
+mod modal;
+mod parsing;
+mod vector_store_settings;
+
+#[cfg(test)]
+mod vector_store_tests;
+
+use crate::vector_store_settings::VectorStoreSettings;
+use anyhow::{anyhow, Result};
+use db::VectorDatabase;
+use embedding::{EmbeddingProvider, OpenAIEmbeddings};
+use futures::{channel::oneshot, Future};
+use gpui::{
+ AppContext, AsyncAppContext, Entity, ModelContext, ModelHandle, Task, ViewContext,
+ WeakModelHandle,
+};
+use language::{Language, LanguageRegistry};
+use modal::{SemanticSearch, SemanticSearchDelegate, Toggle};
+use parsing::{CodeContextRetriever, ParsedFile};
+use project::{Fs, PathChange, Project, ProjectEntryId, WorktreeId};
+use smol::channel;
+use std::{
+ collections::HashMap,
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::{Duration, Instant, SystemTime},
+};
+use tree_sitter::{Parser, QueryCursor};
+use util::{
+ channel::{ReleaseChannel, RELEASE_CHANNEL, RELEASE_CHANNEL_NAME},
+ http::HttpClient,
+ paths::EMBEDDINGS_DIR,
+ ResultExt,
+};
+use workspace::{Workspace, WorkspaceCreated};
+
+const VECTOR_STORE_VERSION: usize = 0;
+const EMBEDDINGS_BATCH_SIZE: usize = 150;
+
+pub fn init(
+ fs: Arc<dyn Fs>,
+ http_client: Arc<dyn HttpClient>,
+ language_registry: Arc<LanguageRegistry>,
+ cx: &mut AppContext,
+) {
+ settings::register::<VectorStoreSettings>(cx);
+
+ let db_file_path = EMBEDDINGS_DIR
+ .join(Path::new(RELEASE_CHANNEL_NAME.as_str()))
+ .join("embeddings_db");
+
+ SemanticSearch::init(cx);
+ cx.add_action(
+ |workspace: &mut Workspace, _: &Toggle, cx: &mut ViewContext<Workspace>| {
+ if cx.has_global::<ModelHandle<VectorStore>>() {
+ let vector_store = cx.global::<ModelHandle<VectorStore>>().clone();
+ workspace.toggle_modal(cx, |workspace, cx| {
+ let project = workspace.project().clone();
+ let workspace = cx.weak_handle();
+ cx.add_view(|cx| {
+ SemanticSearch::new(
+ SemanticSearchDelegate::new(workspace, project, vector_store),
+ cx,
+ )
+ })
+ });
+ }
+ },
+ );
+
+ if *RELEASE_CHANNEL == ReleaseChannel::Stable
+ || !settings::get::<VectorStoreSettings>(cx).enabled
+ {
+ return;
+ }
+
+ cx.spawn(move |mut cx| async move {
+ let vector_store = VectorStore::new(
+ fs,
+ db_file_path,
+ // Arc::new(embedding::DummyEmbeddings {}),
+ Arc::new(OpenAIEmbeddings {
+ client: http_client,
+ executor: cx.background(),
+ }),
+ language_registry,
+ cx.clone(),
+ )
+ .await?;
+
+ cx.update(|cx| {
+ cx.set_global(vector_store.clone());
+ cx.subscribe_global::<WorkspaceCreated, _>({
+ let vector_store = vector_store.clone();
+ move |event, cx| {
+ let workspace = &event.0;
+ if let Some(workspace) = workspace.upgrade(cx) {
+ let project = workspace.read(cx).project().clone();
+ if project.read(cx).is_local() {
+ vector_store.update(cx, |store, cx| {
+ store.add_project(project, cx).detach();
+ });
+ }
+ }
+ }
+ })
+ .detach();
+ });
+
+ anyhow::Ok(())
+ })
+ .detach();
+}
+
+pub struct VectorStore {
+ fs: Arc<dyn Fs>,
+ database_url: Arc<PathBuf>,
+ embedding_provider: Arc<dyn EmbeddingProvider>,
+ language_registry: Arc<LanguageRegistry>,
+ db_update_tx: channel::Sender<DbOperation>,
+ parsing_files_tx: channel::Sender<PendingFile>,
+ _db_update_task: Task<()>,
+ _embed_batch_task: Task<()>,
+ _batch_files_task: Task<()>,
+ _parsing_files_tasks: Vec<Task<()>>,
+ projects: HashMap<WeakModelHandle<Project>, ProjectState>,
+}
+
+struct ProjectState {
+ worktree_db_ids: Vec<(WorktreeId, i64)>,
+ pending_files: HashMap<PathBuf, (PendingFile, SystemTime)>,
+ _subscription: gpui::Subscription,
+}
+
+impl ProjectState {
+ fn db_id_for_worktree_id(&self, id: WorktreeId) -> Option<i64> {
+ self.worktree_db_ids
+ .iter()
+ .find_map(|(worktree_id, db_id)| {
+ if *worktree_id == id {
+ Some(*db_id)
+ } else {
+ None
+ }
+ })
+ }
+
+ fn worktree_id_for_db_id(&self, id: i64) -> Option<WorktreeId> {
+ self.worktree_db_ids
+ .iter()
+ .find_map(|(worktree_id, db_id)| {
+ if *db_id == id {
+ Some(*worktree_id)
+ } else {
+ None
+ }
+ })
+ }
+
+ fn update_pending_files(&mut self, pending_file: PendingFile, indexing_time: SystemTime) {
+ // If Pending File Already Exists, Replace it with the new one
+ // but keep the old indexing time
+ if let Some(old_file) = self
+ .pending_files
+ .remove(&pending_file.relative_path.clone())
+ {
+ self.pending_files.insert(
+ pending_file.relative_path.clone(),
+ (pending_file, old_file.1),
+ );
+ } else {
+ self.pending_files.insert(
+ pending_file.relative_path.clone(),
+ (pending_file, indexing_time),
+ );
+ };
+ }
+
+ fn get_outstanding_files(&mut self) -> Vec<PendingFile> {
+ let mut outstanding_files = vec![];
+ let mut remove_keys = vec![];
+ for key in self.pending_files.keys().into_iter() {
+ if let Some(pending_details) = self.pending_files.get(key) {
+ let (pending_file, index_time) = pending_details;
+ if index_time <= &SystemTime::now() {
+ outstanding_files.push(pending_file.clone());
+ remove_keys.push(key.clone());
+ }
+ }
+ }
+
+ for key in remove_keys.iter() {
+ self.pending_files.remove(key);
+ }
+
+ return outstanding_files;
+ }
+}
+
+#[derive(Clone, Debug)]
+pub struct PendingFile {
+ worktree_db_id: i64,
+ relative_path: PathBuf,
+ absolute_path: PathBuf,
+ language: Arc<Language>,
+ modified_time: SystemTime,
+}
+
+#[derive(Debug, Clone)]
+pub struct SearchResult {
+ pub worktree_id: WorktreeId,
+ pub name: String,
+ pub offset: usize,
+ pub file_path: PathBuf,
+}
+
+enum DbOperation {
+ InsertFile {
+ worktree_id: i64,
+ indexed_file: ParsedFile,
+ },
+ Delete {
+ worktree_id: i64,
+ path: PathBuf,
+ },
+ FindOrCreateWorktree {
+ path: PathBuf,
+ sender: oneshot::Sender<Result<i64>>,
+ },
+ FileMTimes {
+ worktree_id: i64,
+ sender: oneshot::Sender<Result<HashMap<PathBuf, SystemTime>>>,
+ },
+}
+
+enum EmbeddingJob {
+ Enqueue {
+ worktree_id: i64,
+ parsed_file: ParsedFile,
+ document_spans: Vec<String>,
+ },
+ Flush,
+}
+
+impl VectorStore {
+ async fn new(
+ fs: Arc<dyn Fs>,
+ database_url: PathBuf,
+ embedding_provider: Arc<dyn EmbeddingProvider>,
+ language_registry: Arc<LanguageRegistry>,
+ mut cx: AsyncAppContext,
+ ) -> Result<ModelHandle<Self>> {
+ let database_url = Arc::new(database_url);
+
+ let db = cx
+ .background()
+ .spawn({
+ let fs = fs.clone();
+ let database_url = database_url.clone();
+ async move {
+ if let Some(db_directory) = database_url.parent() {
+ fs.create_dir(db_directory).await.log_err();
+ }
+
+ let db = VectorDatabase::new(database_url.to_string_lossy().to_string())?;
+ anyhow::Ok(db)
+ }
+ })
+ .await?;
+
+ Ok(cx.add_model(|cx| {
+ // paths_tx -> embeddings_tx -> db_update_tx
+
+ //db_update_tx/rx: Updating Database
+ let (db_update_tx, db_update_rx) = channel::unbounded();
+ let _db_update_task = cx.background().spawn(async move {
+ while let Ok(job) = db_update_rx.recv().await {
+ match job {
+ DbOperation::InsertFile {
+ worktree_id,
+ indexed_file,
+ } => {
+ db.insert_file(worktree_id, indexed_file).log_err();
+ }
+ DbOperation::Delete { worktree_id, path } => {
+ db.delete_file(worktree_id, path).log_err();
+ }
+ DbOperation::FindOrCreateWorktree { path, sender } => {
+ let id = db.find_or_create_worktree(&path);
+ sender.send(id).ok();
+ }
+ DbOperation::FileMTimes {
+ worktree_id: worktree_db_id,
+ sender,
+ } => {
+ let file_mtimes = db.get_file_mtimes(worktree_db_id);
+ sender.send(file_mtimes).ok();
+ }
+ }
+ }
+ });
+
+ // embed_tx/rx: Embed Batch and Send to Database
+ let (embed_batch_tx, embed_batch_rx) =
+ channel::unbounded::<Vec<(i64, ParsedFile, Vec<String>)>>();
+ let _embed_batch_task = cx.background().spawn({
+ let db_update_tx = db_update_tx.clone();
+ let embedding_provider = embedding_provider.clone();
+ async move {
+ while let Ok(mut embeddings_queue) = embed_batch_rx.recv().await {
+ // Construct Batch
+ let mut document_spans = vec![];
+ for (_, _, document_span) in embeddings_queue.iter() {
+ document_spans.extend(document_span.iter().map(|s| s.as_str()));
+ }
+
+ if let Ok(embeddings) = embedding_provider.embed_batch(document_spans).await
+ {
+ let mut i = 0;
+ let mut j = 0;
+
+ for embedding in embeddings.iter() {
+ while embeddings_queue[i].1.documents.len() == j {
+ i += 1;
+ j = 0;
+ }
+
+ embeddings_queue[i].1.documents[j].embedding = embedding.to_owned();
+ j += 1;
+ }
+
+ for (worktree_id, indexed_file, _) in embeddings_queue.into_iter() {
+ for document in indexed_file.documents.iter() {
+ // TODO: Update this so it doesn't panic
+ assert!(
+ document.embedding.len() > 0,
+ "Document Embedding Not Complete"
+ );
+ }
+
+ db_update_tx
+ .send(DbOperation::InsertFile {
+ worktree_id,
+ indexed_file,
+ })
+ .await
+ .unwrap();
+ }
+ }
+ }
+ }
+ });
+
+ // batch_tx/rx: Batch Files to Send for Embeddings
+ let (batch_files_tx, batch_files_rx) = channel::unbounded::<EmbeddingJob>();
+ let _batch_files_task = cx.background().spawn(async move {
+ let mut queue_len = 0;
+ let mut embeddings_queue = vec![];
+
+ while let Ok(job) = batch_files_rx.recv().await {
+ let should_flush = match job {
+ EmbeddingJob::Enqueue {
+ document_spans,
+ worktree_id,
+ parsed_file,
+ } => {
+ queue_len += &document_spans.len();
+ embeddings_queue.push((worktree_id, parsed_file, document_spans));
+ queue_len >= EMBEDDINGS_BATCH_SIZE
+ }
+ EmbeddingJob::Flush => true,
+ };
+
+ if should_flush {
+ embed_batch_tx.try_send(embeddings_queue).unwrap();
+ embeddings_queue = vec![];
+ queue_len = 0;
+ }
+ }
+ });
+
+ // parsing_files_tx/rx: Parsing Files to Embeddable Documents
+ let (parsing_files_tx, parsing_files_rx) = channel::unbounded::<PendingFile>();
+
+ let mut _parsing_files_tasks = Vec::new();
+ // for _ in 0..cx.background().num_cpus() {
+ for _ in 0..1 {
+ let fs = fs.clone();
+ let parsing_files_rx = parsing_files_rx.clone();
+ let batch_files_tx = batch_files_tx.clone();
+ _parsing_files_tasks.push(cx.background().spawn(async move {
+ let parser = Parser::new();
+ let cursor = QueryCursor::new();
+ let mut retriever = CodeContextRetriever { parser, cursor, fs };
+ while let Ok(pending_file) = parsing_files_rx.recv().await {
+ if let Some((indexed_file, document_spans)) =
+ retriever.parse_file(pending_file.clone()).await.log_err()
+ {
+ batch_files_tx
+ .try_send(EmbeddingJob::Enqueue {
+ worktree_id: pending_file.worktree_db_id,
+ parsed_file: indexed_file,
+ document_spans,
+ })
+ .unwrap();
+ }
+
+ if parsing_files_rx.len() == 0 {
+ batch_files_tx.try_send(EmbeddingJob::Flush).unwrap();
+ }
+ }
+ }));
+ }
+
+ Self {
+ fs,
+ database_url,
+ embedding_provider,
+ language_registry,
+ db_update_tx,
+ parsing_files_tx,
+ _db_update_task,
+ _embed_batch_task,
+ _batch_files_task,
+ _parsing_files_tasks,
+ projects: HashMap::new(),
+ }
+ }))
+ }
+
+ fn find_or_create_worktree(&self, path: PathBuf) -> impl Future<Output = Result<i64>> {
+ let (tx, rx) = oneshot::channel();
+ self.db_update_tx
+ .try_send(DbOperation::FindOrCreateWorktree { path, sender: tx })
+ .unwrap();
+ async move { rx.await? }
+ }
+
+ fn get_file_mtimes(
+ &self,
+ worktree_id: i64,
+ ) -> impl Future<Output = Result<HashMap<PathBuf, SystemTime>>> {
+ let (tx, rx) = oneshot::channel();
+ self.db_update_tx
+ .try_send(DbOperation::FileMTimes {
+ worktree_id,
+ sender: tx,
+ })
+ .unwrap();
+ async move { rx.await? }
+ }
+
+ fn add_project(
+ &mut self,
+ project: ModelHandle<Project>,
+ cx: &mut ModelContext<Self>,
+ ) -> Task<Result<()>> {
+ let worktree_scans_complete = project
+ .read(cx)
+ .worktrees(cx)
+ .map(|worktree| {
+ let scan_complete = worktree.read(cx).as_local().unwrap().scan_complete();
+ async move {
+ scan_complete.await;
+ }
+ })
+ .collect::<Vec<_>>();
+ let worktree_db_ids = project
+ .read(cx)
+ .worktrees(cx)
+ .map(|worktree| {
+ self.find_or_create_worktree(worktree.read(cx).abs_path().to_path_buf())
+ })
+ .collect::<Vec<_>>();
+
+ let fs = self.fs.clone();
+ let language_registry = self.language_registry.clone();
+ let database_url = self.database_url.clone();
+ let db_update_tx = self.db_update_tx.clone();
+ let parsing_files_tx = self.parsing_files_tx.clone();
+
+ cx.spawn(|this, mut cx| async move {
+ futures::future::join_all(worktree_scans_complete).await;
+
+ let worktree_db_ids = futures::future::join_all(worktree_db_ids).await;
+
+ if let Some(db_directory) = database_url.parent() {
+ fs.create_dir(db_directory).await.log_err();
+ }
+
+ let worktrees = project.read_with(&cx, |project, cx| {
+ project
+ .worktrees(cx)
+ .map(|worktree| worktree.read(cx).snapshot())
+ .collect::<Vec<_>>()
+ });
+
+ let mut worktree_file_times = HashMap::new();
+ let mut db_ids_by_worktree_id = HashMap::new();
+ for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
+ let db_id = db_id?;
+ db_ids_by_worktree_id.insert(worktree.id(), db_id);
+ worktree_file_times.insert(
+ worktree.id(),
+ this.read_with(&cx, |this, _| this.get_file_mtimes(db_id))
+ .await?,
+ );
+ }
+
+ cx.background()
+ .spawn({
+ let db_ids_by_worktree_id = db_ids_by_worktree_id.clone();
+ let db_update_tx = db_update_tx.clone();
+ let language_registry = language_registry.clone();
+ let parsing_files_tx = parsing_files_tx.clone();
+ async move {
+ let t0 = Instant::now();
+ for worktree in worktrees.into_iter() {
+ let mut file_mtimes =
+ worktree_file_times.remove(&worktree.id()).unwrap();
+ for file in worktree.files(false, 0) {
+ let absolute_path = worktree.absolutize(&file.path);
+
+ if let Ok(language) = language_registry
+ .language_for_file(&absolute_path, None)
+ .await
+ {
+ if language
+ .grammar()
+ .and_then(|grammar| grammar.embedding_config.as_ref())
+ .is_none()
+ {
+ continue;
+ }
+
+ let path_buf = file.path.to_path_buf();
+ let stored_mtime = file_mtimes.remove(&file.path.to_path_buf());
+ let already_stored = stored_mtime
+ .map_or(false, |existing_mtime| {
+ existing_mtime == file.mtime
+ });
+
+ if !already_stored {
+ parsing_files_tx
+ .try_send(PendingFile {
+ worktree_db_id: db_ids_by_worktree_id
+ [&worktree.id()],
+ relative_path: path_buf,
+ absolute_path,
+ language,
+ modified_time: file.mtime,
+ })
+ .unwrap();
+ }
+ }
+ }
+ for file in file_mtimes.keys() {
+ db_update_tx
+ .try_send(DbOperation::Delete {
+ worktree_id: db_ids_by_worktree_id[&worktree.id()],
+ path: file.to_owned(),
+ })
+ .unwrap();
+ }
+ }
+ log::info!(
+ "Parsing Worktree Completed in {:?}",
+ t0.elapsed().as_millis()
+ );
+ }
+ })
+ .detach();
+
+ // let mut pending_files: Vec<(PathBuf, ((i64, PathBuf, Arc<Language>, SystemTime), SystemTime))> = vec![];
+ this.update(&mut cx, |this, cx| {
+ // The below is managing for updated on save
+ // Currently each time a file is saved, this code is run, and for all the files that were changed, if the current time is
+ // greater than the previous embedded time by the REINDEXING_DELAY variable, we will send the file off to be indexed.
+ let _subscription = cx.subscribe(&project, |this, project, event, cx| {
+ if let project::Event::WorktreeUpdatedEntries(worktree_id, changes) = event {
+ this.project_entries_changed(project, changes.clone(), cx, worktree_id);
+ }
+ });
+
+ this.projects.insert(
+ project.downgrade(),
+ ProjectState {
+ pending_files: HashMap::new(),
+ worktree_db_ids: db_ids_by_worktree_id.into_iter().collect(),
+ _subscription,
+ },
+ );
+ });
+
+ anyhow::Ok(())
+ })
+ }
+
+ pub fn search(
+ &mut self,
+ project: ModelHandle<Project>,
+ phrase: String,
+ limit: usize,
+ cx: &mut ModelContext<Self>,
+ ) -> Task<Result<Vec<SearchResult>>> {
+ let project_state = if let Some(state) = self.projects.get(&project.downgrade()) {
+ state
+ } else {
+ return Task::ready(Err(anyhow!("project not added")));
+ };
+
+ let worktree_db_ids = project
+ .read(cx)
+ .worktrees(cx)
+ .filter_map(|worktree| {
+ let worktree_id = worktree.read(cx).id();
+ project_state.db_id_for_worktree_id(worktree_id)
+ })
+ .collect::<Vec<_>>();
+
+ let embedding_provider = self.embedding_provider.clone();
+ let database_url = self.database_url.clone();
+ cx.spawn(|this, cx| async move {
+ let documents = cx
+ .background()
+ .spawn(async move {
+ let database = VectorDatabase::new(database_url.to_string_lossy().into())?;
+
+ let phrase_embedding = embedding_provider
+ .embed_batch(vec![&phrase])
+ .await?
+ .into_iter()
+ .next()
+ .unwrap();
+
+ database.top_k_search(&worktree_db_ids, &phrase_embedding, limit)
+ })
+ .await?;
+
+ this.read_with(&cx, |this, _| {
+ let project_state = if let Some(state) = this.projects.get(&project.downgrade()) {
+ state
+ } else {
+ return Err(anyhow!("project not added"));
+ };
+
+ Ok(documents
+ .into_iter()
+ .filter_map(|(worktree_db_id, file_path, offset, name)| {
+ let worktree_id = project_state.worktree_id_for_db_id(worktree_db_id)?;
+ Some(SearchResult {
+ worktree_id,
+ name,
+ offset,
+ file_path,
+ })
+ })
+ .collect())
+ })
+ })
+ }
+
+ fn project_entries_changed(
+ &mut self,
+ project: ModelHandle<Project>,
+ changes: Arc<[(Arc<Path>, ProjectEntryId, PathChange)]>,
+ cx: &mut ModelContext<'_, VectorStore>,
+ worktree_id: &WorktreeId,
+ ) -> Option<()> {
+ let reindexing_delay = settings::get::<VectorStoreSettings>(cx).reindexing_delay_seconds;
+
+ let worktree = project
+ .read(cx)
+ .worktree_for_id(worktree_id.clone(), cx)?
+ .read(cx)
+ .snapshot();
+
+ let worktree_db_id = self
+ .projects
+ .get(&project.downgrade())?
+ .db_id_for_worktree_id(worktree.id())?;
+ let file_mtimes = self.get_file_mtimes(worktree_db_id);
+
+ let language_registry = self.language_registry.clone();
+
+ cx.spawn(|this, mut cx| async move {
+ let file_mtimes = file_mtimes.await.log_err()?;
+
+ for change in changes.into_iter() {
+ let change_path = change.0.clone();
+ let absolute_path = worktree.absolutize(&change_path);
+
+ // Skip if git ignored or symlink
+ if let Some(entry) = worktree.entry_for_id(change.1) {
+ if entry.is_ignored || entry.is_symlink || entry.is_external {
+ continue;
+ }
+ }
+
+ match change.2 {
+ PathChange::Removed => this.update(&mut cx, |this, _| {
+ this.db_update_tx
+ .try_send(DbOperation::Delete {
+ worktree_id: worktree_db_id,
+ path: absolute_path,
+ })
+ .unwrap();
+ }),
+ _ => {
+ if let Ok(language) = language_registry
+ .language_for_file(&change_path.to_path_buf(), None)
+ .await
+ {
+ if language
+ .grammar()
+ .and_then(|grammar| grammar.embedding_config.as_ref())
+ .is_none()
+ {
+ continue;
+ }
+
+ let modified_time =
+ change_path.metadata().log_err()?.modified().log_err()?;
+
+ let existing_time = file_mtimes.get(&change_path.to_path_buf());
+ let already_stored = existing_time
+ .map_or(false, |existing_time| &modified_time != existing_time);
+
+ if !already_stored {
+ this.update(&mut cx, |this, _| {
+ let reindex_time = modified_time
+ + Duration::from_secs(reindexing_delay as u64);
+
+ let project_state =
+ this.projects.get_mut(&project.downgrade())?;
+ project_state.update_pending_files(
+ PendingFile {
+ relative_path: change_path.to_path_buf(),
+ absolute_path,
+ modified_time,
+ worktree_db_id,
+ language: language.clone(),
+ },
+ reindex_time,
+ );
+
+ for file in project_state.get_outstanding_files() {
+ this.parsing_files_tx.try_send(file).unwrap();
+ }
+ Some(())
+ });
+ }
+ }
+ }
+ }
+ }
+
+ Some(())
+ })
+ .detach();
+
+ Some(())
+ }
+}
+
+impl Entity for VectorStore {
+ type Event = ();
+}
@@ -0,0 +1,30 @@
+use anyhow;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use settings::Setting;
+
+#[derive(Deserialize, Debug)]
+pub struct VectorStoreSettings {
+ pub enabled: bool,
+ pub reindexing_delay_seconds: usize,
+}
+
+#[derive(Clone, Default, Serialize, Deserialize, JsonSchema, Debug)]
+pub struct VectorStoreSettingsContent {
+ pub enabled: Option<bool>,
+ pub reindexing_delay_seconds: Option<usize>,
+}
+
+impl Setting for VectorStoreSettings {
+ const KEY: Option<&'static str> = Some("vector_store");
+
+ type FileContent = VectorStoreSettingsContent;
+
+ fn load(
+ default_value: &Self::FileContent,
+ user_values: &[&Self::FileContent],
+ _: &gpui::AppContext,
+ ) -> anyhow::Result<Self> {
+ Self::load_via_json_merge(default_value, user_values)
+ }
+}
@@ -0,0 +1,161 @@
+use crate::{
+ db::dot, embedding::EmbeddingProvider, vector_store_settings::VectorStoreSettings, VectorStore,
+};
+use anyhow::Result;
+use async_trait::async_trait;
+use gpui::{Task, TestAppContext};
+use language::{Language, LanguageConfig, LanguageRegistry};
+use project::{project_settings::ProjectSettings, FakeFs, Project};
+use rand::{rngs::StdRng, Rng};
+use serde_json::json;
+use settings::SettingsStore;
+use std::sync::Arc;
+use unindent::Unindent;
+
+#[gpui::test]
+async fn test_vector_store(cx: &mut TestAppContext) {
+ cx.update(|cx| {
+ cx.set_global(SettingsStore::test(cx));
+ settings::register::<VectorStoreSettings>(cx);
+ settings::register::<ProjectSettings>(cx);
+ });
+
+ let fs = FakeFs::new(cx.background());
+ fs.insert_tree(
+ "/the-root",
+ json!({
+ "src": {
+ "file1.rs": "
+ fn aaa() {
+ println!(\"aaaa!\");
+ }
+
+ fn zzzzzzzzz() {
+ println!(\"SLEEPING\");
+ }
+ ".unindent(),
+ "file2.rs": "
+ fn bbb() {
+ println!(\"bbbb!\");
+ }
+ ".unindent(),
+ }
+ }),
+ )
+ .await;
+
+ let languages = Arc::new(LanguageRegistry::new(Task::ready(())));
+ let rust_language = Arc::new(
+ Language::new(
+ LanguageConfig {
+ name: "Rust".into(),
+ path_suffixes: vec!["rs".into()],
+ ..Default::default()
+ },
+ Some(tree_sitter_rust::language()),
+ )
+ .with_embedding_query(
+ r#"
+ (function_item
+ name: (identifier) @name
+ body: (block)) @item
+ "#,
+ )
+ .unwrap(),
+ );
+ languages.add(rust_language);
+
+ let db_dir = tempdir::TempDir::new("vector-store").unwrap();
+ let db_path = db_dir.path().join("db.sqlite");
+
+ let store = VectorStore::new(
+ fs.clone(),
+ db_path,
+ Arc::new(FakeEmbeddingProvider),
+ languages,
+ cx.to_async(),
+ )
+ .await
+ .unwrap();
+
+ let project = Project::test(fs, ["/the-root".as_ref()], cx).await;
+ let worktree_id = project.read_with(cx, |project, cx| {
+ project.worktrees(cx).next().unwrap().read(cx).id()
+ });
+ store
+ .update(cx, |store, cx| store.add_project(project.clone(), cx))
+ .await
+ .unwrap();
+ cx.foreground().run_until_parked();
+
+ let search_results = store
+ .update(cx, |store, cx| {
+ store.search(project.clone(), "aaaa".to_string(), 5, cx)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(search_results[0].offset, 0);
+ assert_eq!(search_results[0].name, "aaa");
+ assert_eq!(search_results[0].worktree_id, worktree_id);
+}
+
+#[gpui::test]
+fn test_dot_product(mut rng: StdRng) {
+ assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.);
+ assert_eq!(dot(&[2., 0., 0., 0., 0.], &[3., 1., 0., 0., 0.]), 6.);
+
+ for _ in 0..100 {
+ let size = 1536;
+ let mut a = vec![0.; size];
+ let mut b = vec![0.; size];
+ for (a, b) in a.iter_mut().zip(b.iter_mut()) {
+ *a = rng.gen();
+ *b = rng.gen();
+ }
+
+ assert_eq!(
+ round_to_decimals(dot(&a, &b), 1),
+ round_to_decimals(reference_dot(&a, &b), 1)
+ );
+ }
+
+ fn round_to_decimals(n: f32, decimal_places: i32) -> f32 {
+ let factor = (10.0 as f32).powi(decimal_places);
+ (n * factor).round() / factor
+ }
+
+ fn reference_dot(a: &[f32], b: &[f32]) -> f32 {
+ a.iter().zip(b.iter()).map(|(a, b)| a * b).sum()
+ }
+}
+
+struct FakeEmbeddingProvider;
+
+#[async_trait]
+impl EmbeddingProvider for FakeEmbeddingProvider {
+ async fn embed_batch(&self, spans: Vec<&str>) -> Result<Vec<Vec<f32>>> {
+ Ok(spans
+ .iter()
+ .map(|span| {
+ let mut result = vec![1.0; 26];
+ for letter in span.chars() {
+ let letter = letter.to_ascii_lowercase();
+ if letter as u32 >= 'a' as u32 {
+ let ix = (letter as u32) - ('a' as u32);
+ if ix < 26 {
+ result[ix as usize] += 1.0;
+ }
+ }
+ }
+
+ let norm = result.iter().map(|x| x * x).sum::<f32>().sqrt();
+ for x in &mut result {
+ *x /= norm;
+ }
+
+ result
+ })
+ .collect())
+ }
+}
@@ -64,6 +64,7 @@ terminal_view = { path = "../terminal_view" }
theme = { path = "../theme" }
theme_selector = { path = "../theme_selector" }
util = { path = "../util" }
+vector_store = { path = "../vector_store" }
vim = { path = "../vim" }
workspace = { path = "../workspace" }
welcome = { path = "../welcome" }
@@ -170,6 +170,7 @@ fn load_queries(name: &str) -> LanguageQueries {
brackets: load_query(name, "/brackets"),
indents: load_query(name, "/indents"),
outline: load_query(name, "/outline"),
+ embedding: load_query(name, "/embedding"),
injections: load_query(name, "/injections"),
overrides: load_query(name, "/overrides"),
}
@@ -0,0 +1,56 @@
+; (internal_module
+; "namespace" @context
+; name: (_) @name) @item
+
+(enum_declaration
+ "enum" @context
+ name: (_) @name) @item
+
+(function_declaration
+ "async"? @context
+ "function" @context
+ name: (_) @name) @item
+
+(interface_declaration
+ "interface" @context
+ name: (_) @name) @item
+
+; (program
+; (export_statement
+; (lexical_declaration
+; ["let" "const"] @context
+; (variable_declarator
+; name: (_) @name) @item)))
+
+(program
+ (lexical_declaration
+ ["let" "const"] @context
+ (variable_declarator
+ name: (_) @name) @item))
+
+(class_declaration
+ "class" @context
+ name: (_) @name) @item
+
+(method_definition
+ [
+ "get"
+ "set"
+ "async"
+ "*"
+ "readonly"
+ "static"
+ (override_modifier)
+ (accessibility_modifier)
+ ]* @context
+ name: (_) @name) @item
+
+; (public_field_definition
+; [
+; "declare"
+; "readonly"
+; "abstract"
+; "static"
+; (accessibility_modifier)
+; ]* @context
+; name: (_) @name) @item
@@ -0,0 +1,9 @@
+(class_definition
+ "class" @context
+ name: (identifier) @name
+ ) @item
+
+(function_definition
+ "async"? @context
+ "def" @context
+ name: (_) @name) @item
@@ -0,0 +1,36 @@
+(struct_item
+ (visibility_modifier)? @context
+ "struct" @context
+ name: (_) @name) @item
+
+(enum_item
+ (visibility_modifier)? @context
+ "enum" @context
+ name: (_) @name) @item
+
+(impl_item
+ "impl" @context
+ trait: (_)? @name
+ "for"? @context
+ type: (_) @name) @item
+
+(trait_item
+ (visibility_modifier)? @context
+ "trait" @context
+ name: (_) @name) @item
+
+(function_item
+ (visibility_modifier)? @context
+ (function_modifiers)? @context
+ "fn" @context
+ name: (_) @name) @item
+
+(function_signature_item
+ (visibility_modifier)? @context
+ (function_modifiers)? @context
+ "fn" @context
+ name: (_) @name) @item
+
+(macro_definition
+ . "macro_rules!" @context
+ name: (_) @name) @item
@@ -0,0 +1,35 @@
+(enum_declaration
+ "enum" @context
+ name: (_) @name) @item
+
+(function_declaration
+ "async"? @context
+ "function" @context
+ name: (_) @name) @item
+
+(interface_declaration
+ "interface" @context
+ name: (_) @name) @item
+
+(program
+ (lexical_declaration
+ ["let" "const"] @context
+ (variable_declarator
+ name: (_) @name) @item))
+
+(class_declaration
+ "class" @context
+ name: (_) @name) @item
+
+(method_definition
+ [
+ "get"
+ "set"
+ "async"
+ "*"
+ "readonly"
+ "static"
+ (override_modifier)
+ (accessibility_modifier)
+ ]* @context
+ name: (_) @name) @item
@@ -0,0 +1,59 @@
+; (internal_module
+; "namespace" @context
+; name: (_) @name) @item
+
+(enum_declaration
+ "enum" @context
+ name: (_) @name) @item
+
+; (type_alias_declaration
+; "type" @context
+; name: (_) @name) @item
+
+(function_declaration
+ "async"? @context
+ "function" @context
+ name: (_) @name) @item
+
+(interface_declaration
+ "interface" @context
+ name: (_) @name) @item
+
+; (export_statement
+; (lexical_declaration
+; ["let" "const"] @context
+; (variable_declarator
+; name: (_) @name) @item))
+
+(program
+ (lexical_declaration
+ ["let" "const"] @context
+ (variable_declarator
+ name: (_) @name) @item))
+
+(class_declaration
+ "class" @context
+ name: (_) @name) @item
+
+(method_definition
+ [
+ "get"
+ "set"
+ "async"
+ "*"
+ "readonly"
+ "static"
+ (override_modifier)
+ (accessibility_modifier)
+ ]* @context
+ name: (_) @name) @item
+
+; (public_field_definition
+; [
+; "declare"
+; "readonly"
+; "abstract"
+; "static"
+; (accessibility_modifier)
+; ]* @context
+; name: (_) @name) @item
@@ -157,6 +157,7 @@ fn main() {
project_panel::init(cx);
diagnostics::init(cx);
search::init(cx);
+ vector_store::init(fs.clone(), http.clone(), languages.clone(), cx);
vim::init(cx);
terminal_view::init(cx);
copilot::init(http.clone(), node_runtime, cx);