diff --git a/Cargo.lock b/Cargo.lock index 239aa6a302ded4391422e1c2d8752236f4019bb3..b6049e611ed0c72a0fc13d822545dc994d80af4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8508,7 +8508,9 @@ dependencies = [ "theme", "tiktoken-rs 0.5.0", "tree-sitter", + "tree-sitter-javascript", "tree-sitter-rust", + "tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)", "unindent", "util", "workspace", diff --git a/crates/vector_store/Cargo.toml b/crates/vector_store/Cargo.toml index bac9cdedfafc4567f24b7502e0f9ea9e4d0e71e3..6b2e77e904016eb9e27584e8fc4e9ede71134d0a 100644 --- a/crates/vector_store/Cargo.toml +++ b/crates/vector_store/Cargo.toml @@ -44,10 +44,13 @@ project = { path = "../project", features = ["test-support"] } rpc = { path = "../rpc", features = ["test-support"] } workspace = { path = "../workspace", features = ["test-support"] } settings = { path = "../settings", features = ["test-support"]} -tree-sitter-rust = "*" rand.workspace = true unindent.workspace = true tempdir.workspace = true ctor.workspace = true env_logger.workspace = true + +tree-sitter-javascript = "*" +tree-sitter-typescript = "*" +tree-sitter-rust = "*" diff --git a/crates/vector_store/src/vector_store_tests.rs b/crates/vector_store/src/vector_store_tests.rs index de82bc2f482351166d4f57d37c0a82087dbaa662..76465b1aaf95ef98b2305e35dbead3628bc461ed 100644 --- a/crates/vector_store/src/vector_store_tests.rs +++ b/crates/vector_store/src/vector_store_tests.rs @@ -144,7 +144,7 @@ async fn test_vector_store(cx: &mut TestAppContext) { } #[gpui::test] -async fn test_code_context_retrieval() { +async fn test_code_context_retrieval_rust() { let language = rust_lang(); let mut retriever = CodeContextRetriever::new(); @@ -200,6 +200,142 @@ async fn test_code_context_retrieval() { ); } +#[gpui::test] +async fn test_code_context_retrieval_javascript() { + let language = js_lang(); + let mut retriever = CodeContextRetriever::new(); + + let text = " +/* globals importScripts, backend */ +function _authorize() {} + +/** + * Sometimes the frontend build is way faster than backend. + */ +export async function authorizeBank() { + _authorize(pushModal, upgradingAccountId, {}); +} + +export class SettingsPage { + /* This is a test setting */ + constructor(page) { + this.page = page; + } +} + +/* This is a test comment */ +class TestClass {} + +/* Schema for editor_events in Clickhouse. */ +export interface ClickhouseEditorEvent { + installation_id: string + operation: string +} +"; + + let parsed_files = retriever + .parse_file(Path::new("foo.js"), &text, language) + .unwrap(); + + let test_documents = &[ + Document { + name: "function _authorize".into(), + range: text.find("function _authorize").unwrap()..(text.find("}").unwrap() + 1), + content: " + The below code snippet is from file 'foo.js' + + ```javascript + /* globals importScripts, backend */ + function _authorize() {} + ```" + .unindent(), + embedding: vec![], + }, + Document { + name: "async function authorizeBank".into(), + range: text.find("export async").unwrap()..224, + content: " + The below code snippet is from file 'foo.js' + + ```javascript + /** + * Sometimes the frontend build is way faster than backend. + */ + export async function authorizeBank() { + _authorize(pushModal, upgradingAccountId, {}); + } + ```" + .unindent(), + embedding: vec![], + }, + Document { + name: "class SettingsPage".into(), + range: 226..344, + content: " + The below code snippet is from file 'foo.js' + + ```javascript + export class SettingsPage { + /* This is a test setting */ + constructor(page) { + this.page = page; + } + } + ```" + .unindent(), + embedding: vec![], + }, + Document { + name: "constructor".into(), + range: 291..342, + content: " + The below code snippet is from file 'foo.js' + + ```javascript + /* This is a test setting */ + constructor(page) { + this.page = page; + } + ```" + .unindent(), + embedding: vec![], + }, + Document { + name: "class TestClass".into(), + range: 375..393, + content: " + The below code snippet is from file 'foo.js' + + ```javascript + /* This is a test comment */ + class TestClass {} + ```" + .unindent(), + embedding: vec![], + }, + Document { + name: "interface ClickhouseEditorEvent".into(), + range: 441..533, + content: " + The below code snippet is from file 'foo.js' + + ```javascript + /* Schema for editor_events in Clickhouse. */ + export interface ClickhouseEditorEvent { + installation_id: string + operation: string + } + ```" + .unindent(), + embedding: vec![], + }, + ]; + + for idx in 0..test_documents.len() { + assert_eq!(test_documents[idx], parsed_files[idx]); + } +} + #[gpui::test] fn test_dot_product(mut rng: StdRng) { assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.); @@ -271,6 +407,110 @@ impl EmbeddingProvider for FakeEmbeddingProvider { } } +fn js_lang() -> Arc { + Arc::new( + Language::new( + LanguageConfig { + name: "Javascript".into(), + path_suffixes: vec!["js".into()], + ..Default::default() + }, + Some(tree_sitter_typescript::language_tsx()), + ) + .with_embedding_query( + &r#" + + ( + (comment)* @context + . + (export_statement + (function_declaration + "async"? @name + "function" @name + name: (_) @name)) @item + ) + + ( + (comment)* @context + . + (function_declaration + "async"? @name + "function" @name + name: (_) @name) @item + ) + + ( + (comment)* @context + . + (export_statement + (class_declaration + "class" @name + name: (_) @name)) @item + ) + + ( + (comment)* @context + . + (class_declaration + "class" @name + name: (_) @name) @item + ) + + ( + (comment)* @context + . + (method_definition + [ + "get" + "set" + "async" + "*" + "static" + ]* @name + name: (_) @name) @item + ) + + ( + (comment)* @context + . + (export_statement + (interface_declaration + "interface" @name + name: (_) @name)) @item + ) + + ( + (comment)* @context + . + (interface_declaration + "interface" @name + name: (_) @name) @item + ) + + ( + (comment)* @context + . + (export_statement + (enum_declaration + "enum" @name + name: (_) @name)) @item + ) + + ( + (comment)* @context + . + (enum_declaration + "enum" @name + name: (_) @name) @item + ) + + "# + .unindent(), + ) + .unwrap(), + ) +} + fn rust_lang() -> Arc { Arc::new( Language::new( diff --git a/crates/zed/src/languages/javascript/embedding.scm b/crates/zed/src/languages/javascript/embedding.scm index ec6eb5ab1a8be481bc7a9987056ce2d1cb7d2474..a2140400318db95a8d29074402ab2d212561a79b 100644 --- a/crates/zed/src/languages/javascript/embedding.scm +++ b/crates/zed/src/languages/javascript/embedding.scm @@ -1,56 +1,83 @@ -; (internal_module -; "namespace" @context -; name: (_) @name) @item - -(enum_declaration - "enum" @context - name: (_) @name) @item - -(function_declaration - "async"? @context - "function" @context - name: (_) @name) @item - -(interface_declaration - "interface" @context - name: (_) @name) @item - -; (program -; (export_statement -; (lexical_declaration -; ["let" "const"] @context -; (variable_declarator -; name: (_) @name) @item))) - -(program - (lexical_declaration - ["let" "const"] @context - (variable_declarator - name: (_) @name) @item)) - -(class_declaration - "class" @context - name: (_) @name) @item - -(method_definition - [ - "get" - "set" - "async" - "*" - "readonly" - "static" - (override_modifier) - (accessibility_modifier) - ]* @context - name: (_) @name) @item - -; (public_field_definition -; [ -; "declare" -; "readonly" -; "abstract" -; "static" -; (accessibility_modifier) -; ]* @context -; name: (_) @name) @item +( + (comment)* @context + . + (export_statement + (function_declaration + "async"? @name + "function" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (function_declaration + "async"? @name + "function" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (class_declaration + "class" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (class_declaration + "class" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (method_definition + [ + "get" + "set" + "async" + "*" + "static" + ]* @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (interface_declaration + "interface" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (interface_declaration + "interface" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (enum_declaration + "enum" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (enum_declaration + "enum" @name + name: (_) @name) @item + ) diff --git a/crates/zed/src/languages/tsx/embedding.scm b/crates/zed/src/languages/tsx/embedding.scm index 1c47a5a238ff9d944dc321b4eb10b0e56d8a6221..4bb4fea254d0cf86f2fbb9d5c8f657e06238971f 100644 --- a/crates/zed/src/languages/tsx/embedding.scm +++ b/crates/zed/src/languages/tsx/embedding.scm @@ -1,33 +1,29 @@ ( (comment)* @context . - (enum_declaration - "enum" @context - name: (_) @name) @item + (export_statement + (function_declaration + "async"? @name + "function" @name + name: (_) @name)) @item ) ( (comment)* @context . - [ - (export_statement - (function_declaration - "async"? @name - "function" @name - name: (_) @name) - ) @item - (function_declaration - "async"? @name - "function" @name - name: (_) @name) @item - ]) + (function_declaration + "async"? @name + "function" @name + name: (_) @name) @item + ) ( (comment)* @context . - (interface_declaration - "interface" @name - name: (_) @name) @item + (export_statement + (class_declaration + "class" @name + name: (_) @name)) @item ) ( @@ -47,10 +43,57 @@ "set" "async" "*" - "readonly" "static" - (override_modifier) - (accessibility_modifier) ]* @name name: (_) @name) @item ) + +( + (comment)* @context + . + (export_statement + (interface_declaration + "interface" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (interface_declaration + "interface" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (enum_declaration + "enum" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (enum_declaration + "enum" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (type_alias_declaration + "type" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (type_alias_declaration + "type" @name + name: (_) @name) @item) diff --git a/crates/zed/src/languages/typescript/embedding.scm b/crates/zed/src/languages/typescript/embedding.scm index d850f9b82307fc9bd0560c866ca149cffe5a1f5e..4bb4fea254d0cf86f2fbb9d5c8f657e06238971f 100644 --- a/crates/zed/src/languages/typescript/embedding.scm +++ b/crates/zed/src/languages/typescript/embedding.scm @@ -1,33 +1,29 @@ ( (comment)* @context . - (enum_declaration - "enum" @context - name: (_) @name) @item + (export_statement + (function_declaration + "async"? @name + "function" @name + name: (_) @name)) @item ) ( (comment)* @context . - [ - (export_statement - (function_declaration - "async"? @name - "function" @name - name: (_) @name) - ) @item - (function_declaration - "async"? @name - "function" @name - name: (_) @name) @item - ]) + (function_declaration + "async"? @name + "function" @name + name: (_) @name) @item + ) ( (comment)* @context . - (interface_declaration - "interface" @name - name: (_) @name) @item + (export_statement + (class_declaration + "class" @name + name: (_) @name)) @item ) ( @@ -47,10 +43,57 @@ "set" "async" "*" - "readonly" "static" - (override_modifier) - (accessibility_modifier) ]* @name name: (_) @name) @item ) + +( + (comment)* @context + . + (export_statement + (interface_declaration + "interface" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (interface_declaration + "interface" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (enum_declaration + "enum" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (enum_declaration + "enum" @name + name: (_) @name) @item + ) + +( + (comment)* @context + . + (export_statement + (type_alias_declaration + "type" @name + name: (_) @name)) @item + ) + +( + (comment)* @context + . + (type_alias_declaration + "type" @name + name: (_) @name) @item)