From 2c17ae9aa63b8c228a879153c018ed6ade168f24 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 29 Nov 2021 17:38:59 +0100 Subject: [PATCH] Introduce a new `Grammar` struct and allow it to be optional Co-Authored-By: Nathan Sobo --- Cargo.lock | 11 ------ crates/editor/src/display_map.rs | 4 +- crates/editor/src/lib.rs | 8 ++-- crates/language/src/language.rs | 66 ++++++++++++++++++++++---------- crates/language/src/lib.rs | 55 +++++++++++++++----------- crates/language/src/tests.rs | 2 +- crates/project/src/worktree.rs | 2 +- crates/server/src/rpc.rs | 2 +- crates/zed/Cargo.toml | 1 - crates/zed/src/language.rs | 5 +-- crates/zed/src/test.rs | 2 +- 11 files changed, 91 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f6f8c1f92a6150610fc7bd5f578710b479332c6..851d49181ab0c292820dc36fb49ee05f8bbad436 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5132,16 +5132,6 @@ dependencies = [ "regex", ] -[[package]] -name = "tree-sitter-markdown" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c1cfe0396b0e500cc99067bcd2d48720eb08a077e2690194f0c3a3d105f9a0" -dependencies = [ - "cc", - "tree-sitter", -] - [[package]] name = "tree-sitter-rust" version = "0.19.0" @@ -5731,7 +5721,6 @@ dependencies = [ "tiny_http", "toml", "tree-sitter", - "tree-sitter-markdown", "tree-sitter-rust", "unindent", "url", diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 506faa27165eba1b2f70d41c296283d1c543c3e9..da92260ec812dc27e334216cb513ddbc2035a6fe 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -778,7 +778,7 @@ mod tests { path_suffixes: vec![".test".to_string()], ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ) .with_highlights_query( r#" @@ -865,7 +865,7 @@ mod tests { path_suffixes: vec![".test".to_string()], ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ) .with_highlights_query( r#" diff --git a/crates/editor/src/lib.rs b/crates/editor/src/lib.rs index 9339f836db697f2f7ac92d74aac26186948d68fe..da1060b78d0a00859e1bc094bceac74831a1cdc5 100644 --- a/crates/editor/src/lib.rs +++ b/crates/editor/src/lib.rs @@ -5295,7 +5295,7 @@ mod tests { let settings = cx.read(EditorSettings::test); let language = Some(Arc::new(Language::new( LanguageConfig::default(), - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let text = r#" @@ -5452,7 +5452,7 @@ mod tests { ], ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let text = r#" @@ -5551,7 +5551,7 @@ mod tests { line_comment: Some("// ".to_string()), ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let text = " @@ -5649,7 +5649,7 @@ mod tests { ], ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let text = concat!( diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index e80f34116cb87cafd5354d19dfe9283aa0a8aa48..8cb40438ace6045623d4577e7661077e20ed40f5 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -1,12 +1,12 @@ use crate::HighlightMap; -use anyhow::Result; +use anyhow::{anyhow, Result}; use gpui::{executor::Background, AppContext}; use lsp::LanguageServer; use parking_lot::Mutex; use serde::Deserialize; use std::{collections::HashSet, path::Path, str, sync::Arc}; use theme::SyntaxTheme; -use tree_sitter::{Language as Grammar, Query}; +use tree_sitter::{self, Query}; pub use tree_sitter::{Parser, Tree}; #[derive(Default, Deserialize)] @@ -37,7 +37,11 @@ pub struct BracketPair { pub struct Language { pub(crate) config: LanguageConfig, - pub(crate) grammar: Grammar, + pub(crate) grammar: Option>, +} + +pub struct Grammar { + pub(crate) ts_language: tree_sitter::Language, pub(crate) highlights_query: Query, pub(crate) brackets_query: Query, pub(crate) indents_query: Query, @@ -86,29 +90,48 @@ impl LanguageRegistry { } impl Language { - pub fn new(config: LanguageConfig, grammar: Grammar) -> Self { + pub fn new(config: LanguageConfig, ts_language: Option) -> Self { Self { config, - brackets_query: Query::new(grammar, "").unwrap(), - highlights_query: Query::new(grammar, "").unwrap(), - indents_query: Query::new(grammar, "").unwrap(), - grammar, - highlight_map: Default::default(), + grammar: ts_language.map(|ts_language| { + Arc::new(Grammar { + brackets_query: Query::new(ts_language, "").unwrap(), + highlights_query: Query::new(ts_language, "").unwrap(), + indents_query: Query::new(ts_language, "").unwrap(), + ts_language, + highlight_map: Default::default(), + }) + }), } } pub fn with_highlights_query(mut self, source: &str) -> Result { - self.highlights_query = Query::new(self.grammar, source)?; + let grammar = self + .grammar + .as_mut() + .and_then(Arc::get_mut) + .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?; + grammar.highlights_query = Query::new(grammar.ts_language, source)?; Ok(self) } pub fn with_brackets_query(mut self, source: &str) -> Result { - self.brackets_query = Query::new(self.grammar, source)?; + let grammar = self + .grammar + .as_mut() + .and_then(Arc::get_mut) + .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?; + grammar.brackets_query = Query::new(grammar.ts_language, source)?; Ok(self) } pub fn with_indents_query(mut self, source: &str) -> Result { - self.indents_query = Query::new(self.grammar, source)?; + let grammar = self + .grammar + .as_mut() + .and_then(Arc::get_mut) + .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?; + grammar.indents_query = Query::new(grammar.ts_language, source)?; Ok(self) } @@ -156,13 +179,17 @@ impl Language { &self.config.brackets } - pub fn highlight_map(&self) -> HighlightMap { - self.highlight_map.lock().clone() + pub fn set_theme(&self, theme: &SyntaxTheme) { + if let Some(grammar) = self.grammar.as_ref() { + *grammar.highlight_map.lock() = + HighlightMap::new(grammar.highlights_query.capture_names(), theme); + } } +} - pub fn set_theme(&self, theme: &SyntaxTheme) { - *self.highlight_map.lock() = - HighlightMap::new(self.highlights_query.capture_names(), theme); +impl Grammar { + pub fn highlight_map(&self) -> HighlightMap { + self.highlight_map.lock().clone() } } @@ -189,7 +216,6 @@ mod tests { #[test] fn test_select_language() { - let grammar = tree_sitter_rust::language(); let registry = LanguageRegistry { languages: vec![ Arc::new(Language::new( @@ -198,7 +224,7 @@ mod tests { path_suffixes: vec!["rs".to_string()], ..Default::default() }, - grammar, + Some(tree_sitter_rust::language()), )), Arc::new(Language::new( LanguageConfig { @@ -206,7 +232,7 @@ mod tests { path_suffixes: vec!["Makefile".to_string(), "mk".to_string()], ..Default::default() }, - grammar, + Some(tree_sitter_rust::language()), )), ], }; diff --git a/crates/language/src/lib.rs b/crates/language/src/lib.rs index 4c3fa9a51511930d2e467a8ee1a0b9173bbd2981..4d16eb6a248b5d654b6549d043cb733c9f566f12 100644 --- a/crates/language/src/lib.rs +++ b/crates/language/src/lib.rs @@ -6,7 +6,9 @@ mod tests; pub use self::{ highlight_map::{HighlightId, HighlightMap}, - language::{BracketPair, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig}, + language::{ + BracketPair, Grammar, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig, + }, }; use anyhow::{anyhow, Result}; pub use buffer::{Buffer as TextBuffer, Operation as _, *}; @@ -594,13 +596,13 @@ impl Buffer { return false; } - if let Some(language) = self.language.clone() { + if let Some(grammar) = self.grammar().cloned() { let old_tree = self.syntax_tree(); let text = self.as_rope().clone(); let parsed_version = self.version(); let parse_task = cx.background().spawn({ - let language = language.clone(); - async move { Self::parse_text(&text, old_tree, &language) } + let grammar = grammar.clone(); + async move { Self::parse_text(&text, old_tree, &grammar) } }); match cx @@ -616,11 +618,10 @@ impl Buffer { cx.spawn(move |this, mut cx| async move { let new_tree = parse_task.await; this.update(&mut cx, move |this, cx| { - let language_changed = - this.language.as_ref().map_or(true, |curr_language| { - !Arc::ptr_eq(curr_language, &language) - }); - let parse_again = this.version.gt(&parsed_version) || language_changed; + let grammar_changed = this + .grammar() + .map_or(true, |curr_grammar| !Arc::ptr_eq(&grammar, curr_grammar)); + let parse_again = this.version.gt(&parsed_version) || grammar_changed; this.parsing_in_background = false; this.did_finish_parsing(new_tree, parsed_version, cx); @@ -636,11 +637,11 @@ impl Buffer { false } - fn parse_text(text: &Rope, old_tree: Option, language: &Language) -> Tree { + fn parse_text(text: &Rope, old_tree: Option, grammar: &Grammar) -> Tree { PARSER.with(|parser| { let mut parser = parser.borrow_mut(); parser - .set_language(language.grammar) + .set_language(grammar.ts_language) .expect("incompatible grammar"); let mut chunks = text.chunks_in_range(0..text.len()); let tree = parser @@ -1069,15 +1070,15 @@ impl Buffer { &self, range: Range, ) -> Option<(Range, Range)> { - let (lang, tree) = self.language.as_ref().zip(self.syntax_tree())?; - let open_capture_ix = lang.brackets_query.capture_index_for_name("open")?; - let close_capture_ix = lang.brackets_query.capture_index_for_name("close")?; + let (grammar, tree) = self.grammar().zip(self.syntax_tree())?; + let open_capture_ix = grammar.brackets_query.capture_index_for_name("open")?; + let close_capture_ix = grammar.brackets_query.capture_index_for_name("close")?; // Find bracket pairs that *inclusively* contain the given range. let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1; let mut cursor = QueryCursorHandle::new(); let matches = cursor.set_byte_range(range).matches( - &lang.brackets_query, + &grammar.brackets_query, tree.root_node(), TextProvider(self.as_rope()), ); @@ -1342,6 +1343,10 @@ impl Buffer { cx.notify(); } + fn grammar(&self) -> Option<&Arc> { + self.language.as_ref().and_then(|l| l.grammar.as_ref()) + } + pub fn add_selection_set( &mut self, selections: &[Selection], @@ -1550,19 +1555,19 @@ impl Snapshot { row_range: Range, ) -> Option + 'a> { let mut query_cursor = QueryCursorHandle::new(); - if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) { + if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) { let prev_non_blank_row = self.prev_non_blank_row(row_range.start); // Get the "indentation ranges" that intersect this row range. - let indent_capture_ix = language.indents_query.capture_index_for_name("indent"); - let end_capture_ix = language.indents_query.capture_index_for_name("end"); + let indent_capture_ix = grammar.indents_query.capture_index_for_name("indent"); + let end_capture_ix = grammar.indents_query.capture_index_for_name("end"); query_cursor.set_point_range( Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point() ..Point::new(row_range.end, 0).to_ts_point(), ); let mut indentation_ranges = Vec::<(Range, &'static str)>::new(); for mat in query_cursor.matches( - &language.indents_query, + &grammar.indents_query, tree.root_node(), TextProvider(self.as_rope()), ) { @@ -1682,7 +1687,7 @@ impl Snapshot { diagnostic_endpoints .sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start)); - if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) { + if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) { let mut query_cursor = QueryCursorHandle::new(); // TODO - add a Tree-sitter API to remove the need for this. @@ -1690,7 +1695,7 @@ impl Snapshot { std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut()) }; let captures = cursor.set_byte_range(range.clone()).captures( - &language.highlights_query, + &grammar.highlights_query, tree.root_node(), TextProvider(self.text.as_rope()), ); @@ -1698,7 +1703,7 @@ impl Snapshot { captures, next_capture: None, stack: Default::default(), - highlight_map: language.highlight_map(), + highlight_map: grammar.highlight_map(), _query_cursor: query_cursor, theme, }) @@ -1719,6 +1724,12 @@ impl Snapshot { highlights, } } + + fn grammar(&self) -> Option<&Arc> { + self.language + .as_ref() + .and_then(|language| language.grammar.as_ref()) + } } impl Clone for Snapshot { diff --git a/crates/language/src/tests.rs b/crates/language/src/tests.rs index 671755195d1b2a537a29234b9dd52d97921562a9..4a05f108cfdd882858bea3b03aa888edf750cf1c 100644 --- a/crates/language/src/tests.rs +++ b/crates/language/src/tests.rs @@ -970,7 +970,7 @@ fn rust_lang() -> Language { language_server: None, ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ) .with_indents_query( r#" diff --git a/crates/project/src/worktree.rs b/crates/project/src/worktree.rs index 7187310be7419fd64db0ffec43b61b54b18f535c..36df02112c701b520916c08860ac3d01e1b065e7 100644 --- a/crates/project/src/worktree.rs +++ b/crates/project/src/worktree.rs @@ -3671,7 +3671,7 @@ mod tests { language_server: Some(language_server_config), ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let dir = temp_tree(json!({ diff --git a/crates/server/src/rpc.rs b/crates/server/src/rpc.rs index 0e317028c70c54d84860be0c3273c2e813d8ce8c..ab406966c1695b0b747379df500d261311c69c22 100644 --- a/crates/server/src/rpc.rs +++ b/crates/server/src/rpc.rs @@ -1616,7 +1616,7 @@ mod tests { language_server: Some(language_server_config), ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); let lang_registry = Arc::new(lang_registry); diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index 1ff73f1c956d4d20b5a78fb272eb3ca1ffa42e58..df274017115a8e5c48b91f8abaa47f9196c00832 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -85,7 +85,6 @@ time = "0.3" tiny_http = "0.8" toml = "0.5" tree-sitter = "0.19.5" -tree-sitter-markdown = "0.7" tree-sitter-rust = "0.19.0" url = "2.2" diff --git a/crates/zed/src/language.rs b/crates/zed/src/language.rs index e3ca38c6efb86c2d99d734a7634b330d104673f0..c045804e92d92abf35c6e183e6bdb3b858536358 100644 --- a/crates/zed/src/language.rs +++ b/crates/zed/src/language.rs @@ -17,7 +17,7 @@ pub fn build_language_registry() -> LanguageRegistry { fn rust() -> Language { let grammar = tree_sitter_rust::language(); let config = toml::from_slice(&LanguageDir::get("rust/config.toml").unwrap().data).unwrap(); - Language::new(config, grammar) + Language::new(config, Some(grammar)) .with_highlights_query(load_query("rust/highlights.scm").as_ref()) .unwrap() .with_brackets_query(load_query("rust/brackets.scm").as_ref()) @@ -27,9 +27,8 @@ fn rust() -> Language { } fn markdown() -> Language { - let grammar = tree_sitter_markdown::language(); let config = toml::from_slice(&LanguageDir::get("markdown/config.toml").unwrap().data).unwrap(); - Language::new(config, grammar) + Language::new(config, None) } fn load_query(path: &str) -> Cow<'static, str> { diff --git a/crates/zed/src/test.rs b/crates/zed/src/test.rs index 5cb9b5f0e8944e0d1c9cdf0067b625729254eb25..91f1c165ee825d8a31a0a2681bf58bc6cb10d24c 100644 --- a/crates/zed/src/test.rs +++ b/crates/zed/src/test.rs @@ -30,7 +30,7 @@ pub fn test_app_state(cx: &mut MutableAppContext) -> Arc { path_suffixes: vec!["rs".to_string()], ..Default::default() }, - tree_sitter_rust::language(), + Some(tree_sitter_rust::language()), ))); Arc::new(AppState { settings_tx: Arc::new(Mutex::new(settings_tx)),