Max Brunsfeld created
Add language injection supportCargo.lock | 2
Cargo.toml | 2
crates/language/src/buffer.rs | 677 +++----
crates/language/src/language.rs | 174 +
crates/language/src/syntax_map.rs | 1930 ++++++++++++++++++++++
crates/language/src/tests.rs | 5
crates/project/src/project.rs | 1
crates/sum_tree/src/cursor.rs | 6
crates/sum_tree/src/sum_tree.rs | 2
crates/text/src/text.rs | 78
crates/zed/src/languages.rs | 5
crates/zed/src/languages/c/injections.scm | 7
crates/zed/src/languages/cpp/highlights.scm | 4
crates/zed/src/languages/cpp/injections.scm | 7
crates/zed/src/languages/rust/highlights.scm | 20
crates/zed/src/languages/rust/injections.scm | 7
16 files changed, 2,514 insertions(+), 413 deletions(-)
@@ -5850,7 +5850,7 @@ dependencies = [
[[package]]
name = "tree-sitter"
version = "0.20.8"
-source = "git+https://github.com/tree-sitter/tree-sitter?rev=1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0#1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0"
+source = "git+https://github.com/tree-sitter/tree-sitter?rev=366210ae925d7ea0891bc7a0c738f60c77c04d7b#366210ae925d7ea0891bc7a0c738f60c77c04d7b"
dependencies = [
"cc",
"regex",
@@ -4,7 +4,7 @@ default-members = ["crates/zed"]
resolver = "2"
[patch.crates-io]
-tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "1f1b1eb4501ed0a2d195d37f7de15f72aa10acd0" }
+tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "366210ae925d7ea0891bc7a0c738f60c77c04d7b" }
async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" }
# TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457
@@ -6,13 +6,15 @@ pub use crate::{
use crate::{
diagnostic_set::{DiagnosticEntry, DiagnosticGroup},
outline::OutlineItem,
+ syntax_map::{
+ SyntaxMap, SyntaxMapCapture, SyntaxMapCaptures, SyntaxSnapshot, ToTreeSitterPoint,
+ },
CodeLabel, Outline,
};
use anyhow::{anyhow, Result};
use clock::ReplicaId;
use futures::FutureExt as _;
use gpui::{fonts::HighlightStyle, AppContext, Entity, ModelContext, MutableAppContext, Task};
-use lazy_static::lazy_static;
use parking_lot::Mutex;
use settings::Settings;
use similar::{ChangeTag, TextDiff};
@@ -25,7 +27,7 @@ use std::{
future::Future,
iter::{self, Iterator, Peekable},
mem,
- ops::{Deref, DerefMut, Range},
+ ops::{Deref, Range},
path::{Path, PathBuf},
str,
sync::Arc,
@@ -36,7 +38,6 @@ use sum_tree::TreeMap;
use text::operation_queue::OperationQueue;
pub use text::{Buffer as TextBuffer, BufferSnapshot as TextBufferSnapshot, Operation as _, *};
use theme::SyntaxTheme;
-use tree_sitter::{InputEdit, QueryCursor, Tree};
use util::TryFutureExt as _;
#[cfg(any(test, feature = "test-support"))]
@@ -44,10 +45,6 @@ pub use {tree_sitter_rust, tree_sitter_typescript};
pub use lsp::DiagnosticSeverity;
-lazy_static! {
- static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
-}
-
pub struct Buffer {
text: TextBuffer,
file: Option<Arc<dyn File>>,
@@ -60,7 +57,7 @@ pub struct Buffer {
autoindent_requests: Vec<Arc<AutoindentRequest>>,
pending_autoindent: Option<Task<()>>,
sync_parse_timeout: Duration,
- syntax_tree: Mutex<Option<SyntaxTree>>,
+ syntax_map: Mutex<SyntaxMap>,
parsing_in_background: bool,
parse_count: usize,
diagnostics: DiagnosticSet,
@@ -76,7 +73,7 @@ pub struct Buffer {
pub struct BufferSnapshot {
text: text::BufferSnapshot,
- tree: Option<Tree>,
+ pub(crate) syntax: SyntaxSnapshot,
file: Option<Arc<dyn File>>,
diagnostics: DiagnosticSet,
diagnostics_update_count: usize,
@@ -222,14 +219,6 @@ pub trait LocalFile: File {
);
}
-pub(crate) struct QueryCursorHandle(Option<QueryCursor>);
-
-#[derive(Clone)]
-struct SyntaxTree {
- tree: Tree,
- version: clock::Global,
-}
-
#[derive(Clone, Debug)]
pub enum AutoindentMode {
/// Indent each line of inserted text.
@@ -269,14 +258,11 @@ struct IndentSuggestion {
delta: Ordering,
}
-pub(crate) struct TextProvider<'a>(pub(crate) &'a Rope);
-
struct BufferChunkHighlights<'a> {
- captures: tree_sitter::QueryCaptures<'a, 'a, TextProvider<'a>>,
- next_capture: Option<(tree_sitter::QueryMatch<'a, 'a>, usize)>,
+ captures: SyntaxMapCaptures<'a>,
+ next_capture: Option<SyntaxMapCapture<'a>>,
stack: Vec<(usize, HighlightId)>,
- highlight_map: HighlightMap,
- _query_cursor: QueryCursorHandle,
+ highlight_maps: Vec<HighlightMap>,
}
pub struct BufferChunks<'a> {
@@ -433,7 +419,7 @@ impl Buffer {
was_dirty_before_starting_transaction: None,
text: buffer,
file,
- syntax_tree: Mutex::new(None),
+ syntax_map: Mutex::new(SyntaxMap::new()),
parsing_in_background: false,
parse_count: 0,
sync_parse_timeout: Duration::from_millis(1),
@@ -453,9 +439,14 @@ impl Buffer {
}
pub fn snapshot(&self) -> BufferSnapshot {
+ let text = self.text.snapshot();
+ let mut syntax_map = self.syntax_map.lock();
+ syntax_map.interpolate(&text);
+ let syntax = syntax_map.snapshot();
+
BufferSnapshot {
- text: self.text.snapshot(),
- tree: self.syntax_tree(),
+ text,
+ syntax,
file: self.file.clone(),
remote_selections: self.remote_selections.clone(),
diagnostics: self.diagnostics.clone(),
@@ -511,11 +502,17 @@ impl Buffer {
}
pub fn set_language(&mut self, language: Option<Arc<Language>>, cx: &mut ModelContext<Self>) {
- *self.syntax_tree.lock() = None;
+ self.syntax_map.lock().clear();
self.language = language;
self.reparse(cx);
}
+ pub fn set_language_registry(&mut self, language_registry: Arc<LanguageRegistry>) {
+ self.syntax_map
+ .lock()
+ .set_language_registry(language_registry);
+ }
+
pub fn did_save(
&mut self,
version: clock::Global,
@@ -660,15 +657,6 @@ impl Buffer {
self.file_update_count
}
- pub(crate) fn syntax_tree(&self) -> Option<Tree> {
- if let Some(syntax_tree) = self.syntax_tree.lock().as_mut() {
- self.interpolate_tree(syntax_tree);
- Some(syntax_tree.tree.clone())
- } else {
- None
- }
- }
-
#[cfg(any(test, feature = "test-support"))]
pub fn is_parsing(&self) -> bool {
self.parsing_in_background
@@ -679,75 +667,73 @@ impl Buffer {
self.sync_parse_timeout = timeout;
}
- fn reparse(&mut self, cx: &mut ModelContext<Self>) -> bool {
+ fn reparse(&mut self, cx: &mut ModelContext<Self>) {
if self.parsing_in_background {
- return false;
+ return;
}
+ let language = if let Some(language) = self.language.clone() {
+ language
+ } else {
+ return;
+ };
- if let Some(grammar) = self.grammar().cloned() {
- let old_tree = self.syntax_tree();
- let text = self.as_rope().clone();
- let parsed_version = self.version();
- let parse_task = cx.background().spawn({
- let grammar = grammar.clone();
- async move { grammar.parse_text(&text, old_tree) }
- });
-
- match cx
- .background()
- .block_with_timeout(self.sync_parse_timeout, parse_task)
- {
- Ok(new_tree) => {
- self.did_finish_parsing(new_tree, parsed_version, cx);
- return true;
- }
- Err(parse_task) => {
- self.parsing_in_background = true;
- cx.spawn(move |this, mut cx| async move {
- let new_tree = parse_task.await;
- this.update(&mut cx, move |this, cx| {
- let grammar_changed = this
- .grammar()
- .map_or(true, |curr_grammar| !Arc::ptr_eq(&grammar, curr_grammar));
- let parse_again =
- this.version.changed_since(&parsed_version) || grammar_changed;
- this.parsing_in_background = false;
- this.did_finish_parsing(new_tree, parsed_version, cx);
-
- if parse_again && this.reparse(cx) {}
- });
- })
- .detach();
- }
+ let text = self.text_snapshot();
+ let parsed_version = self.version();
+
+ let mut syntax_map = self.syntax_map.lock();
+ syntax_map.interpolate(&text);
+ let language_registry = syntax_map.language_registry();
+ let mut syntax_snapshot = syntax_map.snapshot();
+ let syntax_map_version = syntax_map.parsed_version();
+ drop(syntax_map);
+
+ let parse_task = cx.background().spawn({
+ let language = language.clone();
+ async move {
+ syntax_snapshot.reparse(&syntax_map_version, &text, language_registry, language);
+ syntax_snapshot
}
- }
- false
- }
+ });
- fn interpolate_tree(&self, tree: &mut SyntaxTree) {
- for edit in self.edits_since::<(usize, Point)>(&tree.version) {
- let (bytes, lines) = edit.flatten();
- tree.tree.edit(&InputEdit {
- start_byte: bytes.new.start,
- old_end_byte: bytes.new.start + bytes.old.len(),
- new_end_byte: bytes.new.end,
- start_position: lines.new.start.to_ts_point(),
- old_end_position: (lines.new.start + (lines.old.end - lines.old.start))
- .to_ts_point(),
- new_end_position: lines.new.end.to_ts_point(),
- });
+ match cx
+ .background()
+ .block_with_timeout(self.sync_parse_timeout, parse_task)
+ {
+ Ok(new_syntax_snapshot) => {
+ self.did_finish_parsing(new_syntax_snapshot, parsed_version, cx);
+ return;
+ }
+ Err(parse_task) => {
+ self.parsing_in_background = true;
+ cx.spawn(move |this, mut cx| async move {
+ let new_syntax_map = parse_task.await;
+ this.update(&mut cx, move |this, cx| {
+ let grammar_changed =
+ this.language.as_ref().map_or(true, |current_language| {
+ !Arc::ptr_eq(&language, current_language)
+ });
+ let parse_again =
+ this.version.changed_since(&parsed_version) || grammar_changed;
+ this.did_finish_parsing(new_syntax_map, parsed_version, cx);
+ this.parsing_in_background = false;
+ if parse_again {
+ this.reparse(cx);
+ }
+ });
+ })
+ .detach();
+ }
}
- tree.version = self.version();
}
fn did_finish_parsing(
&mut self,
- tree: Tree,
+ syntax_snapshot: SyntaxSnapshot,
version: clock::Global,
cx: &mut ModelContext<Self>,
) {
self.parse_count += 1;
- *self.syntax_tree.lock() = Some(SyntaxTree { tree, version });
+ self.syntax_map.lock().did_parse(syntax_snapshot, version);
self.request_autoindent(cx);
cx.emit(Event::Reparsed);
cx.notify();
@@ -786,10 +772,7 @@ impl Buffer {
fn compute_autoindents(&self) -> Option<impl Future<Output = BTreeMap<u32, IndentSize>>> {
let max_rows_between_yields = 100;
let snapshot = self.snapshot();
- if snapshot.language.is_none()
- || snapshot.tree.is_none()
- || self.autoindent_requests.is_empty()
- {
+ if snapshot.syntax.is_empty() || self.autoindent_requests.is_empty() {
return None;
}
@@ -1288,10 +1271,6 @@ impl Buffer {
cx.notify();
}
- fn grammar(&self) -> Option<&Arc<Grammar>> {
- self.language.as_ref().and_then(|l| l.grammar.as_ref())
- }
-
pub fn apply_ops<I: IntoIterator<Item = Operation>>(
&mut self,
ops: I,
@@ -1626,38 +1605,38 @@ impl BufferSnapshot {
&self,
row_range: Range<u32>,
) -> Option<impl Iterator<Item = Option<IndentSuggestion>> + '_> {
- let language = self.language.as_ref()?;
- let grammar = language.grammar.as_ref()?;
- let config = &language.config;
+ let config = &self.language.as_ref()?.config;
let prev_non_blank_row = self.prev_non_blank_row(row_range.start);
// Find the suggested indentation ranges based on the syntax tree.
- let indents_query = grammar.indents_query.as_ref()?;
- let mut query_cursor = QueryCursorHandle::new();
- let indent_capture_ix = indents_query.capture_index_for_name("indent");
- let end_capture_ix = indents_query.capture_index_for_name("end");
- query_cursor.set_point_range(
- Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point()
- ..Point::new(row_range.end, 0).to_ts_point(),
- );
+ let start = Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0);
+ let end = Point::new(row_range.end, 0);
+ let range = (start..end).to_offset(&self.text);
+ let mut matches = self.syntax.matches(range, &self.text, |grammar| {
+ Some(&grammar.indents_config.as_ref()?.query)
+ });
+ let indent_configs = matches
+ .grammars()
+ .iter()
+ .map(|grammar| grammar.indents_config.as_ref().unwrap())
+ .collect::<Vec<_>>();
let mut indent_ranges = Vec::<Range<Point>>::new();
- for mat in query_cursor.matches(
- indents_query,
- self.tree.as_ref()?.root_node(),
- TextProvider(self.as_rope()),
- ) {
+ while let Some(mat) = matches.peek() {
let mut start: Option<Point> = None;
let mut end: Option<Point> = None;
+
+ let config = &indent_configs[mat.grammar_index];
for capture in mat.captures {
- if Some(capture.index) == indent_capture_ix {
+ if capture.index == config.indent_capture_ix {
start.get_or_insert(Point::from_ts_point(capture.node.start_position()));
end.get_or_insert(Point::from_ts_point(capture.node.end_position()));
- } else if Some(capture.index) == end_capture_ix {
+ } else if Some(capture.index) == config.end_capture_ix {
end = Some(Point::from_ts_point(capture.node.start_position()));
}
}
+ matches.advance();
if let Some((start, end)) = start.zip(end) {
if start.row == end.row {
continue;
@@ -1789,10 +1768,18 @@ impl BufferSnapshot {
pub fn chunks<T: ToOffset>(&self, range: Range<T>, language_aware: bool) -> BufferChunks {
let range = range.start.to_offset(self)..range.end.to_offset(self);
- let mut tree = None;
+ let mut syntax = None;
let mut diagnostic_endpoints = Vec::new();
if language_aware {
- tree = self.tree.as_ref();
+ let captures = self.syntax.captures(range.clone(), &self.text, |grammar| {
+ grammar.highlights_query.as_ref()
+ });
+ let highlight_maps = captures
+ .grammars()
+ .into_iter()
+ .map(|grammar| grammar.highlight_map())
+ .collect();
+ syntax = Some((captures, highlight_maps));
for entry in self.diagnostics_in_range::<_, usize>(range.clone(), false) {
diagnostic_endpoints.push(DiagnosticEndpoint {
offset: entry.range.start,
@@ -1811,13 +1798,7 @@ impl BufferSnapshot {
.sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start));
}
- BufferChunks::new(
- self.text.as_rope(),
- range,
- tree,
- self.grammar(),
- diagnostic_endpoints,
- )
+ BufferChunks::new(self.text.as_rope(), range, syntax, diagnostic_endpoints)
}
pub fn for_each_line(&self, range: Range<Point>, mut callback: impl FnMut(u32, &str)) {
@@ -1843,12 +1824,6 @@ impl BufferSnapshot {
self.language.as_ref()
}
- fn grammar(&self) -> Option<&Arc<Grammar>> {
- self.language
- .as_ref()
- .and_then(|language| language.grammar.as_ref())
- }
-
pub fn surrounding_word<T: ToOffset>(&self, start: T) -> (Range<usize>, Option<CharKind>) {
let mut start = start.to_offset(self);
let mut end = start;
@@ -1879,61 +1854,71 @@ impl BufferSnapshot {
}
pub fn range_for_syntax_ancestor<T: ToOffset>(&self, range: Range<T>) -> Option<Range<usize>> {
- let tree = self.tree.as_ref()?;
let range = range.start.to_offset(self)..range.end.to_offset(self);
- let mut cursor = tree.root_node().walk();
-
- // Descend to the first leaf that touches the start of the range,
- // and if the range is non-empty, extends beyond the start.
- while cursor.goto_first_child_for_byte(range.start).is_some() {
- if !range.is_empty() && cursor.node().end_byte() == range.start {
- cursor.goto_next_sibling();
+ let mut result: Option<Range<usize>> = None;
+ 'outer: for (_, _, node) in self.syntax.layers_for_range(range.clone(), &self.text) {
+ let mut cursor = node.walk();
+
+ // Descend to the first leaf that touches the start of the range,
+ // and if the range is non-empty, extends beyond the start.
+ while cursor.goto_first_child_for_byte(range.start).is_some() {
+ if !range.is_empty() && cursor.node().end_byte() == range.start {
+ cursor.goto_next_sibling();
+ }
}
- }
- // Ascend to the smallest ancestor that strictly contains the range.
- loop {
- let node_range = cursor.node().byte_range();
- if node_range.start <= range.start
- && node_range.end >= range.end
- && node_range.len() > range.len()
- {
- break;
- }
- if !cursor.goto_parent() {
- break;
+ // Ascend to the smallest ancestor that strictly contains the range.
+ loop {
+ let node_range = cursor.node().byte_range();
+ if node_range.start <= range.start
+ && node_range.end >= range.end
+ && node_range.len() > range.len()
+ {
+ break;
+ }
+ if !cursor.goto_parent() {
+ continue 'outer;
+ }
}
- }
- let left_node = cursor.node();
+ let left_node = cursor.node();
+ let mut layer_result = left_node.byte_range();
- // For an empty range, try to find another node immediately to the right of the range.
- if left_node.end_byte() == range.start {
- let mut right_node = None;
- while !cursor.goto_next_sibling() {
- if !cursor.goto_parent() {
- break;
+ // For an empty range, try to find another node immediately to the right of the range.
+ if left_node.end_byte() == range.start {
+ let mut right_node = None;
+ while !cursor.goto_next_sibling() {
+ if !cursor.goto_parent() {
+ break;
+ }
}
- }
- while cursor.node().start_byte() == range.start {
- right_node = Some(cursor.node());
- if !cursor.goto_first_child() {
- break;
+ while cursor.node().start_byte() == range.start {
+ right_node = Some(cursor.node());
+ if !cursor.goto_first_child() {
+ break;
+ }
+ }
+
+ // If there is a candidate node on both sides of the (empty) range, then
+ // decide between the two by favoring a named node over an anonymous token.
+ // If both nodes are the same in that regard, favor the right one.
+ if let Some(right_node) = right_node {
+ if right_node.is_named() || !left_node.is_named() {
+ layer_result = right_node.byte_range();
+ }
}
}
- // If there is a candidate node on both sides of the (empty) range, then
- // decide between the two by favoring a named node over an anonymous token.
- // If both nodes are the same in that regard, favor the right one.
- if let Some(right_node) = right_node {
- if right_node.is_named() || !left_node.is_named() {
- return Some(right_node.byte_range());
+ if let Some(previous_result) = &result {
+ if previous_result.len() < layer_result.len() {
+ continue;
}
}
+ result = Some(layer_result);
}
- Some(left_node.byte_range())
+ result
}
pub fn outline(&self, theme: Option<&SyntaxTheme>) -> Option<Outline<Anchor>> {
@@ -1947,8 +1932,10 @@ impl BufferSnapshot {
theme: Option<&SyntaxTheme>,
) -> Option<Vec<OutlineItem<Anchor>>> {
let position = position.to_offset(self);
- let mut items =
- self.outline_items_containing(position.saturating_sub(1)..position + 1, theme)?;
+ let mut items = self.outline_items_containing(
+ position.saturating_sub(1)..self.len().min(position + 1),
+ theme,
+ )?;
let mut prev_depth = None;
items.retain(|item| {
let result = prev_depth.map_or(true, |prev_depth| item.depth > prev_depth);
@@ -1963,109 +1950,107 @@ impl BufferSnapshot {
range: Range<usize>,
theme: Option<&SyntaxTheme>,
) -> Option<Vec<OutlineItem<Anchor>>> {
- let tree = self.tree.as_ref()?;
- let grammar = self
- .language
- .as_ref()
- .and_then(|language| language.grammar.as_ref())?;
-
- let outline_query = grammar.outline_query.as_ref()?;
- let mut cursor = QueryCursorHandle::new();
- cursor.set_byte_range(range.clone());
- let matches = cursor.matches(
- outline_query,
- tree.root_node(),
- TextProvider(self.as_rope()),
- );
+ let mut matches = self.syntax.matches(range.clone(), &self.text, |grammar| {
+ grammar.outline_config.as_ref().map(|c| &c.query)
+ });
+ let configs = matches
+ .grammars()
+ .iter()
+ .map(|g| g.outline_config.as_ref().unwrap())
+ .collect::<Vec<_>>();
let mut chunks = self.chunks(0..self.len(), true);
+ let mut stack = Vec::<Range<usize>>::new();
+ let mut items = Vec::new();
+ while let Some(mat) = matches.peek() {
+ let config = &configs[mat.grammar_index];
+ let item_node = mat.captures.iter().find_map(|cap| {
+ if cap.index == config.item_capture_ix {
+ Some(cap.node)
+ } else {
+ None
+ }
+ })?;
- let item_capture_ix = outline_query.capture_index_for_name("item")?;
- let name_capture_ix = outline_query.capture_index_for_name("name")?;
- let context_capture_ix = outline_query
- .capture_index_for_name("context")
- .unwrap_or(u32::MAX);
+ let item_range = item_node.byte_range();
+ if item_range.end < range.start || item_range.start > range.end {
+ matches.advance();
+ continue;
+ }
- let mut stack = Vec::<Range<usize>>::new();
- let items = matches
- .filter_map(|mat| {
- let item_node = mat.nodes_for_capture_index(item_capture_ix).next()?;
- let item_range = item_node.start_byte()..item_node.end_byte();
- if item_range.end < range.start || item_range.start > range.end {
- return None;
+ // TODO - move later, after processing captures
+
+ let mut text = String::new();
+ let mut name_ranges = Vec::new();
+ let mut highlight_ranges = Vec::new();
+ for capture in mat.captures {
+ let node_is_name;
+ if capture.index == config.name_capture_ix {
+ node_is_name = true;
+ } else if Some(capture.index) == config.context_capture_ix {
+ node_is_name = false;
+ } else {
+ continue;
}
- let mut text = String::new();
- let mut name_ranges = Vec::new();
- let mut highlight_ranges = Vec::new();
-
- for capture in mat.captures {
- let node_is_name;
- if capture.index == name_capture_ix {
- node_is_name = true;
- } else if capture.index == context_capture_ix {
- node_is_name = false;
- } else {
- continue;
- }
- let range = capture.node.start_byte()..capture.node.end_byte();
- if !text.is_empty() {
- text.push(' ');
+ let range = capture.node.start_byte()..capture.node.end_byte();
+ if !text.is_empty() {
+ text.push(' ');
+ }
+ if node_is_name {
+ let mut start = text.len();
+ let end = start + range.len();
+
+ // When multiple names are captured, then the matcheable text
+ // includes the whitespace in between the names.
+ if !name_ranges.is_empty() {
+ start -= 1;
}
- if node_is_name {
- let mut start = text.len();
- let end = start + range.len();
-
- // When multiple names are captured, then the matcheable text
- // includes the whitespace in between the names.
- if !name_ranges.is_empty() {
- start -= 1;
- }
- name_ranges.push(start..end);
- }
+ name_ranges.push(start..end);
+ }
- let mut offset = range.start;
- chunks.seek(offset);
- for mut chunk in chunks.by_ref() {
- if chunk.text.len() > range.end - offset {
- chunk.text = &chunk.text[0..(range.end - offset)];
- offset = range.end;
- } else {
- offset += chunk.text.len();
- }
- let style = chunk
- .syntax_highlight_id
- .zip(theme)
- .and_then(|(highlight, theme)| highlight.style(theme));
- if let Some(style) = style {
- let start = text.len();
- let end = start + chunk.text.len();
- highlight_ranges.push((start..end, style));
- }
- text.push_str(chunk.text);
- if offset >= range.end {
- break;
- }
+ let mut offset = range.start;
+ chunks.seek(offset);
+ for mut chunk in chunks.by_ref() {
+ if chunk.text.len() > range.end - offset {
+ chunk.text = &chunk.text[0..(range.end - offset)];
+ offset = range.end;
+ } else {
+ offset += chunk.text.len();
+ }
+ let style = chunk
+ .syntax_highlight_id
+ .zip(theme)
+ .and_then(|(highlight, theme)| highlight.style(theme));
+ if let Some(style) = style {
+ let start = text.len();
+ let end = start + chunk.text.len();
+ highlight_ranges.push((start..end, style));
+ }
+ text.push_str(chunk.text);
+ if offset >= range.end {
+ break;
}
}
+ }
- while stack.last().map_or(false, |prev_range| {
- prev_range.start > item_range.start || prev_range.end < item_range.end
- }) {
- stack.pop();
- }
- stack.push(item_range.clone());
-
- Some(OutlineItem {
- depth: stack.len() - 1,
- range: self.anchor_after(item_range.start)..self.anchor_before(item_range.end),
- text,
- highlight_ranges,
- name_ranges,
- })
+ matches.advance();
+ while stack.last().map_or(false, |prev_range| {
+ prev_range.start > item_range.start || prev_range.end < item_range.end
+ }) {
+ stack.pop();
+ }
+ stack.push(item_range.clone());
+
+ items.push(OutlineItem {
+ depth: stack.len() - 1,
+ range: self.anchor_after(item_range.start)..self.anchor_before(item_range.end),
+ text,
+ highlight_ranges,
+ name_ranges,
})
- .collect::<Vec<_>>();
+ }
Some(items)
}
@@ -2073,28 +2058,48 @@ impl BufferSnapshot {
&self,
range: Range<T>,
) -> Option<(Range<usize>, Range<usize>)> {
- let (grammar, tree) = self.grammar().zip(self.tree.as_ref())?;
- let brackets_query = grammar.brackets_query.as_ref()?;
- let open_capture_ix = brackets_query.capture_index_for_name("open")?;
- let close_capture_ix = brackets_query.capture_index_for_name("close")?;
-
// Find bracket pairs that *inclusively* contain the given range.
let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1;
- let mut cursor = QueryCursorHandle::new();
- let matches = cursor.set_byte_range(range).matches(
- brackets_query,
- tree.root_node(),
- TextProvider(self.as_rope()),
- );
+ let mut matches = self.syntax.matches(range, &self.text, |grammar| {
+ grammar.brackets_config.as_ref().map(|c| &c.query)
+ });
+ let configs = matches
+ .grammars()
+ .iter()
+ .map(|grammar| grammar.brackets_config.as_ref().unwrap())
+ .collect::<Vec<_>>();
// Get the ranges of the innermost pair of brackets.
- matches
- .filter_map(|mat| {
- let open = mat.nodes_for_capture_index(open_capture_ix).next()?;
- let close = mat.nodes_for_capture_index(close_capture_ix).next()?;
- Some((open.byte_range(), close.byte_range()))
- })
- .min_by_key(|(open_range, close_range)| close_range.end - open_range.start)
+ let mut result: Option<(Range<usize>, Range<usize>)> = None;
+ while let Some(mat) = matches.peek() {
+ let mut open = None;
+ let mut close = None;
+ let config = &configs[mat.grammar_index];
+ for capture in mat.captures {
+ if capture.index == config.open_capture_ix {
+ open = Some(capture.node.byte_range());
+ } else if capture.index == config.close_capture_ix {
+ close = Some(capture.node.byte_range());
+ }
+ }
+
+ matches.advance();
+
+ if let Some((open, close)) = open.zip(close) {
+ let len = close.end - open.start;
+
+ if let Some((existing_open, existing_close)) = &result {
+ let existing_len = existing_close.end - existing_open.start;
+ if len > existing_len {
+ continue;
+ }
+ }
+
+ result = Some((open, close));
+ }
+ }
+
+ result
}
#[allow(clippy::type_complexity)]
@@ -2206,7 +2211,7 @@ impl Clone for BufferSnapshot {
fn clone(&self) -> Self {
Self {
text: self.text.clone(),
- tree: self.tree.clone(),
+ syntax: self.syntax.clone(),
file: self.file.clone(),
remote_selections: self.remote_selections.clone(),
diagnostics: self.diagnostics.clone(),
@@ -2227,56 +2232,23 @@ impl Deref for BufferSnapshot {
}
}
-impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
- type I = ByteChunks<'a>;
-
- fn text(&mut self, node: tree_sitter::Node) -> Self::I {
- ByteChunks(self.0.chunks_in_range(node.byte_range()))
- }
-}
-
-pub(crate) struct ByteChunks<'a>(rope::Chunks<'a>);
-
-impl<'a> Iterator for ByteChunks<'a> {
- type Item = &'a [u8];
-
- fn next(&mut self) -> Option<Self::Item> {
- self.0.next().map(str::as_bytes)
- }
-}
-
unsafe impl<'a> Send for BufferChunks<'a> {}
impl<'a> BufferChunks<'a> {
pub(crate) fn new(
text: &'a Rope,
range: Range<usize>,
- tree: Option<&'a Tree>,
- grammar: Option<&'a Arc<Grammar>>,
+ syntax: Option<(SyntaxMapCaptures<'a>, Vec<HighlightMap>)>,
diagnostic_endpoints: Vec<DiagnosticEndpoint>,
) -> Self {
let mut highlights = None;
- if let Some((grammar, tree)) = grammar.zip(tree) {
- if let Some(highlights_query) = grammar.highlights_query.as_ref() {
- let mut query_cursor = QueryCursorHandle::new();
-
- // TODO - add a Tree-sitter API to remove the need for this.
- let cursor = unsafe {
- std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
- };
- let captures = cursor.set_byte_range(range.clone()).captures(
- highlights_query,
- tree.root_node(),
- TextProvider(text),
- );
- highlights = Some(BufferChunkHighlights {
- captures,
- next_capture: None,
- stack: Default::default(),
- highlight_map: grammar.highlight_map(),
- _query_cursor: query_cursor,
- })
- }
+ if let Some((captures, highlight_maps)) = syntax {
+ highlights = Some(BufferChunkHighlights {
+ captures,
+ next_capture: None,
+ stack: Default::default(),
+ highlight_maps,
+ })
}
let diagnostic_endpoints = diagnostic_endpoints.into_iter().peekable();
@@ -2302,14 +2274,13 @@ impl<'a> BufferChunks<'a> {
highlights
.stack
.retain(|(end_offset, _)| *end_offset > offset);
- if let Some((mat, capture_ix)) = &highlights.next_capture {
- let capture = mat.captures[*capture_ix as usize];
+ if let Some(capture) = &highlights.next_capture {
if offset >= capture.node.start_byte() {
let next_capture_end = capture.node.end_byte();
if offset < next_capture_end {
highlights.stack.push((
next_capture_end,
- highlights.highlight_map.get(capture.index),
+ highlights.highlight_maps[capture.grammar_index].get(capture.index),
));
}
highlights.next_capture.take();
@@ -2385,13 +2356,13 @@ impl<'a> Iterator for BufferChunks<'a> {
highlights.next_capture = highlights.captures.next();
}
- while let Some((mat, capture_ix)) = highlights.next_capture.as_ref() {
- let capture = mat.captures[*capture_ix as usize];
+ while let Some(capture) = highlights.next_capture.as_ref() {
if self.range.start < capture.node.start_byte() {
next_capture_start = capture.node.start_byte();
break;
} else {
- let highlight_id = highlights.highlight_map.get(capture.index);
+ let highlight_id =
+ highlights.highlight_maps[capture.grammar_index].get(capture.index);
highlights
.stack
.push((capture.node.end_byte(), highlight_id));
@@ -2443,52 +2414,6 @@ impl<'a> Iterator for BufferChunks<'a> {
}
}
-impl QueryCursorHandle {
- pub(crate) fn new() -> Self {
- let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
- cursor.set_match_limit(64);
- QueryCursorHandle(Some(cursor))
- }
-}
-
-impl Deref for QueryCursorHandle {
- type Target = QueryCursor;
-
- fn deref(&self) -> &Self::Target {
- self.0.as_ref().unwrap()
- }
-}
-
-impl DerefMut for QueryCursorHandle {
- fn deref_mut(&mut self) -> &mut Self::Target {
- self.0.as_mut().unwrap()
- }
-}
-
-impl Drop for QueryCursorHandle {
- fn drop(&mut self) {
- let mut cursor = self.0.take().unwrap();
- cursor.set_byte_range(0..usize::MAX);
- cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
- QUERY_CURSORS.lock().push(cursor)
- }
-}
-
-trait ToTreeSitterPoint {
- fn to_ts_point(self) -> tree_sitter::Point;
- fn from_ts_point(point: tree_sitter::Point) -> Self;
-}
-
-impl ToTreeSitterPoint for Point {
- fn to_ts_point(self) -> tree_sitter::Point {
- tree_sitter::Point::new(self.row as usize, self.column as usize)
- }
-
- fn from_ts_point(point: tree_sitter::Point) -> Self {
- Point::new(point.row as u32, point.column as u32)
- }
-}
-
impl operation_queue::Operation for Operation {
fn lamport_timestamp(&self) -> clock::Lamport {
match self {
@@ -3,6 +3,7 @@ mod diagnostic_set;
mod highlight_map;
mod outline;
pub mod proto;
+mod syntax_map;
#[cfg(test)]
mod tests;
@@ -29,8 +30,12 @@ use std::{
ops::Range,
path::{Path, PathBuf},
str,
- sync::Arc,
+ sync::{
+ atomic::{AtomicUsize, Ordering::SeqCst},
+ Arc,
+ },
};
+use syntax_map::SyntaxSnapshot;
use theme::{SyntaxTheme, Theme};
use tree_sitter::{self, Query};
use util::ResultExt;
@@ -49,6 +54,7 @@ thread_local! {
}
lazy_static! {
+ pub static ref NEXT_GRAMMAR_ID: AtomicUsize = Default::default();
pub static ref PLAIN_TEXT: Arc<Language> = Arc::new(Language::new(
LanguageConfig {
name: "Plain Text".into(),
@@ -285,14 +291,42 @@ pub struct Language {
}
pub struct Grammar {
+ id: usize,
pub(crate) ts_language: tree_sitter::Language,
pub(crate) highlights_query: Option<Query>,
- pub(crate) brackets_query: Option<Query>,
- pub(crate) indents_query: Option<Query>,
- pub(crate) outline_query: Option<Query>,
+ pub(crate) brackets_config: Option<BracketConfig>,
+ pub(crate) indents_config: Option<IndentConfig>,
+ pub(crate) outline_config: Option<OutlineConfig>,
+ pub(crate) injection_config: Option<InjectionConfig>,
pub(crate) highlight_map: Mutex<HighlightMap>,
}
+struct IndentConfig {
+ query: Query,
+ indent_capture_ix: u32,
+ end_capture_ix: Option<u32>,
+}
+
+struct OutlineConfig {
+ query: Query,
+ item_capture_ix: u32,
+ name_capture_ix: u32,
+ context_capture_ix: Option<u32>,
+}
+
+struct InjectionConfig {
+ query: Query,
+ content_capture_ix: u32,
+ language_capture_ix: Option<u32>,
+ languages_by_pattern_ix: Vec<Option<Box<str>>>,
+}
+
+struct BracketConfig {
+ query: Query,
+ open_capture_ix: u32,
+ close_capture_ix: u32,
+}
+
#[derive(Clone)]
pub enum LanguageServerBinaryStatus {
CheckingForUpdate,
@@ -490,6 +524,13 @@ impl LanguageRegistry {
}
}
+#[cfg(any(test, feature = "test-support"))]
+impl Default for LanguageRegistry {
+ fn default() -> Self {
+ Self::test()
+ }
+}
+
async fn get_server_binary_path(
adapter: Arc<CachedLspAdapter>,
language: Arc<Language>,
@@ -567,10 +608,12 @@ impl Language {
config,
grammar: ts_language.map(|ts_language| {
Arc::new(Grammar {
+ id: NEXT_GRAMMAR_ID.fetch_add(1, SeqCst),
highlights_query: None,
- brackets_query: None,
- indents_query: None,
- outline_query: None,
+ brackets_config: None,
+ outline_config: None,
+ indents_config: None,
+ injection_config: None,
ts_language,
highlight_map: Default::default(),
})
@@ -594,19 +637,104 @@ impl Language {
pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
- grammar.brackets_query = Some(Query::new(grammar.ts_language, source)?);
+ let query = Query::new(grammar.ts_language, source)?;
+ let mut open_capture_ix = None;
+ let mut close_capture_ix = None;
+ get_capture_indices(
+ &query,
+ &mut [
+ ("open", &mut open_capture_ix),
+ ("close", &mut close_capture_ix),
+ ],
+ );
+ if let Some((open_capture_ix, close_capture_ix)) = open_capture_ix.zip(close_capture_ix) {
+ grammar.brackets_config = Some(BracketConfig {
+ query,
+ open_capture_ix,
+ close_capture_ix,
+ });
+ }
Ok(self)
}
pub fn with_indents_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
- grammar.indents_query = Some(Query::new(grammar.ts_language, source)?);
+ let query = Query::new(grammar.ts_language, source)?;
+ let mut indent_capture_ix = None;
+ let mut end_capture_ix = None;
+ get_capture_indices(
+ &query,
+ &mut [
+ ("indent", &mut indent_capture_ix),
+ ("end", &mut end_capture_ix),
+ ],
+ );
+ if let Some(indent_capture_ix) = indent_capture_ix {
+ grammar.indents_config = Some(IndentConfig {
+ query,
+ indent_capture_ix,
+ end_capture_ix,
+ });
+ }
Ok(self)
}
pub fn with_outline_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
- grammar.outline_query = Some(Query::new(grammar.ts_language, source)?);
+ let query = Query::new(grammar.ts_language, source)?;
+ let mut item_capture_ix = None;
+ let mut name_capture_ix = None;
+ let mut context_capture_ix = None;
+ get_capture_indices(
+ &query,
+ &mut [
+ ("item", &mut item_capture_ix),
+ ("name", &mut name_capture_ix),
+ ("context", &mut context_capture_ix),
+ ],
+ );
+ if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
+ grammar.outline_config = Some(OutlineConfig {
+ query,
+ item_capture_ix,
+ name_capture_ix,
+ context_capture_ix,
+ });
+ }
+ Ok(self)
+ }
+
+ pub fn with_injection_query(mut self, source: &str) -> Result<Self> {
+ let grammar = self.grammar_mut();
+ let query = Query::new(grammar.ts_language, source)?;
+ let mut language_capture_ix = None;
+ let mut content_capture_ix = None;
+ get_capture_indices(
+ &query,
+ &mut [
+ ("language", &mut language_capture_ix),
+ ("content", &mut content_capture_ix),
+ ],
+ );
+ let languages_by_pattern_ix = (0..query.pattern_count())
+ .map(|ix| {
+ query.property_settings(ix).iter().find_map(|setting| {
+ if setting.key.as_ref() == "language" {
+ return setting.value.clone();
+ } else {
+ None
+ }
+ })
+ })
+ .collect();
+ if let Some(content_capture_ix) = content_capture_ix {
+ grammar.injection_config = Some(InjectionConfig {
+ query,
+ language_capture_ix,
+ content_capture_ix,
+ languages_by_pattern_ix,
+ });
+ }
Ok(self)
}
@@ -685,9 +813,16 @@ impl Language {
let mut result = Vec::new();
if let Some(grammar) = &self.grammar {
let tree = grammar.parse_text(text, None);
+ let captures = SyntaxSnapshot::single_tree_captures(
+ range.clone(),
+ text,
+ &tree,
+ grammar,
+ |grammar| grammar.highlights_query.as_ref(),
+ );
+ let highlight_maps = vec![grammar.highlight_map()];
let mut offset = 0;
- for chunk in BufferChunks::new(text, range, Some(&tree), self.grammar.as_ref(), vec![])
- {
+ for chunk in BufferChunks::new(text, range, Some((captures, highlight_maps)), vec![]) {
let end_offset = offset + chunk.text.len();
if let Some(highlight_id) = chunk.syntax_highlight_id {
if !highlight_id.is_default() {
@@ -727,6 +862,10 @@ impl Language {
}
impl Grammar {
+ pub fn id(&self) -> usize {
+ self.id
+ }
+
fn parse_text(&self, text: &Rope, old_tree: Option<Tree>) -> Tree {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
@@ -826,6 +965,17 @@ impl LspAdapter for Arc<FakeLspAdapter> {
}
}
+fn get_capture_indices(query: &Query, captures: &mut [(&str, &mut Option<u32>)]) {
+ for (ix, name) in query.capture_names().iter().enumerate() {
+ for (capture_name, index) in captures.iter_mut() {
+ if capture_name == name {
+ **index = Some(ix as u32);
+ break;
+ }
+ }
+ }
+}
+
pub fn point_to_lsp(point: PointUtf16) -> lsp::Position {
lsp::Position::new(point.row, point.column)
}
@@ -0,0 +1,1930 @@
+use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
+use lazy_static::lazy_static;
+use parking_lot::Mutex;
+use std::{
+ borrow::Cow,
+ cell::RefCell,
+ cmp::{Ordering, Reverse},
+ collections::BinaryHeap,
+ ops::{Deref, DerefMut, Range},
+ sync::Arc,
+};
+use sum_tree::{Bias, SeekTarget, SumTree};
+use text::{rope, Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
+use tree_sitter::{
+ Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
+};
+
+thread_local! {
+ static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
+}
+
+lazy_static! {
+ static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
+}
+
+#[derive(Default)]
+pub struct SyntaxMap {
+ parsed_version: clock::Global,
+ interpolated_version: clock::Global,
+ snapshot: SyntaxSnapshot,
+ language_registry: Option<Arc<LanguageRegistry>>,
+}
+
+#[derive(Clone, Default)]
+pub struct SyntaxSnapshot {
+ layers: SumTree<SyntaxLayer>,
+}
+
+#[derive(Default)]
+pub struct SyntaxMapCaptures<'a> {
+ layers: Vec<SyntaxMapCapturesLayer<'a>>,
+ active_layer_count: usize,
+ grammars: Vec<&'a Grammar>,
+}
+
+#[derive(Default)]
+pub struct SyntaxMapMatches<'a> {
+ layers: Vec<SyntaxMapMatchesLayer<'a>>,
+ active_layer_count: usize,
+ grammars: Vec<&'a Grammar>,
+}
+
+#[derive(Debug)]
+pub struct SyntaxMapCapture<'a> {
+ pub depth: usize,
+ pub node: Node<'a>,
+ pub index: u32,
+ pub grammar_index: usize,
+}
+
+#[derive(Debug)]
+pub struct SyntaxMapMatch<'a> {
+ pub depth: usize,
+ pub pattern_index: usize,
+ pub captures: &'a [QueryCapture<'a>],
+ pub grammar_index: usize,
+}
+
+struct SyntaxMapCapturesLayer<'a> {
+ depth: usize,
+ captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
+ next_capture: Option<QueryCapture<'a>>,
+ grammar_index: usize,
+ _query_cursor: QueryCursorHandle,
+}
+
+struct SyntaxMapMatchesLayer<'a> {
+ depth: usize,
+ next_pattern_index: usize,
+ next_captures: Vec<QueryCapture<'a>>,
+ has_next: bool,
+ matches: QueryMatches<'a, 'a, TextProvider<'a>>,
+ grammar_index: usize,
+ _query_cursor: QueryCursorHandle,
+}
+
+#[derive(Clone)]
+struct SyntaxLayer {
+ depth: usize,
+ range: Range<Anchor>,
+ tree: tree_sitter::Tree,
+ language: Arc<Language>,
+}
+
+#[derive(Debug, Clone)]
+struct SyntaxLayerSummary {
+ min_depth: usize,
+ max_depth: usize,
+ range: Range<Anchor>,
+ last_layer_range: Range<Anchor>,
+}
+
+#[derive(Clone, Debug)]
+struct DepthAndRange(usize, Range<Anchor>);
+
+#[derive(Clone, Debug)]
+struct DepthAndMaxPosition(usize, Anchor);
+
+#[derive(Clone, Debug)]
+struct DepthAndRangeOrMaxPosition(DepthAndRange, DepthAndMaxPosition);
+
+struct ReparseStep {
+ depth: usize,
+ language: Arc<Language>,
+ ranges: Vec<tree_sitter::Range>,
+ range: Range<Anchor>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ChangedRegion {
+ depth: usize,
+ range: Range<Anchor>,
+}
+
+#[derive(Default)]
+struct ChangeRegionSet(Vec<ChangedRegion>);
+
+struct TextProvider<'a>(&'a Rope);
+
+struct ByteChunks<'a>(rope::Chunks<'a>);
+
+struct QueryCursorHandle(Option<QueryCursor>);
+
+impl SyntaxMap {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
+ self.language_registry = Some(registry);
+ }
+
+ pub fn snapshot(&self) -> SyntaxSnapshot {
+ self.snapshot.clone()
+ }
+
+ pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
+ self.language_registry.clone()
+ }
+
+ pub fn parsed_version(&self) -> clock::Global {
+ self.parsed_version.clone()
+ }
+
+ pub fn interpolate(&mut self, text: &BufferSnapshot) {
+ self.snapshot.interpolate(&self.interpolated_version, text);
+ self.interpolated_version = text.version.clone();
+ }
+
+ #[cfg(test)]
+ pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
+ self.snapshot.reparse(
+ &self.parsed_version,
+ text,
+ self.language_registry.clone(),
+ language,
+ );
+ self.parsed_version = text.version.clone();
+ self.interpolated_version = text.version.clone();
+ }
+
+ pub fn did_parse(&mut self, snapshot: SyntaxSnapshot, version: clock::Global) {
+ self.interpolated_version = version.clone();
+ self.parsed_version = version;
+ self.snapshot = snapshot;
+ }
+
+ pub fn clear(&mut self) {
+ self.snapshot = SyntaxSnapshot::default();
+ }
+}
+
+impl SyntaxSnapshot {
+ pub fn is_empty(&self) -> bool {
+ self.layers.is_empty()
+ }
+
+ pub fn interpolate(&mut self, from_version: &clock::Global, text: &BufferSnapshot) {
+ let edits = text
+ .anchored_edits_since::<(usize, Point)>(&from_version)
+ .collect::<Vec<_>>();
+ if edits.is_empty() {
+ return;
+ }
+
+ let mut layers = SumTree::new();
+ let mut first_edit_ix_for_depth = 0;
+ let mut prev_depth = 0;
+ let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
+ cursor.next(text);
+
+ 'outer: loop {
+ let depth = cursor.end(text).max_depth;
+ if depth > prev_depth {
+ first_edit_ix_for_depth = 0;
+ prev_depth = depth;
+ }
+
+ // Preserve any layers at this depth that precede the first edit.
+ if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
+ let target = DepthAndMaxPosition(depth, edit_range.start);
+ if target.cmp(&cursor.start(), text).is_gt() {
+ let slice = cursor.slice(&target, Bias::Left, text);
+ layers.push_tree(slice, text);
+ }
+ }
+ // If this layer follows all of the edits, then preserve it and any
+ // subsequent layers at this same depth.
+ else if cursor.item().is_some() {
+ let slice = cursor.slice(
+ &DepthAndRange(depth + 1, Anchor::MIN..Anchor::MAX),
+ Bias::Left,
+ text,
+ );
+ layers.push_tree(slice, text);
+ continue;
+ };
+
+ let layer = if let Some(layer) = cursor.item() {
+ layer
+ } else {
+ break;
+ };
+ let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
+
+
+ // Ignore edits that end before the start of this layer, and don't consider them
+ // for any subsequent layers at this same depth.
+ loop {
+ if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
+ if edit_range.end.cmp(&layer.range.start, text).is_le() {
+ first_edit_ix_for_depth += 1;
+ } else {
+ break;
+ }
+ } else {
+ continue 'outer;
+ }
+ }
+
+ let mut layer = layer.clone();
+ for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
+ // Ignore any edits that follow this layer.
+ if edit_range.start.cmp(&layer.range.end, text).is_ge() {
+ break;
+ }
+
+ // Apply any edits that intersect this layer to the layer's syntax tree.
+ let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
+ tree_sitter::InputEdit {
+ start_byte: edit.new.start.0 - start_byte,
+ old_end_byte: edit.new.start.0 - start_byte
+ + (edit.old.end.0 - edit.old.start.0),
+ new_end_byte: edit.new.end.0 - start_byte,
+ start_position: (edit.new.start.1 - start_point).to_ts_point(),
+ old_end_position: (edit.new.start.1 - start_point
+ + (edit.old.end.1 - edit.old.start.1))
+ .to_ts_point(),
+ new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
+ }
+ } else {
+ let node = layer.tree.root_node();
+ tree_sitter::InputEdit {
+ start_byte: 0,
+ old_end_byte: node.end_byte(),
+ new_end_byte: 0,
+ start_position: Default::default(),
+ old_end_position: node.end_position(),
+ new_end_position: Default::default(),
+ }
+ };
+
+ layer.tree.edit(&tree_edit);
+ }
+
+ debug_assert!(
+ layer.tree.root_node().end_byte() <= text.len(),
+ "tree's size {}, is larger than text size {}",
+ layer.tree.root_node().end_byte(),
+ text.len(),
+ );
+
+ layers.push(layer, text);
+ cursor.next(text);
+ }
+
+ layers.push_tree(cursor.suffix(&text), &text);
+ drop(cursor);
+ self.layers = layers;
+ }
+
+ pub fn reparse(
+ &mut self,
+ from_version: &clock::Global,
+ text: &BufferSnapshot,
+ registry: Option<Arc<LanguageRegistry>>,
+ language: Arc<Language>,
+ ) {
+ let edits = text.edits_since::<usize>(from_version).collect::<Vec<_>>();
+ let max_depth = self.layers.summary().max_depth;
+ let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
+ cursor.next(&text);
+ let mut layers = SumTree::new();
+
+ let mut changed_regions = ChangeRegionSet::default();
+ let mut queue = BinaryHeap::new();
+ queue.push(ReparseStep {
+ depth: 0,
+ language: language.clone(),
+ ranges: Vec::new(),
+ range: Anchor::MIN..Anchor::MAX,
+ });
+
+ loop {
+ let step = queue.pop();
+ let (depth, range) = if let Some(step) = &step {
+ (step.depth, step.range.clone())
+ } else {
+ (max_depth + 1, Anchor::MAX..Anchor::MAX)
+ };
+
+ let target = DepthAndRange(depth, range.clone());
+ let mut done = cursor.item().is_none();
+ while !done && target.cmp(&cursor.end(text), &text).is_gt() {
+ done = true;
+
+ let bounded_target =
+ DepthAndRangeOrMaxPosition(target.clone(), changed_regions.start_position());
+ if bounded_target.cmp(&cursor.start(), &text).is_gt() {
+ let slice = cursor.slice(&bounded_target, Bias::Left, text);
+ if !slice.is_empty() {
+ layers.push_tree(slice, &text);
+ if changed_regions.prune(cursor.end(text), text) {
+ done = false;
+ }
+ }
+ }
+
+ while target.cmp(&cursor.end(text), text).is_gt() {
+ let layer = if let Some(layer) = cursor.item() {
+ layer
+ } else {
+ break;
+ };
+
+ if changed_regions.intersects(&layer, text) {
+ changed_regions.insert(
+ ChangedRegion {
+ depth: layer.depth + 1,
+ range: layer.range.clone(),
+ },
+ text,
+ );
+ } else {
+ layers.push(layer.clone(), text);
+ }
+
+ cursor.next(text);
+ if changed_regions.prune(cursor.end(text), text) {
+ done = false;
+ }
+ }
+ }
+
+ let (ranges, language) = if let Some(step) = step {
+ (step.ranges, step.language)
+ } else {
+ break;
+ };
+
+ let start_point;
+ let start_byte;
+ let end_byte;
+ if let Some((first, last)) = ranges.first().zip(ranges.last()) {
+ start_point = first.start_point;
+ start_byte = first.start_byte;
+ end_byte = last.end_byte;
+ } else {
+ start_point = Point::zero().to_ts_point();
+ start_byte = 0;
+ end_byte = text.len();
+ };
+
+ let mut old_layer = cursor.item();
+ if let Some(layer) = old_layer {
+ if layer.range.to_offset(text) == (start_byte..end_byte) {
+ cursor.next(&text);
+ } else {
+ old_layer = None;
+ }
+ }
+
+ let grammar = if let Some(grammar) = language.grammar.as_deref() {
+ grammar
+ } else {
+ continue;
+ };
+
+ let tree;
+ let changed_ranges;
+ if let Some(old_layer) = old_layer {
+ tree = parse_text(
+ grammar,
+ text.as_rope(),
+ Some(old_layer.tree.clone()),
+ ranges,
+ );
+ changed_ranges = join_ranges(
+ edits
+ .iter()
+ .map(|e| e.new.clone())
+ .filter(|range| range.start < end_byte && range.end > start_byte),
+ old_layer
+ .tree
+ .changed_ranges(&tree)
+ .map(|r| start_byte + r.start_byte..start_byte + r.end_byte),
+ );
+ } else {
+ tree = parse_text(grammar, text.as_rope(), None, ranges);
+ changed_ranges = vec![start_byte..end_byte];
+ }
+
+ layers.push(
+ SyntaxLayer {
+ depth,
+ range,
+ tree: tree.clone(),
+ language: language.clone(),
+ },
+ &text,
+ );
+
+ if let (Some((config, registry)), false) = (
+ grammar.injection_config.as_ref().zip(registry.as_ref()),
+ changed_ranges.is_empty(),
+ ) {
+ let depth = depth + 1;
+ for range in &changed_ranges {
+ changed_regions.insert(
+ ChangedRegion {
+ depth,
+ range: text.anchor_before(range.start)..text.anchor_after(range.end),
+ },
+ text,
+ );
+ }
+ get_injections(
+ config,
+ text,
+ tree.root_node_with_offset(start_byte, start_point),
+ registry,
+ depth,
+ &changed_ranges,
+ &mut queue,
+ );
+ }
+ }
+
+ drop(cursor);
+ self.layers = layers;
+ }
+
+ pub fn single_tree_captures<'a>(
+ range: Range<usize>,
+ text: &'a Rope,
+ tree: &'a Tree,
+ grammar: &'a Grammar,
+ query: fn(&Grammar) -> Option<&Query>,
+ ) -> SyntaxMapCaptures<'a> {
+ SyntaxMapCaptures::new(
+ range.clone(),
+ text,
+ [(grammar, 0, tree.root_node())].into_iter(),
+ query,
+ )
+ }
+
+ pub fn captures<'a>(
+ &'a self,
+ range: Range<usize>,
+ buffer: &'a BufferSnapshot,
+ query: fn(&Grammar) -> Option<&Query>,
+ ) -> SyntaxMapCaptures {
+ SyntaxMapCaptures::new(
+ range.clone(),
+ buffer.as_rope(),
+ self.layers_for_range(range, buffer).into_iter(),
+ query,
+ )
+ }
+
+ pub fn matches<'a>(
+ &'a self,
+ range: Range<usize>,
+ buffer: &'a BufferSnapshot,
+ query: fn(&Grammar) -> Option<&Query>,
+ ) -> SyntaxMapMatches {
+ SyntaxMapMatches::new(
+ range.clone(),
+ buffer.as_rope(),
+ self.layers_for_range(range, buffer).into_iter(),
+ query,
+ )
+ }
+
+ #[cfg(test)]
+ pub fn layers(&self, buffer: &BufferSnapshot) -> Vec<(&Grammar, usize, Node)> {
+ self.layers_for_range(0..buffer.len(), buffer)
+ }
+
+ pub fn layers_for_range<'a, T: ToOffset>(
+ &self,
+ range: Range<T>,
+ buffer: &BufferSnapshot,
+ ) -> Vec<(&Grammar, usize, Node)> {
+ let start = buffer.anchor_before(range.start.to_offset(buffer));
+ let end = buffer.anchor_after(range.end.to_offset(buffer));
+
+ let mut cursor = self.layers.filter::<_, ()>(|summary| {
+ if summary.max_depth > summary.min_depth {
+ true
+ } else {
+ let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
+ let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
+ !is_before_start && !is_after_end
+ }
+ });
+
+ let mut result = Vec::new();
+ cursor.next(buffer);
+ while let Some(layer) = cursor.item() {
+ if let Some(grammar) = &layer.language.grammar {
+ result.push((
+ grammar.as_ref(),
+ layer.depth,
+ layer.tree.root_node_with_offset(
+ layer.range.start.to_offset(buffer),
+ layer.range.start.to_point(buffer).to_ts_point(),
+ ),
+ ));
+ }
+ cursor.next(buffer)
+ }
+
+ result
+ }
+}
+
+impl<'a> SyntaxMapCaptures<'a> {
+ fn new(
+ range: Range<usize>,
+ text: &'a Rope,
+ layers: impl Iterator<Item = (&'a Grammar, usize, Node<'a>)>,
+ query: fn(&Grammar) -> Option<&Query>,
+ ) -> Self {
+ let mut result = Self {
+ layers: Vec::new(),
+ grammars: Vec::new(),
+ active_layer_count: 0,
+ };
+ for (grammar, depth, node) in layers {
+ let query = if let Some(query) = query(grammar) {
+ query
+ } else {
+ continue;
+ };
+
+ let mut query_cursor = QueryCursorHandle::new();
+
+ // TODO - add a Tree-sitter API to remove the need for this.
+ let cursor = unsafe {
+ std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
+ };
+
+ cursor.set_byte_range(range.clone());
+ let captures = cursor.captures(query, node, TextProvider(text));
+ let grammar_index = result
+ .grammars
+ .iter()
+ .position(|g| g.id == grammar.id())
+ .unwrap_or_else(|| {
+ result.grammars.push(grammar);
+ result.grammars.len() - 1
+ });
+ let mut layer = SyntaxMapCapturesLayer {
+ depth,
+ grammar_index,
+ next_capture: None,
+ captures,
+ _query_cursor: query_cursor,
+ };
+
+ layer.advance();
+ if layer.next_capture.is_some() {
+ let key = layer.sort_key();
+ let ix = match result.layers[..result.active_layer_count]
+ .binary_search_by_key(&key, |layer| layer.sort_key())
+ {
+ Ok(ix) | Err(ix) => ix,
+ };
+ result.layers.insert(ix, layer);
+ result.active_layer_count += 1;
+ } else {
+ result.layers.push(layer);
+ }
+ }
+
+ result
+ }
+
+ pub fn grammars(&self) -> &[&'a Grammar] {
+ &self.grammars
+ }
+
+ pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
+ let layer = self.layers[..self.active_layer_count].first()?;
+ let capture = layer.next_capture?;
+ Some(SyntaxMapCapture {
+ depth: layer.depth,
+ grammar_index: layer.grammar_index,
+ index: capture.index,
+ node: capture.node,
+ })
+ }
+
+ pub fn advance(&mut self) -> bool {
+ let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
+ layer
+ } else {
+ return false;
+ };
+
+ layer.advance();
+ if layer.next_capture.is_some() {
+ let key = layer.sort_key();
+ let i = 1 + self.layers[1..self.active_layer_count]
+ .iter()
+ .position(|later_layer| key < later_layer.sort_key())
+ .unwrap_or(self.active_layer_count - 1);
+ self.layers[0..i].rotate_left(1);
+ } else {
+ self.layers[0..self.active_layer_count].rotate_left(1);
+ self.active_layer_count -= 1;
+ }
+
+ true
+ }
+
+ pub fn set_byte_range(&mut self, range: Range<usize>) {
+ for layer in &mut self.layers {
+ layer.captures.set_byte_range(range.clone());
+ if let Some(capture) = &layer.next_capture {
+ if capture.node.end_byte() > range.start {
+ continue;
+ }
+ }
+ layer.advance();
+ }
+ self.layers.sort_unstable_by_key(|layer| layer.sort_key());
+ self.active_layer_count = self
+ .layers
+ .iter()
+ .position(|layer| layer.next_capture.is_none())
+ .unwrap_or(self.layers.len());
+ }
+}
+
+impl<'a> SyntaxMapMatches<'a> {
+ fn new(
+ range: Range<usize>,
+ text: &'a Rope,
+ layers: impl Iterator<Item = (&'a Grammar, usize, Node<'a>)>,
+ query: fn(&Grammar) -> Option<&Query>,
+ ) -> Self {
+ let mut result = Self::default();
+ for (grammar, depth, node) in layers {
+ let query = if let Some(query) = query(grammar) {
+ query
+ } else {
+ continue;
+ };
+
+ let mut query_cursor = QueryCursorHandle::new();
+
+ // TODO - add a Tree-sitter API to remove the need for this.
+ let cursor = unsafe {
+ std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
+ };
+
+ cursor.set_byte_range(range.clone());
+ let matches = cursor.matches(query, node, TextProvider(text));
+ let grammar_index = result
+ .grammars
+ .iter()
+ .position(|g| g.id == grammar.id())
+ .unwrap_or_else(|| {
+ result.grammars.push(grammar);
+ result.grammars.len() - 1
+ });
+ let mut layer = SyntaxMapMatchesLayer {
+ depth,
+ grammar_index,
+ matches,
+ next_pattern_index: 0,
+ next_captures: Vec::new(),
+ has_next: false,
+ _query_cursor: query_cursor,
+ };
+
+ layer.advance();
+ if layer.has_next {
+ let key = layer.sort_key();
+ let ix = match result.layers[..result.active_layer_count]
+ .binary_search_by_key(&key, |layer| layer.sort_key())
+ {
+ Ok(ix) | Err(ix) => ix,
+ };
+ result.layers.insert(ix, layer);
+ result.active_layer_count += 1;
+ } else {
+ result.layers.push(layer);
+ }
+ }
+ result
+ }
+
+ pub fn grammars(&self) -> &[&'a Grammar] {
+ &self.grammars
+ }
+
+ pub fn peek(&self) -> Option<SyntaxMapMatch> {
+ let layer = self.layers.first()?;
+ if !layer.has_next {
+ return None;
+ }
+ Some(SyntaxMapMatch {
+ depth: layer.depth,
+ grammar_index: layer.grammar_index,
+ pattern_index: layer.next_pattern_index,
+ captures: &layer.next_captures,
+ })
+ }
+
+ pub fn advance(&mut self) -> bool {
+ let layer = if let Some(layer) = self.layers.first_mut() {
+ layer
+ } else {
+ return false;
+ };
+
+ layer.advance();
+ if layer.has_next {
+ let key = layer.sort_key();
+ let i = 1 + self.layers[1..self.active_layer_count]
+ .iter()
+ .position(|later_layer| key < later_layer.sort_key())
+ .unwrap_or(self.active_layer_count - 1);
+ self.layers[0..i].rotate_left(1);
+ } else {
+ self.layers[0..self.active_layer_count].rotate_left(1);
+ self.active_layer_count -= 1;
+ }
+
+ true
+ }
+}
+
+impl<'a> SyntaxMapCapturesLayer<'a> {
+ fn advance(&mut self) {
+ self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
+ }
+
+ fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
+ if let Some(capture) = &self.next_capture {
+ let range = capture.node.byte_range();
+ (range.start, Reverse(range.end), self.depth)
+ } else {
+ (usize::MAX, Reverse(0), usize::MAX)
+ }
+ }
+}
+
+impl<'a> SyntaxMapMatchesLayer<'a> {
+ fn advance(&mut self) {
+ if let Some(mat) = self.matches.next() {
+ self.next_captures.clear();
+ self.next_captures.extend_from_slice(&mat.captures);
+ self.next_pattern_index = mat.pattern_index;
+ self.has_next = true;
+ } else {
+ self.has_next = false;
+ }
+ }
+
+ fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
+ if self.has_next {
+ let captures = &self.next_captures;
+ if let Some((first, last)) = captures.first().zip(captures.last()) {
+ return (
+ first.node.start_byte(),
+ Reverse(last.node.end_byte()),
+ self.depth,
+ );
+ }
+ }
+ (usize::MAX, Reverse(0), usize::MAX)
+ }
+}
+
+impl<'a> Iterator for SyntaxMapCaptures<'a> {
+ type Item = SyntaxMapCapture<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let result = self.peek();
+ self.advance();
+ result
+ }
+}
+
+fn join_ranges(
+ a: impl Iterator<Item = Range<usize>>,
+ b: impl Iterator<Item = Range<usize>>,
+) -> Vec<Range<usize>> {
+ let mut result = Vec::<Range<usize>>::new();
+ let mut a = a.peekable();
+ let mut b = b.peekable();
+ loop {
+ let range = match (a.peek(), b.peek()) {
+ (Some(range_a), Some(range_b)) => {
+ if range_a.start < range_b.start {
+ a.next().unwrap()
+ } else {
+ b.next().unwrap()
+ }
+ }
+ (None, Some(_)) => b.next().unwrap(),
+ (Some(_), None) => a.next().unwrap(),
+ (None, None) => break,
+ };
+
+ if let Some(last) = result.last_mut() {
+ if range.start <= last.end {
+ last.end = last.end.max(range.end);
+ continue;
+ }
+ }
+ result.push(range);
+ }
+ result
+}
+
+fn parse_text(
+ grammar: &Grammar,
+ text: &Rope,
+ old_tree: Option<Tree>,
+ mut ranges: Vec<tree_sitter::Range>,
+) -> Tree {
+ let (start_byte, start_point) = ranges
+ .first()
+ .map(|range| (range.start_byte, Point::from_ts_point(range.start_point)))
+ .unwrap_or_default();
+
+ for range in &mut ranges {
+ range.start_byte -= start_byte;
+ range.end_byte -= start_byte;
+ range.start_point = (Point::from_ts_point(range.start_point) - start_point).to_ts_point();
+ range.end_point = (Point::from_ts_point(range.end_point) - start_point).to_ts_point();
+ }
+
+ PARSER.with(|parser| {
+ let mut parser = parser.borrow_mut();
+ let mut chunks = text.chunks_in_range(start_byte..text.len());
+ parser
+ .set_included_ranges(&ranges)
+ .expect("overlapping ranges");
+ parser
+ .set_language(grammar.ts_language)
+ .expect("incompatible grammar");
+ parser
+ .parse_with(
+ &mut move |offset, _| {
+ chunks.seek(start_byte + offset);
+ chunks.next().unwrap_or("").as_bytes()
+ },
+ old_tree.as_ref(),
+ )
+ .expect("invalid language")
+ })
+}
+
+fn get_injections(
+ config: &InjectionConfig,
+ text: &BufferSnapshot,
+ node: Node,
+ language_registry: &LanguageRegistry,
+ depth: usize,
+ query_ranges: &[Range<usize>],
+ queue: &mut BinaryHeap<ReparseStep>,
+) -> bool {
+ let mut result = false;
+ let mut query_cursor = QueryCursorHandle::new();
+ let mut prev_match = None;
+ for query_range in query_ranges {
+ query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end);
+ for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
+ let content_ranges = mat
+ .nodes_for_capture_index(config.content_capture_ix)
+ .map(|node| node.range())
+ .collect::<Vec<_>>();
+ if content_ranges.is_empty() {
+ continue;
+ }
+
+ // Avoid duplicate matches if two changed ranges intersect the same injection.
+ let content_range =
+ content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
+ if let Some((last_pattern_ix, last_range)) = &prev_match {
+ if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
+ continue;
+ }
+ }
+ prev_match = Some((mat.pattern_index, content_range.clone()));
+
+ let language_name = config.languages_by_pattern_ix[mat.pattern_index]
+ .as_ref()
+ .map(|s| Cow::Borrowed(s.as_ref()))
+ .or_else(|| {
+ let ix = config.language_capture_ix?;
+ let node = mat.nodes_for_capture_index(ix).next()?;
+ Some(Cow::Owned(text.text_for_range(node.byte_range()).collect()))
+ });
+
+ if let Some(language_name) = language_name {
+ if let Some(language) = language_registry.get_language(language_name.as_ref()) {
+ result = true;
+ let range = text.anchor_before(content_range.start)
+ ..text.anchor_after(content_range.end);
+ queue.push(ReparseStep {
+ depth,
+ language,
+ ranges: content_ranges,
+ range,
+ })
+ }
+ }
+ }
+ }
+ result
+}
+
+impl std::ops::Deref for SyntaxMap {
+ type Target = SyntaxSnapshot;
+
+ fn deref(&self) -> &Self::Target {
+ &self.snapshot
+ }
+}
+
+impl PartialEq for ReparseStep {
+ fn eq(&self, _: &Self) -> bool {
+ false
+ }
+}
+
+impl Eq for ReparseStep {}
+
+impl PartialOrd for ReparseStep {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ Some(self.cmp(&other))
+ }
+}
+
+impl Ord for ReparseStep {
+ fn cmp(&self, other: &Self) -> Ordering {
+ let range_a = self.range();
+ let range_b = other.range();
+ Ord::cmp(&other.depth, &self.depth)
+ .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
+ .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
+ }
+}
+
+impl ReparseStep {
+ fn range(&self) -> Range<usize> {
+ let start = self.ranges.first().map_or(0, |r| r.start_byte);
+ let end = self.ranges.last().map_or(0, |r| r.end_byte);
+ start..end
+ }
+}
+
+impl ChangedRegion {
+ fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
+ let range_a = &self.range;
+ let range_b = &other.range;
+ Ord::cmp(&self.depth, &other.depth)
+ .then_with(|| range_a.start.cmp(&range_b.start, buffer))
+ .then_with(|| range_b.end.cmp(&range_a.end, buffer))
+ }
+}
+
+impl ChangeRegionSet {
+ fn start_position(&self) -> DepthAndMaxPosition {
+ self.0
+ .first()
+ .map_or(DepthAndMaxPosition(usize::MAX, Anchor::MAX), |region| {
+ DepthAndMaxPosition(region.depth, region.range.start)
+ })
+ }
+
+ fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
+ for region in &self.0 {
+ if region.depth < layer.depth {
+ continue;
+ }
+ if region.depth > layer.depth {
+ break;
+ }
+ if region.range.end.cmp(&layer.range.start, text).is_le() {
+ continue;
+ }
+ if region.range.start.cmp(&layer.range.end, text).is_ge() {
+ break;
+ }
+ return true;
+ }
+ false
+ }
+
+ fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
+ if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
+ self.0.insert(ix, region);
+ }
+ }
+
+ fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
+ let prev_len = self.0.len();
+ self.0.retain(|region| {
+ region.depth > summary.max_depth
+ || (region.depth == summary.max_depth
+ && region
+ .range
+ .end
+ .cmp(&summary.last_layer_range.start, text)
+ .is_gt())
+ });
+ self.0.len() < prev_len
+ }
+}
+
+impl Default for SyntaxLayerSummary {
+ fn default() -> Self {
+ Self {
+ max_depth: 0,
+ min_depth: 0,
+ range: Anchor::MAX..Anchor::MIN,
+ last_layer_range: Anchor::MIN..Anchor::MAX,
+ }
+ }
+}
+
+impl sum_tree::Summary for SyntaxLayerSummary {
+ type Context = BufferSnapshot;
+
+ fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
+ if other.max_depth > self.max_depth {
+ self.max_depth = other.max_depth;
+ self.range = other.range.clone();
+ } else {
+ if other.range.start.cmp(&self.range.start, buffer).is_lt() {
+ self.range.start = other.range.start;
+ }
+ if other.range.end.cmp(&self.range.end, buffer).is_gt() {
+ self.range.end = other.range.end;
+ }
+ }
+ self.last_layer_range = other.last_layer_range.clone();
+ }
+}
+
+impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndRange {
+ fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
+ Ord::cmp(&self.0, &cursor_location.max_depth)
+ .then_with(|| {
+ self.1
+ .start
+ .cmp(&cursor_location.last_layer_range.start, buffer)
+ })
+ .then_with(|| {
+ cursor_location
+ .last_layer_range
+ .end
+ .cmp(&self.1.end, buffer)
+ })
+ }
+}
+
+impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndMaxPosition {
+ fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
+ Ord::cmp(&self.0, &cursor_location.max_depth)
+ .then_with(|| self.1.cmp(&cursor_location.range.end, text))
+ }
+}
+
+impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndRangeOrMaxPosition {
+ fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
+ if self.1.cmp(cursor_location, buffer).is_le() {
+ return Ordering::Less;
+ } else {
+ self.0.cmp(cursor_location, buffer)
+ }
+ }
+}
+
+impl sum_tree::Item for SyntaxLayer {
+ type Summary = SyntaxLayerSummary;
+
+ fn summary(&self) -> Self::Summary {
+ SyntaxLayerSummary {
+ min_depth: self.depth,
+ max_depth: self.depth,
+ range: self.range.clone(),
+ last_layer_range: self.range.clone(),
+ }
+ }
+}
+
+impl std::fmt::Debug for SyntaxLayer {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SyntaxLayer")
+ .field("depth", &self.depth)
+ .field("range", &self.range)
+ .field("tree", &self.tree)
+ .finish()
+ }
+}
+
+impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
+ type I = ByteChunks<'a>;
+
+ fn text(&mut self, node: tree_sitter::Node) -> Self::I {
+ ByteChunks(self.0.chunks_in_range(node.byte_range()))
+ }
+}
+
+impl<'a> Iterator for ByteChunks<'a> {
+ type Item = &'a [u8];
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.0.next().map(str::as_bytes)
+ }
+}
+
+impl QueryCursorHandle {
+ pub(crate) fn new() -> Self {
+ let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
+ cursor.set_match_limit(64);
+ QueryCursorHandle(Some(cursor))
+ }
+}
+
+impl Deref for QueryCursorHandle {
+ type Target = QueryCursor;
+
+ fn deref(&self) -> &Self::Target {
+ self.0.as_ref().unwrap()
+ }
+}
+
+impl DerefMut for QueryCursorHandle {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ self.0.as_mut().unwrap()
+ }
+}
+
+impl Drop for QueryCursorHandle {
+ fn drop(&mut self) {
+ let mut cursor = self.0.take().unwrap();
+ cursor.set_byte_range(0..usize::MAX);
+ cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
+ QUERY_CURSORS.lock().push(cursor)
+ }
+}
+
+pub(crate) trait ToTreeSitterPoint {
+ fn to_ts_point(self) -> tree_sitter::Point;
+ fn from_ts_point(point: tree_sitter::Point) -> Self;
+}
+
+impl ToTreeSitterPoint for Point {
+ fn to_ts_point(self) -> tree_sitter::Point {
+ tree_sitter::Point::new(self.row as usize, self.column as usize)
+ }
+
+ fn from_ts_point(point: tree_sitter::Point) -> Self {
+ Point::new(point.row as u32, point.column as u32)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::LanguageConfig;
+ use rand::rngs::StdRng;
+ use std::env;
+ use text::{Buffer, Point};
+ use unindent::Unindent as _;
+ use util::test::marked_text_ranges;
+
+ #[gpui::test]
+ fn test_syntax_map_layers_for_range() {
+ let registry = Arc::new(LanguageRegistry::test());
+ let language = Arc::new(rust_lang());
+ registry.add(language.clone());
+
+ let mut buffer = Buffer::new(
+ 0,
+ 0,
+ r#"
+ fn a() {
+ assert_eq!(
+ b(vec![C {}]),
+ vec![d.e],
+ );
+ println!("{}", f(|_| true));
+ }
+ "#
+ .unindent(),
+ );
+
+ let mut syntax_map = SyntaxMap::new();
+ syntax_map.set_language_registry(registry.clone());
+ syntax_map.reparse(language.clone(), &buffer);
+
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(2, 0)..Point::new(2, 0),
+ &[
+ "...(function_item ... (block (expression_statement (macro_invocation...",
+ "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
+ ],
+ );
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(2, 14)..Point::new(2, 16),
+ &[
+ "...(function_item ...",
+ "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
+ "...(array_expression (struct_expression ...",
+ ],
+ );
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(3, 14)..Point::new(3, 16),
+ &[
+ "...(function_item ...",
+ "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
+ "...(array_expression (field_expression ...",
+ ],
+ );
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(5, 12)..Point::new(5, 16),
+ &[
+ "...(function_item ...",
+ "...(call_expression ... (arguments (closure_expression ...",
+ ],
+ );
+
+ // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
+ let macro_name_range = range_for_text(&buffer, "vec!");
+ buffer.edit([(macro_name_range, "&")]);
+ syntax_map.interpolate(&buffer);
+ syntax_map.reparse(language.clone(), &buffer);
+
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(2, 14)..Point::new(2, 16),
+ &[
+ "...(function_item ...",
+ "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
+ ],
+ );
+
+ // Put the vec! macro back, adding back the syntactic layer.
+ buffer.undo();
+ syntax_map.interpolate(&buffer);
+ syntax_map.reparse(language.clone(), &buffer);
+
+ assert_layers_for_range(
+ &syntax_map,
+ &buffer,
+ Point::new(2, 14)..Point::new(2, 16),
+ &[
+ "...(function_item ...",
+ "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
+ "...(array_expression (struct_expression ...",
+ ],
+ );
+ }
+
+ #[gpui::test]
+ fn test_typing_multiple_new_injections() {
+ let (buffer, syntax_map) = test_edit_sequence(&[
+ "fn a() { dbg }",
+ "fn a() { dbg«!» }",
+ "fn a() { dbg!«()» }",
+ "fn a() { dbg!(«b») }",
+ "fn a() { dbg!(b«.») }",
+ "fn a() { dbg!(b.«c») }",
+ "fn a() { dbg!(b.c«()») }",
+ "fn a() { dbg!(b.c(«vec»)) }",
+ "fn a() { dbg!(b.c(vec«!»)) }",
+ "fn a() { dbg!(b.c(vec!«[]»)) }",
+ "fn a() { dbg!(b.c(vec![«d»])) }",
+ "fn a() { dbg!(b.c(vec![d«.»])) }",
+ "fn a() { dbg!(b.c(vec![d.«e»])) }",
+ ]);
+
+ assert_capture_ranges(
+ &syntax_map,
+ &buffer,
+ &["field"],
+ "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
+ );
+ }
+
+ #[gpui::test]
+ fn test_pasting_new_injection_line_between_others() {
+ let (buffer, syntax_map) = test_edit_sequence(&[
+ "
+ fn a() {
+ b!(B {});
+ c!(C {});
+ d!(D {});
+ e!(E {});
+ f!(F {});
+ g!(G {});
+ }
+ ",
+ "
+ fn a() {
+ b!(B {});
+ c!(C {});
+ d!(D {});
+ « h!(H {});
+ » e!(E {});
+ f!(F {});
+ g!(G {});
+ }
+ ",
+ ]);
+
+ assert_capture_ranges(
+ &syntax_map,
+ &buffer,
+ &["struct"],
+ "
+ fn a() {
+ b!(«B {}»);
+ c!(«C {}»);
+ d!(«D {}»);
+ h!(«H {}»);
+ e!(«E {}»);
+ f!(«F {}»);
+ g!(«G {}»);
+ }
+ ",
+ );
+ }
+
+ #[gpui::test]
+ fn test_joining_injections_with_child_injections() {
+ let (buffer, syntax_map) = test_edit_sequence(&[
+ "
+ fn a() {
+ b!(
+ c![one.two.three],
+ d![four.five.six],
+ );
+ e!(
+ f![seven.eight],
+ );
+ }
+ ",
+ "
+ fn a() {
+ b!(
+ c![one.two.three],
+ d![four.five.six],
+ ˇ f![seven.eight],
+ );
+ }
+ ",
+ ]);
+
+ assert_capture_ranges(
+ &syntax_map,
+ &buffer,
+ &["field"],
+ "
+ fn a() {
+ b!(
+ c![one.«two».«three»],
+ d![four.«five».«six»],
+ f![seven.«eight»],
+ );
+ }
+ ",
+ );
+ }
+
+ #[gpui::test]
+ fn test_editing_edges_of_injection() {
+ test_edit_sequence(&[
+ "
+ fn a() {
+ b!(c!())
+ }
+ ",
+ "
+ fn a() {
+ «d»!(c!())
+ }
+ ",
+ "
+ fn a() {
+ «e»d!(c!())
+ }
+ ",
+ "
+ fn a() {
+ ed!«[»c!()«]»
+ }
+ ",
+ ]);
+ }
+
+ #[gpui::test]
+ fn test_edits_preceding_and_intersecting_injection() {
+ test_edit_sequence(&[
+ //
+ "const aaaaaaaaaaaa: B = c!(d(e.f));",
+ "const aˇa: B = c!(d(eˇ));",
+ ]);
+ }
+
+ #[gpui::test]
+ fn test_non_local_changes_create_injections() {
+ test_edit_sequence(&[
+ "
+ // a! {
+ static B: C = d;
+ // }
+ ",
+ "
+ ˇa! {
+ static B: C = d;
+ ˇ}
+ ",
+ ]);
+ }
+
+ #[gpui::test]
+ fn test_creating_many_injections_in_one_edit() {
+ test_edit_sequence(&[
+ "
+ fn a() {
+ one(Two::three(3));
+ four(Five::six(6));
+ seven(Eight::nine(9));
+ }
+ ",
+ "
+ fn a() {
+ one«!»(Two::three(3));
+ four«!»(Five::six(6));
+ seven«!»(Eight::nine(9));
+ }
+ ",
+ "
+ fn a() {
+ one!(Two::three«!»(3));
+ four!(Five::six«!»(6));
+ seven!(Eight::nine«!»(9));
+ }
+ ",
+ ]);
+ }
+
+ #[gpui::test]
+ fn test_editing_across_injection_boundary() {
+ test_edit_sequence(&[
+ "
+ fn one() {
+ two();
+ three!(
+ three.four,
+ five.six,
+ );
+ }
+ ",
+ "
+ fn one() {
+ two();
+ th«irty_five![»
+ three.four,
+ five.six,
+ « seven.eight,
+ ];»
+ }
+ ",
+ ]);
+ }
+
+ #[gpui::test]
+ fn test_removing_injection_by_replacing_across_boundary() {
+ test_edit_sequence(&[
+ "
+ fn one() {
+ two!(
+ three.four,
+ );
+ }
+ ",
+ "
+ fn one() {
+ t«en
+ .eleven(
+ twelve,
+ »
+ three.four,
+ );
+ }
+ ",
+ ]);
+ }
+
+ #[gpui::test(iterations = 100)]
+ fn test_random_syntax_map_edits(mut rng: StdRng) {
+ let operations = env::var("OPERATIONS")
+ .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
+ .unwrap_or(10);
+
+ let text = r#"
+ fn test_something() {
+ let vec = vec![5, 1, 3, 8];
+ assert_eq!(
+ vec
+ .into_iter()
+ .map(|i| i * 2)
+ .collect::<Vec<usize>>(),
+ vec![
+ 5 * 2, 1 * 2, 3 * 2, 8 * 2
+ ],
+ );
+ }
+ "#
+ .unindent()
+ .repeat(2);
+
+ let registry = Arc::new(LanguageRegistry::test());
+ let language = Arc::new(rust_lang());
+ registry.add(language.clone());
+ let mut buffer = Buffer::new(0, 0, text);
+
+ let mut syntax_map = SyntaxMap::new();
+ syntax_map.set_language_registry(registry.clone());
+ syntax_map.reparse(language.clone(), &buffer);
+
+ let mut reference_syntax_map = SyntaxMap::new();
+ reference_syntax_map.set_language_registry(registry.clone());
+
+ log::info!("initial text:\n{}", buffer.text());
+
+ for _ in 0..operations {
+ let prev_buffer = buffer.snapshot();
+ let prev_syntax_map = syntax_map.snapshot();
+
+ buffer.randomly_edit(&mut rng, 3);
+ log::info!("text:\n{}", buffer.text());
+
+ syntax_map.interpolate(&buffer);
+ check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
+
+ syntax_map.reparse(language.clone(), &buffer);
+
+ reference_syntax_map.clear();
+ reference_syntax_map.reparse(language.clone(), &buffer);
+ }
+
+ for i in 0..operations {
+ let i = operations - i - 1;
+ buffer.undo();
+ log::info!("undoing operation {}", i);
+ log::info!("text:\n{}", buffer.text());
+
+ syntax_map.interpolate(&buffer);
+ syntax_map.reparse(language.clone(), &buffer);
+
+ reference_syntax_map.clear();
+ reference_syntax_map.reparse(language.clone(), &buffer);
+ assert_eq!(
+ syntax_map.layers(&buffer).len(),
+ reference_syntax_map.layers(&buffer).len(),
+ "wrong number of layers after undoing edit {i}"
+ );
+ }
+
+ let layers = syntax_map.layers(&buffer);
+ let reference_layers = reference_syntax_map.layers(&buffer);
+ for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
+ {
+ assert_eq!(edited_layer.2.to_sexp(), reference_layer.2.to_sexp());
+ assert_eq!(edited_layer.2.range(), reference_layer.2.range());
+ }
+ }
+
+ fn check_interpolation(
+ old_syntax_map: &SyntaxSnapshot,
+ new_syntax_map: &SyntaxSnapshot,
+ old_buffer: &BufferSnapshot,
+ new_buffer: &BufferSnapshot,
+ ) {
+ let edits = new_buffer
+ .edits_since::<usize>(&old_buffer.version())
+ .collect::<Vec<_>>();
+
+ for (old_layer, new_layer) in old_syntax_map
+ .layers
+ .iter()
+ .zip(new_syntax_map.layers.iter())
+ {
+ assert_eq!(old_layer.range, new_layer.range);
+ let old_start_byte = old_layer.range.start.to_offset(old_buffer);
+ let new_start_byte = new_layer.range.start.to_offset(new_buffer);
+ let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
+ let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
+ let old_node = old_layer
+ .tree
+ .root_node_with_offset(old_start_byte, old_start_point);
+ let new_node = new_layer
+ .tree
+ .root_node_with_offset(new_start_byte, new_start_point);
+ check_node_edits(
+ old_layer.depth,
+ &old_layer.range,
+ old_node,
+ new_node,
+ old_buffer,
+ new_buffer,
+ &edits,
+ );
+ }
+
+ fn check_node_edits(
+ depth: usize,
+ range: &Range<Anchor>,
+ old_node: Node,
+ new_node: Node,
+ old_buffer: &BufferSnapshot,
+ new_buffer: &BufferSnapshot,
+ edits: &[text::Edit<usize>],
+ ) {
+ assert_eq!(old_node.kind(), new_node.kind());
+
+ let old_range = old_node.byte_range();
+ let new_range = new_node.byte_range();
+
+ let is_edited = edits
+ .iter()
+ .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
+ if is_edited {
+ assert!(
+ new_node.has_changes(),
+ concat!(
+ "failed to mark node as edited.\n",
+ "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
+ "node kind: {}, old node range: {:?}, new node range: {:?}",
+ ),
+ depth,
+ range.to_offset(old_buffer),
+ range.to_offset(new_buffer),
+ new_node.kind(),
+ old_range,
+ new_range,
+ );
+ }
+
+ if !new_node.has_changes() {
+ assert_eq!(
+ old_buffer
+ .text_for_range(old_range.clone())
+ .collect::<String>(),
+ new_buffer
+ .text_for_range(new_range.clone())
+ .collect::<String>(),
+ concat!(
+ "mismatched text for node\n",
+ "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
+ "node kind: {}, old node range:{:?}, new node range:{:?}",
+ ),
+ depth,
+ range.to_offset(old_buffer),
+ range.to_offset(new_buffer),
+ new_node.kind(),
+ old_range,
+ new_range,
+ );
+ }
+
+ for i in 0..new_node.child_count() {
+ check_node_edits(
+ depth,
+ range,
+ old_node.child(i).unwrap(),
+ new_node.child(i).unwrap(),
+ old_buffer,
+ new_buffer,
+ edits,
+ )
+ }
+ }
+ }
+
+ fn test_edit_sequence(steps: &[&str]) -> (Buffer, SyntaxMap) {
+ let registry = Arc::new(LanguageRegistry::test());
+ let language = Arc::new(rust_lang());
+ registry.add(language.clone());
+ let mut buffer = Buffer::new(0, 0, Default::default());
+
+ let mut mutated_syntax_map = SyntaxMap::new();
+ mutated_syntax_map.set_language_registry(registry.clone());
+ mutated_syntax_map.reparse(language.clone(), &buffer);
+
+ for (i, marked_string) in steps.into_iter().enumerate() {
+ edit_buffer(&mut buffer, &marked_string.unindent());
+
+ // Reparse the syntax map
+ mutated_syntax_map.interpolate(&buffer);
+ mutated_syntax_map.reparse(language.clone(), &buffer);
+
+ // Create a second syntax map from scratch
+ let mut reference_syntax_map = SyntaxMap::new();
+ reference_syntax_map.set_language_registry(registry.clone());
+ reference_syntax_map.reparse(language.clone(), &buffer);
+
+ // Compare the mutated syntax map to the new syntax map
+ let mutated_layers = mutated_syntax_map.layers(&buffer);
+ let reference_layers = reference_syntax_map.layers(&buffer);
+ assert_eq!(
+ mutated_layers.len(),
+ reference_layers.len(),
+ "wrong number of layers at step {i}"
+ );
+ for (edited_layer, reference_layer) in
+ mutated_layers.into_iter().zip(reference_layers.into_iter())
+ {
+ assert_eq!(
+ edited_layer.2.to_sexp(),
+ reference_layer.2.to_sexp(),
+ "different layer at step {i}"
+ );
+ assert_eq!(
+ edited_layer.2.range(),
+ reference_layer.2.range(),
+ "different layer at step {i}"
+ );
+ }
+ }
+
+ (buffer, mutated_syntax_map)
+ }
+
+ fn rust_lang() -> Language {
+ Language::new(
+ LanguageConfig {
+ name: "Rust".into(),
+ path_suffixes: vec!["rs".to_string()],
+ ..Default::default()
+ },
+ Some(tree_sitter_rust::language()),
+ )
+ .with_highlights_query(
+ r#"
+ (field_identifier) @field
+ (struct_expression) @struct
+ "#,
+ )
+ .unwrap()
+ .with_injection_query(
+ r#"
+ (macro_invocation
+ (token_tree) @content
+ (#set! "language" "rust"))
+ "#,
+ )
+ .unwrap()
+ }
+
+ fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
+ let start = buffer.as_rope().to_string().find(text).unwrap();
+ start..start + text.len()
+ }
+
+ fn assert_layers_for_range(
+ syntax_map: &SyntaxMap,
+ buffer: &BufferSnapshot,
+ range: Range<Point>,
+ expected_layers: &[&str],
+ ) {
+ let layers = syntax_map.layers_for_range(range, &buffer);
+ assert_eq!(
+ layers.len(),
+ expected_layers.len(),
+ "wrong number of layers"
+ );
+ for (i, ((_, _, node), expected_s_exp)) in
+ layers.iter().zip(expected_layers.iter()).enumerate()
+ {
+ let actual_s_exp = node.to_sexp();
+ assert!(
+ string_contains_sequence(
+ &actual_s_exp,
+ &expected_s_exp.split("...").collect::<Vec<_>>()
+ ),
+ "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
+ );
+ }
+ }
+
+ fn assert_capture_ranges(
+ syntax_map: &SyntaxMap,
+ buffer: &BufferSnapshot,
+ highlight_query_capture_names: &[&str],
+ marked_string: &str,
+ ) {
+ let mut actual_ranges = Vec::<Range<usize>>::new();
+ let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
+ grammar.highlights_query.as_ref()
+ });
+ let queries = captures
+ .grammars()
+ .iter()
+ .map(|grammar| grammar.highlights_query.as_ref().unwrap())
+ .collect::<Vec<_>>();
+ for capture in captures {
+ let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
+ if highlight_query_capture_names.contains(&name.as_str()) {
+ actual_ranges.push(capture.node.byte_range());
+ }
+ }
+
+ let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
+ assert_eq!(text, buffer.text());
+ assert_eq!(actual_ranges, expected_ranges);
+ }
+
+ fn edit_buffer(buffer: &mut Buffer, marked_string: &str) {
+ let old_text = buffer.text();
+ let (new_text, mut ranges) = marked_text_ranges(marked_string, false);
+ if ranges.is_empty() {
+ ranges.push(0..new_text.len());
+ }
+
+ assert_eq!(
+ old_text[..ranges[0].start],
+ new_text[..ranges[0].start],
+ "invalid edit"
+ );
+
+ let mut delta = 0;
+ let mut edits = Vec::new();
+ let mut ranges = ranges.into_iter().peekable();
+
+ while let Some(inserted_range) = ranges.next() {
+ let new_start = inserted_range.start;
+ let old_start = (new_start as isize - delta) as usize;
+
+ let following_text = if let Some(next_range) = ranges.peek() {
+ &new_text[inserted_range.end..next_range.start]
+ } else {
+ &new_text[inserted_range.end..]
+ };
+
+ let inserted_len = inserted_range.len();
+ let deleted_len = old_text[old_start..]
+ .find(following_text)
+ .expect("invalid edit");
+
+ let old_range = old_start..old_start + deleted_len;
+ edits.push((old_range, new_text[inserted_range].to_string()));
+ delta += inserted_len as isize - deleted_len as isize;
+ }
+
+ assert_eq!(
+ old_text.len() as isize + delta,
+ new_text.len() as isize,
+ "invalid edit"
+ );
+
+ buffer.edit(edits);
+ }
+
+ pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
+ let mut last_part_end = 0;
+ for part in parts {
+ if let Some(start_ix) = text[last_part_end..].find(part) {
+ last_part_end = start_ix + part.len();
+ } else {
+ return false;
+ }
+ }
+ true
+ }
+}
@@ -999,6 +999,7 @@ fn test_autoindent_language_without_indents_query(cx: &mut MutableAppContext) {
Arc::new(Language::new(
LanguageConfig {
name: "Markdown".into(),
+ auto_indent_using_last_non_empty_line: false,
..Default::default()
},
Some(tree_sitter_json::language()),
@@ -1437,7 +1438,9 @@ fn json_lang() -> Language {
fn get_tree_sexp(buffer: &ModelHandle<Buffer>, cx: &gpui::TestAppContext) -> String {
buffer.read_with(cx, |buffer, _| {
- buffer.syntax_tree().unwrap().root_node().to_sexp()
+ let snapshot = buffer.snapshot();
+ let layers = snapshot.syntax.layers(buffer.as_text_snapshot());
+ layers[0].2.to_sexp()
})
}
@@ -2067,6 +2067,7 @@ impl Project {
let full_path = buffer.read(cx).file()?.full_path(cx);
let language = self.languages.select_language(&full_path)?;
buffer.update(cx, |buffer, cx| {
+ buffer.set_language_registry(self.languages.clone());
buffer.set_language(Some(language.clone()), cx);
});
@@ -608,9 +608,9 @@ where
impl<'a, F, T, S, U> Iterator for FilterCursor<'a, F, T, U>
where
- F: Fn(&T::Summary) -> bool,
+ F: FnMut(&T::Summary) -> bool,
T: Item<Summary = S>,
- S: Summary<Context = ()>,
+ S: Summary<Context = ()>, //Context for the summary must be unit type, as .next() doesn't take arguments
U: Dimension<'a, T::Summary>,
{
type Item = &'a T;
@@ -621,7 +621,7 @@ where
}
if let Some(item) = self.item() {
- self.cursor.next_internal(&self.filter_node, &());
+ self.cursor.next_internal(&mut self.filter_node, &());
Some(item)
} else {
None
@@ -168,6 +168,8 @@ impl<T: Item> SumTree<T> {
Cursor::new(self)
}
+ /// Note: If the summary type requires a non `()` context, then the filter cursor
+ /// that is returned cannot be used with Rust's iterators.
pub fn filter<'a, F, U>(&'a self, filter_node: F) -> FilterCursor<F, T, U>
where
F: FnMut(&T::Summary) -> bool,
@@ -382,6 +382,7 @@ struct Edits<'a, D: TextDimension, F: FnMut(&FragmentSummary) -> bool> {
old_end: D,
new_end: D,
range: Range<(&'a Locator, usize)>,
+ buffer_id: u64,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
@@ -1917,11 +1918,33 @@ impl BufferSnapshot {
self.edits_since_in_range(since, Anchor::MIN..Anchor::MAX)
}
+ pub fn anchored_edits_since<'a, D>(
+ &'a self,
+ since: &'a clock::Global,
+ ) -> impl 'a + Iterator<Item = (Edit<D>, Range<Anchor>)>
+ where
+ D: TextDimension + Ord,
+ {
+ self.anchored_edits_since_in_range(since, Anchor::MIN..Anchor::MAX)
+ }
+
pub fn edits_since_in_range<'a, D>(
&'a self,
since: &'a clock::Global,
range: Range<Anchor>,
) -> impl 'a + Iterator<Item = Edit<D>>
+ where
+ D: TextDimension + Ord,
+ {
+ self.anchored_edits_since_in_range(since, range)
+ .map(|item| item.0)
+ }
+
+ pub fn anchored_edits_since_in_range<'a, D>(
+ &'a self,
+ since: &'a clock::Global,
+ range: Range<Anchor>,
+ ) -> impl 'a + Iterator<Item = (Edit<D>, Range<Anchor>)>
where
D: TextDimension + Ord,
{
@@ -1961,6 +1984,7 @@ impl BufferSnapshot {
old_end: Default::default(),
new_end: Default::default(),
range: (start_fragment_id, range.start.offset)..(end_fragment_id, range.end.offset),
+ buffer_id: self.remote_id,
}
}
}
@@ -2019,10 +2043,10 @@ impl<'a> RopeBuilder<'a> {
}
impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator for Edits<'a, D, F> {
- type Item = Edit<D>;
+ type Item = (Edit<D>, Range<Anchor>);
fn next(&mut self) -> Option<Self::Item> {
- let mut pending_edit: Option<Edit<D>> = None;
+ let mut pending_edit: Option<Self::Item> = None;
let cursor = self.fragments_cursor.as_mut()?;
while let Some(fragment) = cursor.item() {
@@ -2041,11 +2065,25 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo
if pending_edit
.as_ref()
- .map_or(false, |change| change.new.end < self.new_end)
+ .map_or(false, |(change, _)| change.new.end < self.new_end)
{
break;
}
+ let timestamp = fragment.insertion_timestamp.local();
+ let start_anchor = Anchor {
+ timestamp,
+ offset: fragment.insertion_offset,
+ bias: Bias::Right,
+ buffer_id: Some(self.buffer_id),
+ };
+ let end_anchor = Anchor {
+ timestamp,
+ offset: fragment.insertion_offset + fragment.len,
+ bias: Bias::Left,
+ buffer_id: Some(self.buffer_id),
+ };
+
if !fragment.was_visible(self.since, self.undos) && fragment.visible {
let mut visible_end = cursor.end(&None).visible;
if fragment.id == *self.range.end.0 {
@@ -2058,13 +2096,17 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo
let fragment_summary = self.visible_cursor.summary(visible_end);
let mut new_end = self.new_end.clone();
new_end.add_assign(&fragment_summary);
- if let Some(pending_edit) = pending_edit.as_mut() {
- pending_edit.new.end = new_end.clone();
+ if let Some((edit, range)) = pending_edit.as_mut() {
+ edit.new.end = new_end.clone();
+ range.end = end_anchor;
} else {
- pending_edit = Some(Edit {
- old: self.old_end.clone()..self.old_end.clone(),
- new: self.new_end.clone()..new_end.clone(),
- });
+ pending_edit = Some((
+ Edit {
+ old: self.old_end.clone()..self.old_end.clone(),
+ new: self.new_end.clone()..new_end.clone(),
+ },
+ start_anchor..end_anchor,
+ ));
}
self.new_end = new_end;
@@ -2083,13 +2125,17 @@ impl<'a, D: TextDimension + Ord, F: FnMut(&FragmentSummary) -> bool> Iterator fo
let fragment_summary = self.deleted_cursor.summary(deleted_end);
let mut old_end = self.old_end.clone();
old_end.add_assign(&fragment_summary);
- if let Some(pending_edit) = pending_edit.as_mut() {
- pending_edit.old.end = old_end.clone();
+ if let Some((edit, range)) = pending_edit.as_mut() {
+ edit.old.end = old_end.clone();
+ range.end = end_anchor;
} else {
- pending_edit = Some(Edit {
- old: self.old_end.clone()..old_end.clone(),
- new: self.new_end.clone()..self.new_end.clone(),
- });
+ pending_edit = Some((
+ Edit {
+ old: self.old_end.clone()..old_end.clone(),
+ new: self.new_end.clone()..self.new_end.clone(),
+ },
+ start_anchor..end_anchor,
+ ));
}
self.old_end = old_end;
@@ -2435,7 +2481,7 @@ impl ToOffset for PointUtf16 {
impl ToOffset for usize {
fn to_offset<'a>(&self, snapshot: &BufferSnapshot) -> usize {
- assert!(*self <= snapshot.len(), "offset is out of range");
+ assert!(*self <= snapshot.len(), "offset {self} is out of range");
*self
}
}
@@ -134,6 +134,11 @@ pub(crate) fn language(
.with_outline_query(query.as_ref())
.expect("failed to load outline query");
}
+ if let Some(query) = load_query(name, "/injections") {
+ language = language
+ .with_injection_query(query.as_ref())
+ .expect("failed to load injection query");
+ }
if let Some(lsp_adapter) = lsp_adapter {
language = language.with_lsp_adapter(lsp_adapter)
}
@@ -0,0 +1,7 @@
+(preproc_def
+ value: (preproc_arg) @content
+ (#set! "language" "c"))
+
+(preproc_function_def
+ value: (preproc_arg) @content
+ (#set! "language" "c"))
@@ -1,3 +1,5 @@
+(identifier) @variable
+
(call_expression
function: (qualified_identifier
name: (identifier) @function))
@@ -34,8 +36,6 @@
(auto) @type
(type_identifier) @type
-(identifier) @variable
-
((identifier) @constant
(#match? @constant "^[A-Z][A-Z\\d_]*$"))
@@ -0,0 +1,7 @@
+(preproc_def
+ value: (preproc_arg) @content
+ (#set! "language" "c++"))
+
+(preproc_function_def
+ value: (preproc_arg) @content
+ (#set! "language" "c++"))
@@ -1,6 +1,6 @@
(type_identifier) @type
(primitive_type) @type.builtin
-
+(self) @variable.builtin
(field_identifier) @property
(call_expression
@@ -15,6 +15,16 @@
(function_item name: (identifier) @function.definition)
(function_signature_item name: (identifier) @function.definition)
+(macro_invocation
+ macro: [
+ (identifier) @function.special
+ (scoped_identifier
+ name: (identifier) @function.special)
+ ])
+
+(macro_definition
+ name: (identifier) @function.special.definition)
+
; Identifier conventions
; Assume uppercase names are enum constructors
@@ -71,6 +81,7 @@
"mod"
"move"
"pub"
+ "ref"
"return"
"static"
"struct"
@@ -91,6 +102,13 @@
(char_literal)
] @string
+[
+ (integer_literal)
+ (float_literal)
+] @number
+
+(boolean_literal) @constant
+
[
(line_comment)
(block_comment)
@@ -0,0 +1,7 @@
+(macro_invocation
+ (token_tree) @content
+ (#set! "language" "rust"))
+
+(macro_rule
+ (token_tree) @content
+ (#set! "language" "rust"))