1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use lazy_static::lazy_static;
4use parking_lot::Mutex;
5use std::{
6 borrow::Cow,
7 cell::RefCell,
8 cmp::{Ordering, Reverse},
9 collections::BinaryHeap,
10 ops::{Deref, DerefMut, Range},
11 sync::Arc,
12};
13use sum_tree::{Bias, SeekTarget, SumTree};
14use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
15use tree_sitter::{
16 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
17};
18
19thread_local! {
20 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
21}
22
23lazy_static! {
24 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
25}
26
27#[derive(Default)]
28pub struct SyntaxMap {
29 parsed_version: clock::Global,
30 interpolated_version: clock::Global,
31 snapshot: SyntaxSnapshot,
32 language_registry: Option<Arc<LanguageRegistry>>,
33}
34
35#[derive(Clone, Default)]
36pub struct SyntaxSnapshot {
37 layers: SumTree<SyntaxLayer>,
38}
39
40#[derive(Default)]
41pub struct SyntaxMapCaptures<'a> {
42 layers: Vec<SyntaxMapCapturesLayer<'a>>,
43 active_layer_count: usize,
44 grammars: Vec<&'a Grammar>,
45}
46
47#[derive(Default)]
48pub struct SyntaxMapMatches<'a> {
49 layers: Vec<SyntaxMapMatchesLayer<'a>>,
50 active_layer_count: usize,
51 grammars: Vec<&'a Grammar>,
52}
53
54#[derive(Debug)]
55pub struct SyntaxMapCapture<'a> {
56 pub depth: usize,
57 pub node: Node<'a>,
58 pub index: u32,
59 pub grammar_index: usize,
60}
61
62#[derive(Debug)]
63pub struct SyntaxMapMatch<'a> {
64 pub depth: usize,
65 pub pattern_index: usize,
66 pub captures: &'a [QueryCapture<'a>],
67 pub grammar_index: usize,
68}
69
70struct SyntaxMapCapturesLayer<'a> {
71 depth: usize,
72 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
73 next_capture: Option<QueryCapture<'a>>,
74 grammar_index: usize,
75 _query_cursor: QueryCursorHandle,
76}
77
78struct SyntaxMapMatchesLayer<'a> {
79 depth: usize,
80 next_pattern_index: usize,
81 next_captures: Vec<QueryCapture<'a>>,
82 has_next: bool,
83 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
84 grammar_index: usize,
85 _query_cursor: QueryCursorHandle,
86}
87
88#[derive(Clone)]
89struct SyntaxLayer {
90 depth: usize,
91 range: Range<Anchor>,
92 tree: tree_sitter::Tree,
93 language: Arc<Language>,
94 combined: bool,
95}
96
97#[derive(Debug)]
98pub struct SyntaxLayerInfo<'a> {
99 pub depth: usize,
100 pub node: Node<'a>,
101 pub language: &'a Arc<Language>,
102}
103
104#[derive(Debug, Clone)]
105struct SyntaxLayerSummary {
106 min_depth: usize,
107 max_depth: usize,
108 range: Range<Anchor>,
109 last_layer_range: Range<Anchor>,
110 last_layer_language: Option<usize>,
111}
112
113#[derive(Clone, Debug)]
114struct SyntaxLayerPosition {
115 depth: usize,
116 range: Range<Anchor>,
117 language: Option<usize>,
118}
119
120#[derive(Clone, Debug)]
121struct DepthAndMaxPosition(usize, Anchor);
122
123#[derive(Clone, Debug)]
124struct SyntaxLayerPositionBeforeChange {
125 position: SyntaxLayerPosition,
126 change: DepthAndMaxPosition,
127}
128
129struct ReparseStep {
130 depth: usize,
131 language: Arc<Language>,
132 range: Range<Anchor>,
133 included_ranges: Vec<tree_sitter::Range>,
134 mode: ReparseMode,
135}
136
137enum ReparseMode {
138 Single,
139 Combined {
140 parent_layer_range: Range<usize>,
141 parent_layer_changed_ranges: Vec<Range<usize>>,
142 },
143}
144
145#[derive(Debug, PartialEq, Eq)]
146struct ChangedRegion {
147 depth: usize,
148 range: Range<Anchor>,
149}
150
151#[derive(Default)]
152struct ChangeRegionSet(Vec<ChangedRegion>);
153
154struct TextProvider<'a>(&'a Rope);
155
156struct ByteChunks<'a>(text::Chunks<'a>);
157
158struct QueryCursorHandle(Option<QueryCursor>);
159
160impl SyntaxMap {
161 pub fn new() -> Self {
162 Self::default()
163 }
164
165 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
166 self.language_registry = Some(registry);
167 }
168
169 pub fn snapshot(&self) -> SyntaxSnapshot {
170 self.snapshot.clone()
171 }
172
173 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
174 self.language_registry.clone()
175 }
176
177 pub fn parsed_version(&self) -> clock::Global {
178 self.parsed_version.clone()
179 }
180
181 pub fn interpolate(&mut self, text: &BufferSnapshot) {
182 self.snapshot.interpolate(&self.interpolated_version, text);
183 self.interpolated_version = text.version.clone();
184 }
185
186 #[cfg(test)]
187 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
188 self.snapshot.reparse(
189 &self.parsed_version,
190 text,
191 self.language_registry.clone(),
192 language,
193 );
194 self.parsed_version = text.version.clone();
195 self.interpolated_version = text.version.clone();
196 }
197
198 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot, version: clock::Global) {
199 self.interpolated_version = version.clone();
200 self.parsed_version = version;
201 self.snapshot = snapshot;
202 }
203
204 pub fn clear(&mut self) {
205 self.snapshot = SyntaxSnapshot::default();
206 }
207}
208
209impl SyntaxSnapshot {
210 pub fn is_empty(&self) -> bool {
211 self.layers.is_empty()
212 }
213
214 pub fn interpolate(&mut self, from_version: &clock::Global, text: &BufferSnapshot) {
215 let edits = text
216 .anchored_edits_since::<(usize, Point)>(&from_version)
217 .collect::<Vec<_>>();
218 if edits.is_empty() {
219 return;
220 }
221
222 let mut layers = SumTree::new();
223 let mut first_edit_ix_for_depth = 0;
224 let mut prev_depth = 0;
225 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
226 cursor.next(text);
227
228 'outer: loop {
229 let depth = cursor.end(text).max_depth;
230 if depth > prev_depth {
231 first_edit_ix_for_depth = 0;
232 prev_depth = depth;
233 }
234
235 // Preserve any layers at this depth that precede the first edit.
236 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
237 let target = DepthAndMaxPosition(depth, edit_range.start);
238 if target.cmp(&cursor.start(), text).is_gt() {
239 let slice = cursor.slice(&target, Bias::Left, text);
240 layers.push_tree(slice, text);
241 }
242 }
243 // If this layer follows all of the edits, then preserve it and any
244 // subsequent layers at this same depth.
245 else if cursor.item().is_some() {
246 let slice = cursor.slice(
247 &SyntaxLayerPosition {
248 depth: depth + 1,
249 range: Anchor::MIN..Anchor::MAX,
250 language: None,
251 },
252 Bias::Left,
253 text,
254 );
255 layers.push_tree(slice, text);
256 continue;
257 };
258
259 let layer = if let Some(layer) = cursor.item() {
260 layer
261 } else {
262 break;
263 };
264 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
265
266 // Ignore edits that end before the start of this layer, and don't consider them
267 // for any subsequent layers at this same depth.
268 loop {
269 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
270 if edit_range.end.cmp(&layer.range.start, text).is_le() {
271 first_edit_ix_for_depth += 1;
272 } else {
273 break;
274 }
275 } else {
276 continue 'outer;
277 }
278 }
279
280 let mut layer = layer.clone();
281 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
282 // Ignore any edits that follow this layer.
283 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
284 break;
285 }
286
287 // Apply any edits that intersect this layer to the layer's syntax tree.
288 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
289 tree_sitter::InputEdit {
290 start_byte: edit.new.start.0 - start_byte,
291 old_end_byte: edit.new.start.0 - start_byte
292 + (edit.old.end.0 - edit.old.start.0),
293 new_end_byte: edit.new.end.0 - start_byte,
294 start_position: (edit.new.start.1 - start_point).to_ts_point(),
295 old_end_position: (edit.new.start.1 - start_point
296 + (edit.old.end.1 - edit.old.start.1))
297 .to_ts_point(),
298 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
299 }
300 } else {
301 let node = layer.tree.root_node();
302 tree_sitter::InputEdit {
303 start_byte: 0,
304 old_end_byte: node.end_byte(),
305 new_end_byte: 0,
306 start_position: Default::default(),
307 old_end_position: node.end_position(),
308 new_end_position: Default::default(),
309 }
310 };
311
312 layer.tree.edit(&tree_edit);
313 }
314
315 debug_assert!(
316 layer.tree.root_node().end_byte() <= text.len(),
317 "tree's size {}, is larger than text size {}",
318 layer.tree.root_node().end_byte(),
319 text.len(),
320 );
321
322 layers.push(layer, text);
323 cursor.next(text);
324 }
325
326 layers.push_tree(cursor.suffix(&text), &text);
327 drop(cursor);
328 self.layers = layers;
329 }
330
331 pub fn reparse(
332 &mut self,
333 from_version: &clock::Global,
334 text: &BufferSnapshot,
335 registry: Option<Arc<LanguageRegistry>>,
336 language: Arc<Language>,
337 ) {
338 let edits = text.edits_since::<usize>(from_version).collect::<Vec<_>>();
339 let max_depth = self.layers.summary().max_depth;
340 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
341 cursor.next(&text);
342 let mut layers = SumTree::new();
343
344 let mut changed_regions = ChangeRegionSet::default();
345 let mut queue = BinaryHeap::new();
346 let mut combined_injection_ranges = HashMap::default();
347 queue.push(ReparseStep {
348 depth: 0,
349 language: language.clone(),
350 included_ranges: vec![tree_sitter::Range {
351 start_byte: 0,
352 end_byte: text.len(),
353 start_point: Point::zero().to_ts_point(),
354 end_point: text.max_point().to_ts_point(),
355 }],
356 range: Anchor::MIN..Anchor::MAX,
357 mode: ReparseMode::Single,
358 });
359
360 loop {
361 let step = queue.pop();
362 let target = if let Some(step) = &step {
363 SyntaxLayerPosition {
364 depth: step.depth,
365 range: step.range.clone(),
366 language: step.language.id(),
367 }
368 } else {
369 SyntaxLayerPosition {
370 depth: max_depth + 1,
371 range: Anchor::MAX..Anchor::MAX,
372 language: None,
373 }
374 };
375
376 let mut done = cursor.item().is_none();
377 while !done && target.cmp(&cursor.end(text), &text).is_gt() {
378 done = true;
379
380 let bounded_target = SyntaxLayerPositionBeforeChange {
381 position: target.clone(),
382 change: changed_regions.start_position(),
383 };
384 if bounded_target.cmp(&cursor.start(), &text).is_gt() {
385 let slice = cursor.slice(&bounded_target, Bias::Left, text);
386 if !slice.is_empty() {
387 layers.push_tree(slice, &text);
388 if changed_regions.prune(cursor.end(text), text) {
389 done = false;
390 }
391 }
392 }
393
394 while target.cmp(&cursor.end(text), text).is_gt() {
395 let Some(layer) = cursor.item() else { break };
396
397 if changed_regions.intersects(&layer, text) {
398 changed_regions.insert(
399 ChangedRegion {
400 depth: layer.depth + 1,
401 range: layer.range.clone(),
402 },
403 text,
404 );
405 } else {
406 layers.push(layer.clone(), text);
407 }
408
409 cursor.next(text);
410 if changed_regions.prune(cursor.end(text), text) {
411 done = false;
412 }
413 }
414 }
415
416 let Some(step) = step else { break };
417 let (step_start_byte, step_start_point) =
418 step.range.start.summary::<(usize, Point)>(text);
419 let step_end_byte = step.range.end.to_offset(text);
420 let Some(grammar) = step.language.grammar.as_deref() else { continue };
421
422 let mut old_layer = cursor.item();
423 if let Some(layer) = old_layer {
424 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
425 && layer.language.id() == step.language.id()
426 {
427 cursor.next(&text);
428 } else {
429 old_layer = None;
430 }
431 }
432
433 let mut combined = false;
434 let mut included_ranges = step.included_ranges;
435
436 let tree;
437 let changed_ranges;
438 if let Some(old_layer) = old_layer {
439 if let ReparseMode::Combined {
440 parent_layer_changed_ranges,
441 ..
442 } = step.mode
443 {
444 combined = true;
445 included_ranges = splice_included_ranges(
446 old_layer.tree.included_ranges(),
447 &parent_layer_changed_ranges,
448 &included_ranges,
449 );
450 }
451
452 tree = parse_text(
453 grammar,
454 text.as_rope(),
455 step_start_byte,
456 step_start_point,
457 included_ranges,
458 Some(old_layer.tree.clone()),
459 );
460 changed_ranges = join_ranges(
461 edits
462 .iter()
463 .map(|e| e.new.clone())
464 .filter(|range| range.start < step_end_byte && range.end > step_start_byte),
465 old_layer
466 .tree
467 .changed_ranges(&tree)
468 .map(|r| step_start_byte + r.start_byte..step_start_byte + r.end_byte),
469 );
470 } else {
471 tree = parse_text(
472 grammar,
473 text.as_rope(),
474 step_start_byte,
475 step_start_point,
476 included_ranges,
477 None,
478 );
479 changed_ranges = vec![step_start_byte..step_end_byte];
480 }
481
482 layers.push(
483 SyntaxLayer {
484 depth: step.depth,
485 range: step.range,
486 tree: tree.clone(),
487 language: language.clone(),
488 combined,
489 },
490 &text,
491 );
492
493 if let (Some((config, registry)), false) = (
494 grammar.injection_config.as_ref().zip(registry.as_ref()),
495 changed_ranges.is_empty(),
496 ) {
497 for range in &changed_ranges {
498 changed_regions.insert(
499 ChangedRegion {
500 depth: step.depth + 1,
501 range: text.anchor_before(range.start)..text.anchor_after(range.end),
502 },
503 text,
504 );
505 }
506 get_injections(
507 config,
508 text,
509 tree.root_node_with_offset(step_start_byte, step_start_point.to_ts_point()),
510 registry,
511 step.depth + 1,
512 &changed_ranges,
513 &mut combined_injection_ranges,
514 &mut queue,
515 );
516 }
517 }
518
519 drop(cursor);
520 self.layers = layers;
521 }
522
523 pub fn single_tree_captures<'a>(
524 range: Range<usize>,
525 text: &'a Rope,
526 tree: &'a Tree,
527 language: &'a Arc<Language>,
528 query: fn(&Grammar) -> Option<&Query>,
529 ) -> SyntaxMapCaptures<'a> {
530 SyntaxMapCaptures::new(
531 range.clone(),
532 text,
533 [SyntaxLayerInfo {
534 language,
535 depth: 0,
536 node: tree.root_node(),
537 }]
538 .into_iter(),
539 query,
540 )
541 }
542
543 pub fn captures<'a>(
544 &'a self,
545 range: Range<usize>,
546 buffer: &'a BufferSnapshot,
547 query: fn(&Grammar) -> Option<&Query>,
548 ) -> SyntaxMapCaptures {
549 SyntaxMapCaptures::new(
550 range.clone(),
551 buffer.as_rope(),
552 self.layers_for_range(range, buffer).into_iter(),
553 query,
554 )
555 }
556
557 pub fn matches<'a>(
558 &'a self,
559 range: Range<usize>,
560 buffer: &'a BufferSnapshot,
561 query: fn(&Grammar) -> Option<&Query>,
562 ) -> SyntaxMapMatches {
563 SyntaxMapMatches::new(
564 range.clone(),
565 buffer.as_rope(),
566 self.layers_for_range(range, buffer).into_iter(),
567 query,
568 )
569 }
570
571 #[cfg(test)]
572 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
573 self.layers_for_range(0..buffer.len(), buffer).collect()
574 }
575
576 pub fn layers_for_range<'a, T: ToOffset>(
577 &'a self,
578 range: Range<T>,
579 buffer: &'a BufferSnapshot,
580 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
581 let start = buffer.anchor_before(range.start.to_offset(buffer));
582 let end = buffer.anchor_after(range.end.to_offset(buffer));
583
584 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
585 if summary.max_depth > summary.min_depth {
586 true
587 } else {
588 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
589 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
590 !is_before_start && !is_after_end
591 }
592 });
593
594 cursor.next(buffer);
595 std::iter::from_fn(move || {
596 if let Some(layer) = cursor.item() {
597 let info = SyntaxLayerInfo {
598 language: &layer.language,
599 depth: layer.depth,
600 node: layer.tree.root_node_with_offset(
601 layer.range.start.to_offset(buffer),
602 layer.range.start.to_point(buffer).to_ts_point(),
603 ),
604 };
605 cursor.next(buffer);
606 Some(info)
607 } else {
608 None
609 }
610 })
611 }
612}
613
614impl<'a> SyntaxMapCaptures<'a> {
615 fn new(
616 range: Range<usize>,
617 text: &'a Rope,
618 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
619 query: fn(&Grammar) -> Option<&Query>,
620 ) -> Self {
621 let mut result = Self {
622 layers: Vec::new(),
623 grammars: Vec::new(),
624 active_layer_count: 0,
625 };
626 for SyntaxLayerInfo {
627 language,
628 depth,
629 node,
630 } in layers
631 {
632 let grammar = match &language.grammar {
633 Some(grammer) => grammer,
634 None => continue,
635 };
636 let query = match query(&grammar) {
637 Some(query) => query,
638 None => continue,
639 };
640
641 let mut query_cursor = QueryCursorHandle::new();
642
643 // TODO - add a Tree-sitter API to remove the need for this.
644 let cursor = unsafe {
645 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
646 };
647
648 cursor.set_byte_range(range.clone());
649 let captures = cursor.captures(query, node, TextProvider(text));
650 let grammar_index = result
651 .grammars
652 .iter()
653 .position(|g| g.id == grammar.id())
654 .unwrap_or_else(|| {
655 result.grammars.push(grammar);
656 result.grammars.len() - 1
657 });
658 let mut layer = SyntaxMapCapturesLayer {
659 depth,
660 grammar_index,
661 next_capture: None,
662 captures,
663 _query_cursor: query_cursor,
664 };
665
666 layer.advance();
667 if layer.next_capture.is_some() {
668 let key = layer.sort_key();
669 let ix = match result.layers[..result.active_layer_count]
670 .binary_search_by_key(&key, |layer| layer.sort_key())
671 {
672 Ok(ix) | Err(ix) => ix,
673 };
674 result.layers.insert(ix, layer);
675 result.active_layer_count += 1;
676 } else {
677 result.layers.push(layer);
678 }
679 }
680
681 result
682 }
683
684 pub fn grammars(&self) -> &[&'a Grammar] {
685 &self.grammars
686 }
687
688 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
689 let layer = self.layers[..self.active_layer_count].first()?;
690 let capture = layer.next_capture?;
691 Some(SyntaxMapCapture {
692 depth: layer.depth,
693 grammar_index: layer.grammar_index,
694 index: capture.index,
695 node: capture.node,
696 })
697 }
698
699 pub fn advance(&mut self) -> bool {
700 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
701 layer
702 } else {
703 return false;
704 };
705
706 layer.advance();
707 if layer.next_capture.is_some() {
708 let key = layer.sort_key();
709 let i = 1 + self.layers[1..self.active_layer_count]
710 .iter()
711 .position(|later_layer| key < later_layer.sort_key())
712 .unwrap_or(self.active_layer_count - 1);
713 self.layers[0..i].rotate_left(1);
714 } else {
715 self.layers[0..self.active_layer_count].rotate_left(1);
716 self.active_layer_count -= 1;
717 }
718
719 true
720 }
721
722 pub fn set_byte_range(&mut self, range: Range<usize>) {
723 for layer in &mut self.layers {
724 layer.captures.set_byte_range(range.clone());
725 if let Some(capture) = &layer.next_capture {
726 if capture.node.end_byte() > range.start {
727 continue;
728 }
729 }
730 layer.advance();
731 }
732 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
733 self.active_layer_count = self
734 .layers
735 .iter()
736 .position(|layer| layer.next_capture.is_none())
737 .unwrap_or(self.layers.len());
738 }
739}
740
741impl<'a> SyntaxMapMatches<'a> {
742 fn new(
743 range: Range<usize>,
744 text: &'a Rope,
745 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
746 query: fn(&Grammar) -> Option<&Query>,
747 ) -> Self {
748 let mut result = Self::default();
749 for SyntaxLayerInfo {
750 language,
751 depth,
752 node,
753 } in layers
754 {
755 let grammar = match &language.grammar {
756 Some(grammer) => grammer,
757 None => continue,
758 };
759 let query = match query(&grammar) {
760 Some(query) => query,
761 None => continue,
762 };
763
764 let mut query_cursor = QueryCursorHandle::new();
765
766 // TODO - add a Tree-sitter API to remove the need for this.
767 let cursor = unsafe {
768 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
769 };
770
771 cursor.set_byte_range(range.clone());
772 let matches = cursor.matches(query, node, TextProvider(text));
773 let grammar_index = result
774 .grammars
775 .iter()
776 .position(|g| g.id == grammar.id())
777 .unwrap_or_else(|| {
778 result.grammars.push(grammar);
779 result.grammars.len() - 1
780 });
781 let mut layer = SyntaxMapMatchesLayer {
782 depth,
783 grammar_index,
784 matches,
785 next_pattern_index: 0,
786 next_captures: Vec::new(),
787 has_next: false,
788 _query_cursor: query_cursor,
789 };
790
791 layer.advance();
792 if layer.has_next {
793 let key = layer.sort_key();
794 let ix = match result.layers[..result.active_layer_count]
795 .binary_search_by_key(&key, |layer| layer.sort_key())
796 {
797 Ok(ix) | Err(ix) => ix,
798 };
799 result.layers.insert(ix, layer);
800 result.active_layer_count += 1;
801 } else {
802 result.layers.push(layer);
803 }
804 }
805 result
806 }
807
808 pub fn grammars(&self) -> &[&'a Grammar] {
809 &self.grammars
810 }
811
812 pub fn peek(&self) -> Option<SyntaxMapMatch> {
813 let layer = self.layers.first()?;
814 if !layer.has_next {
815 return None;
816 }
817 Some(SyntaxMapMatch {
818 depth: layer.depth,
819 grammar_index: layer.grammar_index,
820 pattern_index: layer.next_pattern_index,
821 captures: &layer.next_captures,
822 })
823 }
824
825 pub fn advance(&mut self) -> bool {
826 let layer = if let Some(layer) = self.layers.first_mut() {
827 layer
828 } else {
829 return false;
830 };
831
832 layer.advance();
833 if layer.has_next {
834 let key = layer.sort_key();
835 let i = 1 + self.layers[1..self.active_layer_count]
836 .iter()
837 .position(|later_layer| key < later_layer.sort_key())
838 .unwrap_or(self.active_layer_count - 1);
839 self.layers[0..i].rotate_left(1);
840 } else {
841 self.layers[0..self.active_layer_count].rotate_left(1);
842 self.active_layer_count -= 1;
843 }
844
845 true
846 }
847}
848
849impl<'a> SyntaxMapCapturesLayer<'a> {
850 fn advance(&mut self) {
851 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
852 }
853
854 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
855 if let Some(capture) = &self.next_capture {
856 let range = capture.node.byte_range();
857 (range.start, Reverse(range.end), self.depth)
858 } else {
859 (usize::MAX, Reverse(0), usize::MAX)
860 }
861 }
862}
863
864impl<'a> SyntaxMapMatchesLayer<'a> {
865 fn advance(&mut self) {
866 if let Some(mat) = self.matches.next() {
867 self.next_captures.clear();
868 self.next_captures.extend_from_slice(&mat.captures);
869 self.next_pattern_index = mat.pattern_index;
870 self.has_next = true;
871 } else {
872 self.has_next = false;
873 }
874 }
875
876 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
877 if self.has_next {
878 let captures = &self.next_captures;
879 if let Some((first, last)) = captures.first().zip(captures.last()) {
880 return (
881 first.node.start_byte(),
882 Reverse(last.node.end_byte()),
883 self.depth,
884 );
885 }
886 }
887 (usize::MAX, Reverse(0), usize::MAX)
888 }
889}
890
891impl<'a> Iterator for SyntaxMapCaptures<'a> {
892 type Item = SyntaxMapCapture<'a>;
893
894 fn next(&mut self) -> Option<Self::Item> {
895 let result = self.peek();
896 self.advance();
897 result
898 }
899}
900
901fn join_ranges(
902 a: impl Iterator<Item = Range<usize>>,
903 b: impl Iterator<Item = Range<usize>>,
904) -> Vec<Range<usize>> {
905 let mut result = Vec::<Range<usize>>::new();
906 let mut a = a.peekable();
907 let mut b = b.peekable();
908 loop {
909 let range = match (a.peek(), b.peek()) {
910 (Some(range_a), Some(range_b)) => {
911 if range_a.start < range_b.start {
912 a.next().unwrap()
913 } else {
914 b.next().unwrap()
915 }
916 }
917 (None, Some(_)) => b.next().unwrap(),
918 (Some(_), None) => a.next().unwrap(),
919 (None, None) => break,
920 };
921
922 if let Some(last) = result.last_mut() {
923 if range.start <= last.end {
924 last.end = last.end.max(range.end);
925 continue;
926 }
927 }
928 result.push(range);
929 }
930 result
931}
932
933fn parse_text(
934 grammar: &Grammar,
935 text: &Rope,
936 start_byte: usize,
937 start_point: Point,
938 mut ranges: Vec<tree_sitter::Range>,
939 old_tree: Option<Tree>,
940) -> Tree {
941 for range in &mut ranges {
942 range.start_byte -= start_byte;
943 range.end_byte -= start_byte;
944 range.start_point = (Point::from_ts_point(range.start_point) - start_point).to_ts_point();
945 range.end_point = (Point::from_ts_point(range.end_point) - start_point).to_ts_point();
946 }
947
948 PARSER.with(|parser| {
949 let mut parser = parser.borrow_mut();
950 let mut chunks = text.chunks_in_range(start_byte..text.len());
951 parser
952 .set_included_ranges(&ranges)
953 .expect("overlapping ranges");
954 parser
955 .set_language(grammar.ts_language)
956 .expect("incompatible grammar");
957 parser
958 .parse_with(
959 &mut move |offset, _| {
960 chunks.seek(start_byte + offset);
961 chunks.next().unwrap_or("").as_bytes()
962 },
963 old_tree.as_ref(),
964 )
965 .expect("invalid language")
966 })
967}
968
969fn get_injections(
970 config: &InjectionConfig,
971 text: &BufferSnapshot,
972 node: Node,
973 language_registry: &LanguageRegistry,
974 depth: usize,
975 changed_ranges: &[Range<usize>],
976 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
977 queue: &mut BinaryHeap<ReparseStep>,
978) -> bool {
979 let mut result = false;
980 let mut query_cursor = QueryCursorHandle::new();
981 let mut prev_match = None;
982
983 combined_injection_ranges.clear();
984 for query_range in changed_ranges {
985 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end);
986 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
987 let content_ranges = mat
988 .nodes_for_capture_index(config.content_capture_ix)
989 .map(|node| node.range())
990 .collect::<Vec<_>>();
991 if content_ranges.is_empty() {
992 continue;
993 }
994
995 // Avoid duplicate matches if two changed ranges intersect the same injection.
996 let content_range =
997 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
998 if let Some((last_pattern_ix, last_range)) = &prev_match {
999 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1000 continue;
1001 }
1002 }
1003 prev_match = Some((mat.pattern_index, content_range.clone()));
1004
1005 let combined = config.patterns[mat.pattern_index].combined;
1006 let language_name = config.patterns[mat.pattern_index]
1007 .language
1008 .as_ref()
1009 .map(|s| Cow::Borrowed(s.as_ref()))
1010 .or_else(|| {
1011 let ix = config.language_capture_ix?;
1012 let node = mat.nodes_for_capture_index(ix).next()?;
1013 Some(Cow::Owned(text.text_for_range(node.byte_range()).collect()))
1014 });
1015
1016 if let Some(language_name) = language_name {
1017 if let Some(language) = language_registry.get_language(language_name.as_ref()) {
1018 result = true;
1019 let range = text.anchor_before(content_range.start)
1020 ..text.anchor_after(content_range.end);
1021 if combined {
1022 combined_injection_ranges
1023 .entry(language.clone())
1024 .or_default()
1025 .extend(content_ranges);
1026 } else {
1027 queue.push(ReparseStep {
1028 depth,
1029 language,
1030 included_ranges: content_ranges,
1031 range,
1032 mode: ReparseMode::Single,
1033 });
1034 }
1035 }
1036 }
1037 }
1038 }
1039
1040 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1041 included_ranges.sort_unstable();
1042 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1043 queue.push(ReparseStep {
1044 depth,
1045 language,
1046 range,
1047 included_ranges,
1048 mode: ReparseMode::Combined {
1049 parent_layer_range: node.start_byte()..node.end_byte(),
1050 parent_layer_changed_ranges: changed_ranges.to_vec(),
1051 },
1052 })
1053 }
1054
1055 result
1056}
1057
1058fn splice_included_ranges(
1059 mut ranges: Vec<tree_sitter::Range>,
1060 changed_ranges: &[Range<usize>],
1061 new_ranges: &[tree_sitter::Range],
1062) -> Vec<tree_sitter::Range> {
1063 let mut changed_ranges = changed_ranges.into_iter().peekable();
1064 let mut new_ranges = new_ranges.into_iter().peekable();
1065 let mut ranges_ix = 0;
1066 loop {
1067 let new_range = new_ranges.peek();
1068 let mut changed_range = changed_ranges.peek();
1069
1070 // process changed ranges before any overlapping new ranges
1071 if let Some((changed, new)) = changed_range.zip(new_range) {
1072 if new.end_byte < changed.start {
1073 changed_range = None;
1074 }
1075 }
1076
1077 if let Some(changed) = changed_range {
1078 let start_ix = ranges_ix
1079 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1080 Ok(ix) | Err(ix) => ix,
1081 };
1082 let end_ix = ranges_ix
1083 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1084 Ok(ix) | Err(ix) => ix,
1085 };
1086 if end_ix > start_ix {
1087 ranges.splice(start_ix..end_ix, []);
1088 }
1089 changed_ranges.next();
1090 ranges_ix = start_ix;
1091 } else if let Some(new_range) = new_range {
1092 let ix = ranges_ix
1093 + match ranges[ranges_ix..]
1094 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1095 {
1096 Ok(ix) | Err(ix) => ix,
1097 };
1098 ranges.insert(ix, **new_range);
1099 new_ranges.next();
1100 ranges_ix = ix + 1;
1101 } else {
1102 break;
1103 }
1104 }
1105 ranges
1106}
1107
1108impl std::ops::Deref for SyntaxMap {
1109 type Target = SyntaxSnapshot;
1110
1111 fn deref(&self) -> &Self::Target {
1112 &self.snapshot
1113 }
1114}
1115
1116impl PartialEq for ReparseStep {
1117 fn eq(&self, _: &Self) -> bool {
1118 false
1119 }
1120}
1121
1122impl Eq for ReparseStep {}
1123
1124impl PartialOrd for ReparseStep {
1125 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1126 Some(self.cmp(&other))
1127 }
1128}
1129
1130impl Ord for ReparseStep {
1131 fn cmp(&self, other: &Self) -> Ordering {
1132 let range_a = self.range();
1133 let range_b = other.range();
1134 Ord::cmp(&other.depth, &self.depth)
1135 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1136 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1137 .then_with(|| self.language.id().cmp(&other.language.id()))
1138 }
1139}
1140
1141impl ReparseStep {
1142 fn range(&self) -> Range<usize> {
1143 if let ReparseMode::Combined {
1144 parent_layer_range, ..
1145 } = &self.mode
1146 {
1147 parent_layer_range.clone()
1148 } else {
1149 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1150 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1151 start..end
1152 }
1153 }
1154}
1155
1156impl ChangedRegion {
1157 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1158 let range_a = &self.range;
1159 let range_b = &other.range;
1160 Ord::cmp(&self.depth, &other.depth)
1161 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1162 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1163 }
1164}
1165
1166impl ChangeRegionSet {
1167 fn start_position(&self) -> DepthAndMaxPosition {
1168 self.0
1169 .first()
1170 .map_or(DepthAndMaxPosition(usize::MAX, Anchor::MAX), |region| {
1171 DepthAndMaxPosition(region.depth, region.range.start)
1172 })
1173 }
1174
1175 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1176 for region in &self.0 {
1177 if region.depth < layer.depth {
1178 continue;
1179 }
1180 if region.depth > layer.depth {
1181 break;
1182 }
1183 if region.range.end.cmp(&layer.range.start, text).is_le() {
1184 continue;
1185 }
1186 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1187 break;
1188 }
1189 return true;
1190 }
1191 false
1192 }
1193
1194 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1195 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1196 self.0.insert(ix, region);
1197 }
1198 }
1199
1200 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1201 let prev_len = self.0.len();
1202 self.0.retain(|region| {
1203 region.depth > summary.max_depth
1204 || (region.depth == summary.max_depth
1205 && region
1206 .range
1207 .end
1208 .cmp(&summary.last_layer_range.start, text)
1209 .is_gt())
1210 });
1211 self.0.len() < prev_len
1212 }
1213}
1214
1215impl Default for SyntaxLayerSummary {
1216 fn default() -> Self {
1217 Self {
1218 max_depth: 0,
1219 min_depth: 0,
1220 range: Anchor::MAX..Anchor::MIN,
1221 last_layer_range: Anchor::MIN..Anchor::MAX,
1222 last_layer_language: None,
1223 }
1224 }
1225}
1226
1227impl sum_tree::Summary for SyntaxLayerSummary {
1228 type Context = BufferSnapshot;
1229
1230 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1231 if other.max_depth > self.max_depth {
1232 self.max_depth = other.max_depth;
1233 self.range = other.range.clone();
1234 } else {
1235 if other.range.start.cmp(&self.range.start, buffer).is_lt() {
1236 self.range.start = other.range.start;
1237 }
1238 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1239 self.range.end = other.range.end;
1240 }
1241 }
1242 self.last_layer_range = other.last_layer_range.clone();
1243 self.last_layer_language = other.last_layer_language;
1244 }
1245}
1246
1247impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1248 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1249 Ord::cmp(&self.depth, &cursor_location.max_depth)
1250 .then_with(|| {
1251 self.range
1252 .start
1253 .cmp(&cursor_location.last_layer_range.start, buffer)
1254 })
1255 .then_with(|| {
1256 cursor_location
1257 .last_layer_range
1258 .end
1259 .cmp(&self.range.end, buffer)
1260 })
1261 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1262 }
1263}
1264
1265impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for DepthAndMaxPosition {
1266 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1267 Ord::cmp(&self.0, &cursor_location.max_depth)
1268 .then_with(|| self.1.cmp(&cursor_location.range.end, text))
1269 }
1270}
1271
1272impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1273 for SyntaxLayerPositionBeforeChange
1274{
1275 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1276 if self.change.cmp(cursor_location, buffer).is_le() {
1277 return Ordering::Less;
1278 } else {
1279 self.position.cmp(cursor_location, buffer)
1280 }
1281 }
1282}
1283
1284impl sum_tree::Item for SyntaxLayer {
1285 type Summary = SyntaxLayerSummary;
1286
1287 fn summary(&self) -> Self::Summary {
1288 SyntaxLayerSummary {
1289 min_depth: self.depth,
1290 max_depth: self.depth,
1291 range: self.range.clone(),
1292 last_layer_range: self.range.clone(),
1293 last_layer_language: self.language.id(),
1294 }
1295 }
1296}
1297
1298impl std::fmt::Debug for SyntaxLayer {
1299 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1300 f.debug_struct("SyntaxLayer")
1301 .field("depth", &self.depth)
1302 .field("range", &self.range)
1303 .field("tree", &self.tree)
1304 .finish()
1305 }
1306}
1307
1308impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1309 type I = ByteChunks<'a>;
1310
1311 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1312 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1313 }
1314}
1315
1316impl<'a> Iterator for ByteChunks<'a> {
1317 type Item = &'a [u8];
1318
1319 fn next(&mut self) -> Option<Self::Item> {
1320 self.0.next().map(str::as_bytes)
1321 }
1322}
1323
1324impl QueryCursorHandle {
1325 pub(crate) fn new() -> Self {
1326 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1327 cursor.set_match_limit(64);
1328 QueryCursorHandle(Some(cursor))
1329 }
1330}
1331
1332impl Deref for QueryCursorHandle {
1333 type Target = QueryCursor;
1334
1335 fn deref(&self) -> &Self::Target {
1336 self.0.as_ref().unwrap()
1337 }
1338}
1339
1340impl DerefMut for QueryCursorHandle {
1341 fn deref_mut(&mut self) -> &mut Self::Target {
1342 self.0.as_mut().unwrap()
1343 }
1344}
1345
1346impl Drop for QueryCursorHandle {
1347 fn drop(&mut self) {
1348 let mut cursor = self.0.take().unwrap();
1349 cursor.set_byte_range(0..usize::MAX);
1350 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1351 QUERY_CURSORS.lock().push(cursor)
1352 }
1353}
1354
1355pub(crate) trait ToTreeSitterPoint {
1356 fn to_ts_point(self) -> tree_sitter::Point;
1357 fn from_ts_point(point: tree_sitter::Point) -> Self;
1358}
1359
1360impl ToTreeSitterPoint for Point {
1361 fn to_ts_point(self) -> tree_sitter::Point {
1362 tree_sitter::Point::new(self.row as usize, self.column as usize)
1363 }
1364
1365 fn from_ts_point(point: tree_sitter::Point) -> Self {
1366 Point::new(point.row as u32, point.column as u32)
1367 }
1368}
1369
1370#[cfg(test)]
1371mod tests {
1372 use super::*;
1373 use crate::LanguageConfig;
1374 use rand::rngs::StdRng;
1375 use std::env;
1376 use text::Buffer;
1377 use unindent::Unindent as _;
1378 use util::test::marked_text_ranges;
1379
1380 #[test]
1381 fn test_splice_included_ranges() {
1382 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1383
1384 let new_ranges = splice_included_ranges(
1385 ranges.clone(),
1386 &[54..56, 58..68],
1387 &[ts_range(50..54), ts_range(59..67)],
1388 );
1389 assert_eq!(
1390 new_ranges,
1391 &[
1392 ts_range(20..30),
1393 ts_range(50..54),
1394 ts_range(59..67),
1395 ts_range(80..90),
1396 ]
1397 );
1398
1399 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1400 assert_eq!(
1401 new_ranges,
1402 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1403 );
1404
1405 let new_ranges =
1406 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1407 assert_eq!(
1408 new_ranges,
1409 &[
1410 ts_range(0..2),
1411 ts_range(20..30),
1412 ts_range(50..60),
1413 ts_range(70..75),
1414 ts_range(80..90)
1415 ]
1416 );
1417
1418 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1419 tree_sitter::Range {
1420 start_byte: range.start,
1421 start_point: tree_sitter::Point {
1422 row: 0,
1423 column: range.start,
1424 },
1425 end_byte: range.end,
1426 end_point: tree_sitter::Point {
1427 row: 0,
1428 column: range.end,
1429 },
1430 }
1431 }
1432 }
1433
1434 #[gpui::test]
1435 fn test_syntax_map_layers_for_range() {
1436 let registry = Arc::new(LanguageRegistry::test());
1437 let language = Arc::new(rust_lang());
1438 registry.add(language.clone());
1439
1440 let mut buffer = Buffer::new(
1441 0,
1442 0,
1443 r#"
1444 fn a() {
1445 assert_eq!(
1446 b(vec![C {}]),
1447 vec![d.e],
1448 );
1449 println!("{}", f(|_| true));
1450 }
1451 "#
1452 .unindent(),
1453 );
1454
1455 let mut syntax_map = SyntaxMap::new();
1456 syntax_map.set_language_registry(registry.clone());
1457 syntax_map.reparse(language.clone(), &buffer);
1458
1459 assert_layers_for_range(
1460 &syntax_map,
1461 &buffer,
1462 Point::new(2, 0)..Point::new(2, 0),
1463 &[
1464 "...(function_item ... (block (expression_statement (macro_invocation...",
1465 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1466 ],
1467 );
1468 assert_layers_for_range(
1469 &syntax_map,
1470 &buffer,
1471 Point::new(2, 14)..Point::new(2, 16),
1472 &[
1473 "...(function_item ...",
1474 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1475 "...(array_expression (struct_expression ...",
1476 ],
1477 );
1478 assert_layers_for_range(
1479 &syntax_map,
1480 &buffer,
1481 Point::new(3, 14)..Point::new(3, 16),
1482 &[
1483 "...(function_item ...",
1484 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1485 "...(array_expression (field_expression ...",
1486 ],
1487 );
1488 assert_layers_for_range(
1489 &syntax_map,
1490 &buffer,
1491 Point::new(5, 12)..Point::new(5, 16),
1492 &[
1493 "...(function_item ...",
1494 "...(call_expression ... (arguments (closure_expression ...",
1495 ],
1496 );
1497
1498 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1499 let macro_name_range = range_for_text(&buffer, "vec!");
1500 buffer.edit([(macro_name_range, "&")]);
1501 syntax_map.interpolate(&buffer);
1502 syntax_map.reparse(language.clone(), &buffer);
1503
1504 assert_layers_for_range(
1505 &syntax_map,
1506 &buffer,
1507 Point::new(2, 14)..Point::new(2, 16),
1508 &[
1509 "...(function_item ...",
1510 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1511 ],
1512 );
1513
1514 // Put the vec! macro back, adding back the syntactic layer.
1515 buffer.undo();
1516 syntax_map.interpolate(&buffer);
1517 syntax_map.reparse(language.clone(), &buffer);
1518
1519 assert_layers_for_range(
1520 &syntax_map,
1521 &buffer,
1522 Point::new(2, 14)..Point::new(2, 16),
1523 &[
1524 "...(function_item ...",
1525 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1526 "...(array_expression (struct_expression ...",
1527 ],
1528 );
1529 }
1530
1531 #[gpui::test]
1532 fn test_typing_multiple_new_injections() {
1533 let (buffer, syntax_map) = test_edit_sequence(&[
1534 "fn a() { dbg }",
1535 "fn a() { dbg«!» }",
1536 "fn a() { dbg!«()» }",
1537 "fn a() { dbg!(«b») }",
1538 "fn a() { dbg!(b«.») }",
1539 "fn a() { dbg!(b.«c») }",
1540 "fn a() { dbg!(b.c«()») }",
1541 "fn a() { dbg!(b.c(«vec»)) }",
1542 "fn a() { dbg!(b.c(vec«!»)) }",
1543 "fn a() { dbg!(b.c(vec!«[]»)) }",
1544 "fn a() { dbg!(b.c(vec![«d»])) }",
1545 "fn a() { dbg!(b.c(vec![d«.»])) }",
1546 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1547 ]);
1548
1549 assert_capture_ranges(
1550 &syntax_map,
1551 &buffer,
1552 &["field"],
1553 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1554 );
1555 }
1556
1557 #[gpui::test]
1558 fn test_pasting_new_injection_line_between_others() {
1559 let (buffer, syntax_map) = test_edit_sequence(&[
1560 "
1561 fn a() {
1562 b!(B {});
1563 c!(C {});
1564 d!(D {});
1565 e!(E {});
1566 f!(F {});
1567 g!(G {});
1568 }
1569 ",
1570 "
1571 fn a() {
1572 b!(B {});
1573 c!(C {});
1574 d!(D {});
1575 « h!(H {});
1576 » e!(E {});
1577 f!(F {});
1578 g!(G {});
1579 }
1580 ",
1581 ]);
1582
1583 assert_capture_ranges(
1584 &syntax_map,
1585 &buffer,
1586 &["struct"],
1587 "
1588 fn a() {
1589 b!(«B {}»);
1590 c!(«C {}»);
1591 d!(«D {}»);
1592 h!(«H {}»);
1593 e!(«E {}»);
1594 f!(«F {}»);
1595 g!(«G {}»);
1596 }
1597 ",
1598 );
1599 }
1600
1601 #[gpui::test]
1602 fn test_joining_injections_with_child_injections() {
1603 let (buffer, syntax_map) = test_edit_sequence(&[
1604 "
1605 fn a() {
1606 b!(
1607 c![one.two.three],
1608 d![four.five.six],
1609 );
1610 e!(
1611 f![seven.eight],
1612 );
1613 }
1614 ",
1615 "
1616 fn a() {
1617 b!(
1618 c![one.two.three],
1619 d![four.five.six],
1620 ˇ f![seven.eight],
1621 );
1622 }
1623 ",
1624 ]);
1625
1626 assert_capture_ranges(
1627 &syntax_map,
1628 &buffer,
1629 &["field"],
1630 "
1631 fn a() {
1632 b!(
1633 c![one.«two».«three»],
1634 d![four.«five».«six»],
1635 f![seven.«eight»],
1636 );
1637 }
1638 ",
1639 );
1640 }
1641
1642 #[gpui::test]
1643 fn test_editing_edges_of_injection() {
1644 test_edit_sequence(&[
1645 "
1646 fn a() {
1647 b!(c!())
1648 }
1649 ",
1650 "
1651 fn a() {
1652 «d»!(c!())
1653 }
1654 ",
1655 "
1656 fn a() {
1657 «e»d!(c!())
1658 }
1659 ",
1660 "
1661 fn a() {
1662 ed!«[»c!()«]»
1663 }
1664 ",
1665 ]);
1666 }
1667
1668 #[gpui::test]
1669 fn test_edits_preceding_and_intersecting_injection() {
1670 test_edit_sequence(&[
1671 //
1672 "const aaaaaaaaaaaa: B = c!(d(e.f));",
1673 "const aˇa: B = c!(d(eˇ));",
1674 ]);
1675 }
1676
1677 #[gpui::test]
1678 fn test_non_local_changes_create_injections() {
1679 test_edit_sequence(&[
1680 "
1681 // a! {
1682 static B: C = d;
1683 // }
1684 ",
1685 "
1686 ˇa! {
1687 static B: C = d;
1688 ˇ}
1689 ",
1690 ]);
1691 }
1692
1693 #[gpui::test]
1694 fn test_creating_many_injections_in_one_edit() {
1695 test_edit_sequence(&[
1696 "
1697 fn a() {
1698 one(Two::three(3));
1699 four(Five::six(6));
1700 seven(Eight::nine(9));
1701 }
1702 ",
1703 "
1704 fn a() {
1705 one«!»(Two::three(3));
1706 four«!»(Five::six(6));
1707 seven«!»(Eight::nine(9));
1708 }
1709 ",
1710 "
1711 fn a() {
1712 one!(Two::three«!»(3));
1713 four!(Five::six«!»(6));
1714 seven!(Eight::nine«!»(9));
1715 }
1716 ",
1717 ]);
1718 }
1719
1720 #[gpui::test]
1721 fn test_editing_across_injection_boundary() {
1722 test_edit_sequence(&[
1723 "
1724 fn one() {
1725 two();
1726 three!(
1727 three.four,
1728 five.six,
1729 );
1730 }
1731 ",
1732 "
1733 fn one() {
1734 two();
1735 th«irty_five![»
1736 three.four,
1737 five.six,
1738 « seven.eight,
1739 ];»
1740 }
1741 ",
1742 ]);
1743 }
1744
1745 #[gpui::test]
1746 fn test_removing_injection_by_replacing_across_boundary() {
1747 test_edit_sequence(&[
1748 "
1749 fn one() {
1750 two!(
1751 three.four,
1752 );
1753 }
1754 ",
1755 "
1756 fn one() {
1757 t«en
1758 .eleven(
1759 twelve,
1760 »
1761 three.four,
1762 );
1763 }
1764 ",
1765 ]);
1766 }
1767
1768 #[gpui::test(iterations = 100)]
1769 fn test_random_syntax_map_edits(mut rng: StdRng) {
1770 let operations = env::var("OPERATIONS")
1771 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
1772 .unwrap_or(10);
1773
1774 let text = r#"
1775 fn test_something() {
1776 let vec = vec![5, 1, 3, 8];
1777 assert_eq!(
1778 vec
1779 .into_iter()
1780 .map(|i| i * 2)
1781 .collect::<Vec<usize>>(),
1782 vec![
1783 5 * 2, 1 * 2, 3 * 2, 8 * 2
1784 ],
1785 );
1786 }
1787 "#
1788 .unindent()
1789 .repeat(2);
1790
1791 let registry = Arc::new(LanguageRegistry::test());
1792 let language = Arc::new(rust_lang());
1793 registry.add(language.clone());
1794 let mut buffer = Buffer::new(0, 0, text);
1795
1796 let mut syntax_map = SyntaxMap::new();
1797 syntax_map.set_language_registry(registry.clone());
1798 syntax_map.reparse(language.clone(), &buffer);
1799
1800 let mut reference_syntax_map = SyntaxMap::new();
1801 reference_syntax_map.set_language_registry(registry.clone());
1802
1803 log::info!("initial text:\n{}", buffer.text());
1804
1805 for _ in 0..operations {
1806 let prev_buffer = buffer.snapshot();
1807 let prev_syntax_map = syntax_map.snapshot();
1808
1809 buffer.randomly_edit(&mut rng, 3);
1810 log::info!("text:\n{}", buffer.text());
1811
1812 syntax_map.interpolate(&buffer);
1813 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
1814
1815 syntax_map.reparse(language.clone(), &buffer);
1816
1817 reference_syntax_map.clear();
1818 reference_syntax_map.reparse(language.clone(), &buffer);
1819 }
1820
1821 for i in 0..operations {
1822 let i = operations - i - 1;
1823 buffer.undo();
1824 log::info!("undoing operation {}", i);
1825 log::info!("text:\n{}", buffer.text());
1826
1827 syntax_map.interpolate(&buffer);
1828 syntax_map.reparse(language.clone(), &buffer);
1829
1830 reference_syntax_map.clear();
1831 reference_syntax_map.reparse(language.clone(), &buffer);
1832 assert_eq!(
1833 syntax_map.layers(&buffer).len(),
1834 reference_syntax_map.layers(&buffer).len(),
1835 "wrong number of layers after undoing edit {i}"
1836 );
1837 }
1838
1839 let layers = syntax_map.layers(&buffer);
1840 let reference_layers = reference_syntax_map.layers(&buffer);
1841 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
1842 {
1843 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
1844 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
1845 }
1846 }
1847
1848 fn check_interpolation(
1849 old_syntax_map: &SyntaxSnapshot,
1850 new_syntax_map: &SyntaxSnapshot,
1851 old_buffer: &BufferSnapshot,
1852 new_buffer: &BufferSnapshot,
1853 ) {
1854 let edits = new_buffer
1855 .edits_since::<usize>(&old_buffer.version())
1856 .collect::<Vec<_>>();
1857
1858 for (old_layer, new_layer) in old_syntax_map
1859 .layers
1860 .iter()
1861 .zip(new_syntax_map.layers.iter())
1862 {
1863 assert_eq!(old_layer.range, new_layer.range);
1864 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
1865 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
1866 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
1867 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
1868 let old_node = old_layer
1869 .tree
1870 .root_node_with_offset(old_start_byte, old_start_point);
1871 let new_node = new_layer
1872 .tree
1873 .root_node_with_offset(new_start_byte, new_start_point);
1874 check_node_edits(
1875 old_layer.depth,
1876 &old_layer.range,
1877 old_node,
1878 new_node,
1879 old_buffer,
1880 new_buffer,
1881 &edits,
1882 );
1883 }
1884
1885 fn check_node_edits(
1886 depth: usize,
1887 range: &Range<Anchor>,
1888 old_node: Node,
1889 new_node: Node,
1890 old_buffer: &BufferSnapshot,
1891 new_buffer: &BufferSnapshot,
1892 edits: &[text::Edit<usize>],
1893 ) {
1894 assert_eq!(old_node.kind(), new_node.kind());
1895
1896 let old_range = old_node.byte_range();
1897 let new_range = new_node.byte_range();
1898
1899 let is_edited = edits
1900 .iter()
1901 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
1902 if is_edited {
1903 assert!(
1904 new_node.has_changes(),
1905 concat!(
1906 "failed to mark node as edited.\n",
1907 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
1908 "node kind: {}, old node range: {:?}, new node range: {:?}",
1909 ),
1910 depth,
1911 range.to_offset(old_buffer),
1912 range.to_offset(new_buffer),
1913 new_node.kind(),
1914 old_range,
1915 new_range,
1916 );
1917 }
1918
1919 if !new_node.has_changes() {
1920 assert_eq!(
1921 old_buffer
1922 .text_for_range(old_range.clone())
1923 .collect::<String>(),
1924 new_buffer
1925 .text_for_range(new_range.clone())
1926 .collect::<String>(),
1927 concat!(
1928 "mismatched text for node\n",
1929 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
1930 "node kind: {}, old node range:{:?}, new node range:{:?}",
1931 ),
1932 depth,
1933 range.to_offset(old_buffer),
1934 range.to_offset(new_buffer),
1935 new_node.kind(),
1936 old_range,
1937 new_range,
1938 );
1939 }
1940
1941 for i in 0..new_node.child_count() {
1942 check_node_edits(
1943 depth,
1944 range,
1945 old_node.child(i).unwrap(),
1946 new_node.child(i).unwrap(),
1947 old_buffer,
1948 new_buffer,
1949 edits,
1950 )
1951 }
1952 }
1953 }
1954
1955 fn test_edit_sequence(steps: &[&str]) -> (Buffer, SyntaxMap) {
1956 let registry = Arc::new(LanguageRegistry::test());
1957 let language = Arc::new(rust_lang());
1958 registry.add(language.clone());
1959 let mut buffer = Buffer::new(0, 0, Default::default());
1960
1961 let mut mutated_syntax_map = SyntaxMap::new();
1962 mutated_syntax_map.set_language_registry(registry.clone());
1963 mutated_syntax_map.reparse(language.clone(), &buffer);
1964
1965 for (i, marked_string) in steps.into_iter().enumerate() {
1966 edit_buffer(&mut buffer, &marked_string.unindent());
1967
1968 // Reparse the syntax map
1969 mutated_syntax_map.interpolate(&buffer);
1970 mutated_syntax_map.reparse(language.clone(), &buffer);
1971
1972 // Create a second syntax map from scratch
1973 let mut reference_syntax_map = SyntaxMap::new();
1974 reference_syntax_map.set_language_registry(registry.clone());
1975 reference_syntax_map.reparse(language.clone(), &buffer);
1976
1977 // Compare the mutated syntax map to the new syntax map
1978 let mutated_layers = mutated_syntax_map.layers(&buffer);
1979 let reference_layers = reference_syntax_map.layers(&buffer);
1980 assert_eq!(
1981 mutated_layers.len(),
1982 reference_layers.len(),
1983 "wrong number of layers at step {i}"
1984 );
1985 for (edited_layer, reference_layer) in
1986 mutated_layers.into_iter().zip(reference_layers.into_iter())
1987 {
1988 assert_eq!(
1989 edited_layer.node.to_sexp(),
1990 reference_layer.node.to_sexp(),
1991 "different layer at step {i}"
1992 );
1993 assert_eq!(
1994 edited_layer.node.range(),
1995 reference_layer.node.range(),
1996 "different layer at step {i}"
1997 );
1998 }
1999 }
2000
2001 (buffer, mutated_syntax_map)
2002 }
2003
2004 fn rust_lang() -> Language {
2005 Language::new(
2006 LanguageConfig {
2007 name: "Rust".into(),
2008 path_suffixes: vec!["rs".to_string()],
2009 ..Default::default()
2010 },
2011 Some(tree_sitter_rust::language()),
2012 )
2013 .with_highlights_query(
2014 r#"
2015 (field_identifier) @field
2016 (struct_expression) @struct
2017 "#,
2018 )
2019 .unwrap()
2020 .with_injection_query(
2021 r#"
2022 (macro_invocation
2023 (token_tree) @content
2024 (#set! "language" "rust"))
2025 "#,
2026 )
2027 .unwrap()
2028 }
2029
2030 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2031 let start = buffer.as_rope().to_string().find(text).unwrap();
2032 start..start + text.len()
2033 }
2034
2035 fn assert_layers_for_range(
2036 syntax_map: &SyntaxMap,
2037 buffer: &BufferSnapshot,
2038 range: Range<Point>,
2039 expected_layers: &[&str],
2040 ) {
2041 let layers = syntax_map
2042 .layers_for_range(range, &buffer)
2043 .collect::<Vec<_>>();
2044 assert_eq!(
2045 layers.len(),
2046 expected_layers.len(),
2047 "wrong number of layers"
2048 );
2049 for (i, (SyntaxLayerInfo { node, .. }, expected_s_exp)) in
2050 layers.iter().zip(expected_layers.iter()).enumerate()
2051 {
2052 let actual_s_exp = node.to_sexp();
2053 assert!(
2054 string_contains_sequence(
2055 &actual_s_exp,
2056 &expected_s_exp.split("...").collect::<Vec<_>>()
2057 ),
2058 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2059 );
2060 }
2061 }
2062
2063 fn assert_capture_ranges(
2064 syntax_map: &SyntaxMap,
2065 buffer: &BufferSnapshot,
2066 highlight_query_capture_names: &[&str],
2067 marked_string: &str,
2068 ) {
2069 let mut actual_ranges = Vec::<Range<usize>>::new();
2070 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2071 grammar.highlights_query.as_ref()
2072 });
2073 let queries = captures
2074 .grammars()
2075 .iter()
2076 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2077 .collect::<Vec<_>>();
2078 for capture in captures {
2079 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2080 if highlight_query_capture_names.contains(&name.as_str()) {
2081 actual_ranges.push(capture.node.byte_range());
2082 }
2083 }
2084
2085 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2086 assert_eq!(text, buffer.text());
2087 assert_eq!(actual_ranges, expected_ranges);
2088 }
2089
2090 fn edit_buffer(buffer: &mut Buffer, marked_string: &str) {
2091 let old_text = buffer.text();
2092 let (new_text, mut ranges) = marked_text_ranges(marked_string, false);
2093 if ranges.is_empty() {
2094 ranges.push(0..new_text.len());
2095 }
2096
2097 assert_eq!(
2098 old_text[..ranges[0].start],
2099 new_text[..ranges[0].start],
2100 "invalid edit"
2101 );
2102
2103 let mut delta = 0;
2104 let mut edits = Vec::new();
2105 let mut ranges = ranges.into_iter().peekable();
2106
2107 while let Some(inserted_range) = ranges.next() {
2108 let new_start = inserted_range.start;
2109 let old_start = (new_start as isize - delta) as usize;
2110
2111 let following_text = if let Some(next_range) = ranges.peek() {
2112 &new_text[inserted_range.end..next_range.start]
2113 } else {
2114 &new_text[inserted_range.end..]
2115 };
2116
2117 let inserted_len = inserted_range.len();
2118 let deleted_len = old_text[old_start..]
2119 .find(following_text)
2120 .expect("invalid edit");
2121
2122 let old_range = old_start..old_start + deleted_len;
2123 edits.push((old_range, new_text[inserted_range].to_string()));
2124 delta += inserted_len as isize - deleted_len as isize;
2125 }
2126
2127 assert_eq!(
2128 old_text.len() as isize + delta,
2129 new_text.len() as isize,
2130 "invalid edit"
2131 );
2132
2133 buffer.edit(edits);
2134 }
2135
2136 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2137 let mut last_part_end = 0;
2138 for part in parts {
2139 if let Some(start_ix) = text[last_part_end..].find(part) {
2140 last_part_end = start_ix + part.len();
2141 } else {
2142 return false;
2143 }
2144 }
2145 true
2146 }
2147}