1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use lazy_static::lazy_static;
4use parking_lot::Mutex;
5use std::{
6 borrow::Cow,
7 cell::RefCell,
8 cmp::{self, Ordering, Reverse},
9 collections::BinaryHeap,
10 iter,
11 ops::{Deref, DerefMut, Range},
12 sync::Arc,
13};
14use sum_tree::{Bias, SeekTarget, SumTree};
15use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
16use tree_sitter::{
17 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
18};
19
20thread_local! {
21 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
22}
23
24lazy_static! {
25 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
26}
27
28#[derive(Default)]
29pub struct SyntaxMap {
30 parsed_version: clock::Global,
31 interpolated_version: clock::Global,
32 snapshot: SyntaxSnapshot,
33 language_registry: Option<Arc<LanguageRegistry>>,
34}
35
36#[derive(Clone, Default)]
37pub struct SyntaxSnapshot {
38 layers: SumTree<SyntaxLayer>,
39}
40
41#[derive(Default)]
42pub struct SyntaxMapCaptures<'a> {
43 layers: Vec<SyntaxMapCapturesLayer<'a>>,
44 active_layer_count: usize,
45 grammars: Vec<&'a Grammar>,
46}
47
48#[derive(Default)]
49pub struct SyntaxMapMatches<'a> {
50 layers: Vec<SyntaxMapMatchesLayer<'a>>,
51 active_layer_count: usize,
52 grammars: Vec<&'a Grammar>,
53}
54
55#[derive(Debug)]
56pub struct SyntaxMapCapture<'a> {
57 pub depth: usize,
58 pub node: Node<'a>,
59 pub index: u32,
60 pub grammar_index: usize,
61}
62
63#[derive(Debug)]
64pub struct SyntaxMapMatch<'a> {
65 pub depth: usize,
66 pub pattern_index: usize,
67 pub captures: &'a [QueryCapture<'a>],
68 pub grammar_index: usize,
69}
70
71struct SyntaxMapCapturesLayer<'a> {
72 depth: usize,
73 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
74 next_capture: Option<QueryCapture<'a>>,
75 grammar_index: usize,
76 _query_cursor: QueryCursorHandle,
77}
78
79struct SyntaxMapMatchesLayer<'a> {
80 depth: usize,
81 next_pattern_index: usize,
82 next_captures: Vec<QueryCapture<'a>>,
83 has_next: bool,
84 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
85 grammar_index: usize,
86 _query_cursor: QueryCursorHandle,
87}
88
89#[derive(Clone)]
90struct SyntaxLayer {
91 depth: usize,
92 range: Range<Anchor>,
93 content: SyntaxLayerContent,
94}
95
96#[derive(Clone)]
97enum SyntaxLayerContent {
98 Parsed {
99 tree: tree_sitter::Tree,
100 language: Arc<Language>,
101 },
102 Pending {
103 language_name: Arc<str>,
104 },
105}
106
107impl SyntaxLayerContent {
108 fn language_id(&self) -> Option<usize> {
109 match self {
110 SyntaxLayerContent::Parsed { language, .. } => language.id(),
111 SyntaxLayerContent::Pending { .. } => None,
112 }
113 }
114
115 fn tree(&self) -> Option<&Tree> {
116 match self {
117 SyntaxLayerContent::Parsed { tree, .. } => Some(tree),
118 SyntaxLayerContent::Pending { .. } => None,
119 }
120 }
121}
122
123#[derive(Debug)]
124pub struct SyntaxLayerInfo<'a> {
125 pub depth: usize,
126 pub node: Node<'a>,
127 pub language: &'a Arc<Language>,
128}
129
130#[derive(Debug, Clone)]
131struct SyntaxLayerSummary {
132 min_depth: usize,
133 max_depth: usize,
134 range: Range<Anchor>,
135 last_layer_range: Range<Anchor>,
136 last_layer_language: Option<usize>,
137 contains_pending_layer: bool,
138}
139
140#[derive(Clone, Debug)]
141struct SyntaxLayerPosition {
142 depth: usize,
143 range: Range<Anchor>,
144 language: Option<usize>,
145}
146
147#[derive(Clone, Debug)]
148struct ChangeStartPosition {
149 depth: usize,
150 position: Anchor,
151}
152
153#[derive(Clone, Debug)]
154struct SyntaxLayerPositionBeforeChange {
155 position: SyntaxLayerPosition,
156 change: ChangeStartPosition,
157}
158
159struct ParseStep {
160 depth: usize,
161 language: ParseStepLanguage,
162 range: Range<Anchor>,
163 included_ranges: Vec<tree_sitter::Range>,
164 mode: ParseMode,
165}
166
167enum ParseStepLanguage {
168 Loaded { language: Arc<Language> },
169 Pending { name: Arc<str> },
170}
171
172impl ParseStepLanguage {
173 fn id(&self) -> Option<usize> {
174 match self {
175 ParseStepLanguage::Loaded { language } => language.id(),
176 ParseStepLanguage::Pending { .. } => None,
177 }
178 }
179}
180
181enum ParseMode {
182 Single,
183 Combined {
184 parent_layer_range: Range<usize>,
185 parent_layer_changed_ranges: Vec<Range<usize>>,
186 },
187}
188
189#[derive(Debug, PartialEq, Eq)]
190struct ChangedRegion {
191 depth: usize,
192 range: Range<Anchor>,
193}
194
195#[derive(Default)]
196struct ChangeRegionSet(Vec<ChangedRegion>);
197
198struct TextProvider<'a>(&'a Rope);
199
200struct ByteChunks<'a>(text::Chunks<'a>);
201
202struct QueryCursorHandle(Option<QueryCursor>);
203
204impl SyntaxMap {
205 pub fn new() -> Self {
206 Self::default()
207 }
208
209 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
210 self.language_registry = Some(registry);
211 }
212
213 pub fn snapshot(&self) -> SyntaxSnapshot {
214 self.snapshot.clone()
215 }
216
217 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
218 self.language_registry.clone()
219 }
220
221 pub fn parsed_version(&self) -> clock::Global {
222 self.parsed_version.clone()
223 }
224
225 pub fn interpolate(&mut self, text: &BufferSnapshot) {
226 self.snapshot.interpolate(&self.interpolated_version, text);
227 self.interpolated_version = text.version.clone();
228 }
229
230 #[cfg(test)]
231 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
232 self.snapshot.reparse(
233 &self.parsed_version,
234 text,
235 self.language_registry.clone(),
236 language,
237 );
238 self.parsed_version = text.version.clone();
239 self.interpolated_version = text.version.clone();
240 }
241
242 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot, version: clock::Global) {
243 self.interpolated_version = version.clone();
244 self.parsed_version = version;
245 self.snapshot = snapshot;
246 }
247
248 pub fn clear(&mut self) {
249 self.snapshot = SyntaxSnapshot::default();
250 }
251}
252
253impl SyntaxSnapshot {
254 pub fn is_empty(&self) -> bool {
255 self.layers.is_empty()
256 }
257
258 pub fn interpolate(&mut self, from_version: &clock::Global, text: &BufferSnapshot) {
259 let edits = text
260 .anchored_edits_since::<(usize, Point)>(&from_version)
261 .collect::<Vec<_>>();
262 if edits.is_empty() {
263 return;
264 }
265
266 let mut layers = SumTree::new();
267 let mut first_edit_ix_for_depth = 0;
268 let mut prev_depth = 0;
269 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
270 cursor.next(text);
271
272 'outer: loop {
273 let depth = cursor.end(text).max_depth;
274 if depth > prev_depth {
275 first_edit_ix_for_depth = 0;
276 prev_depth = depth;
277 }
278
279 // Preserve any layers at this depth that precede the first edit.
280 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
281 let target = ChangeStartPosition {
282 depth,
283 position: edit_range.start,
284 };
285 if target.cmp(&cursor.start(), text).is_gt() {
286 let slice = cursor.slice(&target, Bias::Left, text);
287 layers.push_tree(slice, text);
288 }
289 }
290 // If this layer follows all of the edits, then preserve it and any
291 // subsequent layers at this same depth.
292 else if cursor.item().is_some() {
293 let slice = cursor.slice(
294 &SyntaxLayerPosition {
295 depth: depth + 1,
296 range: Anchor::MIN..Anchor::MAX,
297 language: None,
298 },
299 Bias::Left,
300 text,
301 );
302 layers.push_tree(slice, text);
303 continue;
304 };
305
306 let Some(layer) = cursor.item() else { break };
307 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
308
309 // Ignore edits that end before the start of this layer, and don't consider them
310 // for any subsequent layers at this same depth.
311 loop {
312 let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) else { continue 'outer };
313 if edit_range.end.cmp(&layer.range.start, text).is_le() {
314 first_edit_ix_for_depth += 1;
315 } else {
316 break;
317 }
318 }
319
320 let mut layer = layer.clone();
321 if let SyntaxLayerContent::Parsed { tree, .. } = &mut layer.content {
322 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
323 // Ignore any edits that follow this layer.
324 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
325 break;
326 }
327
328 // Apply any edits that intersect this layer to the layer's syntax tree.
329 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
330 tree_sitter::InputEdit {
331 start_byte: edit.new.start.0 - start_byte,
332 old_end_byte: edit.new.start.0 - start_byte
333 + (edit.old.end.0 - edit.old.start.0),
334 new_end_byte: edit.new.end.0 - start_byte,
335 start_position: (edit.new.start.1 - start_point).to_ts_point(),
336 old_end_position: (edit.new.start.1 - start_point
337 + (edit.old.end.1 - edit.old.start.1))
338 .to_ts_point(),
339 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
340 }
341 } else {
342 let node = tree.root_node();
343 tree_sitter::InputEdit {
344 start_byte: 0,
345 old_end_byte: node.end_byte(),
346 new_end_byte: 0,
347 start_position: Default::default(),
348 old_end_position: node.end_position(),
349 new_end_position: Default::default(),
350 }
351 };
352
353 tree.edit(&tree_edit);
354 }
355
356 debug_assert!(
357 tree.root_node().end_byte() <= text.len(),
358 "tree's size {}, is larger than text size {}",
359 tree.root_node().end_byte(),
360 text.len(),
361 );
362 }
363
364 layers.push(layer, text);
365 cursor.next(text);
366 }
367
368 layers.push_tree(cursor.suffix(&text), &text);
369 drop(cursor);
370 self.layers = layers;
371 }
372
373 pub fn reparse(
374 &mut self,
375 from_version: &clock::Global,
376 text: &BufferSnapshot,
377 registry: Option<Arc<LanguageRegistry>>,
378 root_language: Arc<Language>,
379 ) {
380 let edits = text.edits_since::<usize>(from_version).collect::<Vec<_>>();
381 let max_depth = self.layers.summary().max_depth;
382 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
383 cursor.next(&text);
384 let mut layers = SumTree::new();
385
386 let mut changed_regions = ChangeRegionSet::default();
387 let mut queue = BinaryHeap::new();
388 let mut combined_injection_ranges = HashMap::default();
389 queue.push(ParseStep {
390 depth: 0,
391 language: ParseStepLanguage::Loaded {
392 language: root_language,
393 },
394 included_ranges: vec![tree_sitter::Range {
395 start_byte: 0,
396 end_byte: text.len(),
397 start_point: Point::zero().to_ts_point(),
398 end_point: text.max_point().to_ts_point(),
399 }],
400 range: Anchor::MIN..Anchor::MAX,
401 mode: ParseMode::Single,
402 });
403
404 loop {
405 let step = queue.pop();
406 let position = if let Some(step) = &step {
407 SyntaxLayerPosition {
408 depth: step.depth,
409 range: step.range.clone(),
410 language: step.language.id(),
411 }
412 } else {
413 SyntaxLayerPosition {
414 depth: max_depth + 1,
415 range: Anchor::MAX..Anchor::MAX,
416 language: None,
417 }
418 };
419
420 let mut done = cursor.item().is_none();
421 while !done && position.cmp(&cursor.end(text), &text).is_gt() {
422 done = true;
423
424 let bounded_position = SyntaxLayerPositionBeforeChange {
425 position: position.clone(),
426 change: changed_regions.start_position(),
427 };
428 if bounded_position.cmp(&cursor.start(), &text).is_gt() {
429 let slice = cursor.slice(&bounded_position, Bias::Left, text);
430 if !slice.is_empty() {
431 layers.push_tree(slice, &text);
432 if changed_regions.prune(cursor.end(text), text) {
433 done = false;
434 }
435 }
436 }
437
438 while position.cmp(&cursor.end(text), text).is_gt() {
439 let Some(layer) = cursor.item() else { break };
440
441 if changed_regions.intersects(&layer, text) {
442 changed_regions.insert(
443 ChangedRegion {
444 depth: layer.depth + 1,
445 range: layer.range.clone(),
446 },
447 text,
448 );
449 } else {
450 layers.push(layer.clone(), text);
451 }
452
453 cursor.next(text);
454 if changed_regions.prune(cursor.end(text), text) {
455 done = false;
456 }
457 }
458 }
459
460 let Some(step) = step else { break };
461 let (step_start_byte, step_start_point) =
462 step.range.start.summary::<(usize, Point)>(text);
463 let step_end_byte = step.range.end.to_offset(text);
464
465 let mut old_layer = cursor.item();
466 if let Some(layer) = old_layer {
467 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
468 && layer.content.language_id() == step.language.id()
469 {
470 cursor.next(&text);
471 } else {
472 old_layer = None;
473 }
474 }
475
476 let content = match step.language {
477 ParseStepLanguage::Loaded { language } => {
478 let Some(grammar) = language.grammar() else { continue };
479 let tree;
480 let changed_ranges;
481 let mut included_ranges = step.included_ranges;
482 if let Some(SyntaxLayerContent::Parsed { tree: old_tree, .. }) =
483 old_layer.map(|layer| &layer.content)
484 {
485 if let ParseMode::Combined {
486 parent_layer_changed_ranges,
487 ..
488 } = step.mode
489 {
490 included_ranges = splice_included_ranges(
491 old_tree.included_ranges(),
492 &parent_layer_changed_ranges,
493 &included_ranges,
494 );
495 }
496
497 tree = parse_text(
498 grammar,
499 text.as_rope(),
500 step_start_byte,
501 step_start_point,
502 included_ranges,
503 Some(old_tree.clone()),
504 );
505 changed_ranges = join_ranges(
506 edits.iter().map(|e| e.new.clone()).filter(|range| {
507 range.start <= step_end_byte && range.end >= step_start_byte
508 }),
509 old_tree.changed_ranges(&tree).map(|r| {
510 step_start_byte + r.start_byte..step_start_byte + r.end_byte
511 }),
512 );
513 } else {
514 tree = parse_text(
515 grammar,
516 text.as_rope(),
517 step_start_byte,
518 step_start_point,
519 included_ranges,
520 None,
521 );
522 changed_ranges = vec![step_start_byte..step_end_byte];
523 }
524
525 if let (Some((config, registry)), false) = (
526 grammar.injection_config.as_ref().zip(registry.as_ref()),
527 changed_ranges.is_empty(),
528 ) {
529 for range in &changed_ranges {
530 changed_regions.insert(
531 ChangedRegion {
532 depth: step.depth + 1,
533 range: text.anchor_before(range.start)
534 ..text.anchor_after(range.end),
535 },
536 text,
537 );
538 }
539 get_injections(
540 config,
541 text,
542 tree.root_node_with_offset(
543 step_start_byte,
544 step_start_point.to_ts_point(),
545 ),
546 registry,
547 step.depth + 1,
548 &changed_ranges,
549 &mut combined_injection_ranges,
550 &mut queue,
551 );
552 }
553
554 SyntaxLayerContent::Parsed { tree, language }
555 }
556 ParseStepLanguage::Pending { name } => SyntaxLayerContent::Pending {
557 language_name: name,
558 },
559 };
560
561 layers.push(
562 SyntaxLayer {
563 depth: step.depth,
564 range: step.range,
565 content,
566 },
567 &text,
568 );
569 }
570
571 drop(cursor);
572 self.layers = layers;
573 }
574
575 pub fn single_tree_captures<'a>(
576 range: Range<usize>,
577 text: &'a Rope,
578 tree: &'a Tree,
579 language: &'a Arc<Language>,
580 query: fn(&Grammar) -> Option<&Query>,
581 ) -> SyntaxMapCaptures<'a> {
582 SyntaxMapCaptures::new(
583 range.clone(),
584 text,
585 [SyntaxLayerInfo {
586 language,
587 depth: 0,
588 node: tree.root_node(),
589 }]
590 .into_iter(),
591 query,
592 )
593 }
594
595 pub fn captures<'a>(
596 &'a self,
597 range: Range<usize>,
598 buffer: &'a BufferSnapshot,
599 query: fn(&Grammar) -> Option<&Query>,
600 ) -> SyntaxMapCaptures {
601 SyntaxMapCaptures::new(
602 range.clone(),
603 buffer.as_rope(),
604 self.layers_for_range(range, buffer).into_iter(),
605 query,
606 )
607 }
608
609 pub fn matches<'a>(
610 &'a self,
611 range: Range<usize>,
612 buffer: &'a BufferSnapshot,
613 query: fn(&Grammar) -> Option<&Query>,
614 ) -> SyntaxMapMatches {
615 SyntaxMapMatches::new(
616 range.clone(),
617 buffer.as_rope(),
618 self.layers_for_range(range, buffer).into_iter(),
619 query,
620 )
621 }
622
623 #[cfg(test)]
624 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
625 self.layers_for_range(0..buffer.len(), buffer).collect()
626 }
627
628 pub fn layers_for_range<'a, T: ToOffset>(
629 &'a self,
630 range: Range<T>,
631 buffer: &'a BufferSnapshot,
632 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
633 let start = buffer.anchor_before(range.start.to_offset(buffer));
634 let end = buffer.anchor_after(range.end.to_offset(buffer));
635
636 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
637 if summary.max_depth > summary.min_depth {
638 true
639 } else {
640 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
641 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
642 !is_before_start && !is_after_end
643 }
644 });
645
646 cursor.next(buffer);
647 iter::from_fn(move || {
648 while let Some(layer) = cursor.item() {
649 if let SyntaxLayerContent::Parsed { tree, language } = &layer.content {
650 let info = SyntaxLayerInfo {
651 language,
652 depth: layer.depth,
653 node: tree.root_node_with_offset(
654 layer.range.start.to_offset(buffer),
655 layer.range.start.to_point(buffer).to_ts_point(),
656 ),
657 };
658 cursor.next(buffer);
659 return Some(info);
660 } else {
661 cursor.next(buffer);
662 }
663 }
664 None
665 })
666 }
667
668 pub fn unknown_injection_languages<'a>(
669 &'a self,
670 buffer: &'a BufferSnapshot,
671 ) -> impl 'a + Iterator<Item = &Arc<str>> {
672 let mut cursor = self
673 .layers
674 .filter::<_, ()>(|summary| summary.contains_pending_layer);
675 cursor.next(buffer);
676 iter::from_fn(move || {
677 while let Some(layer) = cursor.item() {
678 if let SyntaxLayerContent::Pending { language_name } = &layer.content {
679 cursor.next(buffer);
680 return Some(language_name);
681 } else {
682 cursor.next(buffer);
683 }
684 }
685 None
686 })
687 }
688}
689
690impl<'a> SyntaxMapCaptures<'a> {
691 fn new(
692 range: Range<usize>,
693 text: &'a Rope,
694 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
695 query: fn(&Grammar) -> Option<&Query>,
696 ) -> Self {
697 let mut result = Self {
698 layers: Vec::new(),
699 grammars: Vec::new(),
700 active_layer_count: 0,
701 };
702 for SyntaxLayerInfo {
703 language,
704 depth,
705 node,
706 } in layers
707 {
708 let grammar = match &language.grammar {
709 Some(grammer) => grammer,
710 None => continue,
711 };
712 let query = match query(&grammar) {
713 Some(query) => query,
714 None => continue,
715 };
716
717 let mut query_cursor = QueryCursorHandle::new();
718
719 // TODO - add a Tree-sitter API to remove the need for this.
720 let cursor = unsafe {
721 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
722 };
723
724 cursor.set_byte_range(range.clone());
725 let captures = cursor.captures(query, node, TextProvider(text));
726 let grammar_index = result
727 .grammars
728 .iter()
729 .position(|g| g.id == grammar.id())
730 .unwrap_or_else(|| {
731 result.grammars.push(grammar);
732 result.grammars.len() - 1
733 });
734 let mut layer = SyntaxMapCapturesLayer {
735 depth,
736 grammar_index,
737 next_capture: None,
738 captures,
739 _query_cursor: query_cursor,
740 };
741
742 layer.advance();
743 if layer.next_capture.is_some() {
744 let key = layer.sort_key();
745 let ix = match result.layers[..result.active_layer_count]
746 .binary_search_by_key(&key, |layer| layer.sort_key())
747 {
748 Ok(ix) | Err(ix) => ix,
749 };
750 result.layers.insert(ix, layer);
751 result.active_layer_count += 1;
752 } else {
753 result.layers.push(layer);
754 }
755 }
756
757 result
758 }
759
760 pub fn grammars(&self) -> &[&'a Grammar] {
761 &self.grammars
762 }
763
764 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
765 let layer = self.layers[..self.active_layer_count].first()?;
766 let capture = layer.next_capture?;
767 Some(SyntaxMapCapture {
768 depth: layer.depth,
769 grammar_index: layer.grammar_index,
770 index: capture.index,
771 node: capture.node,
772 })
773 }
774
775 pub fn advance(&mut self) -> bool {
776 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
777 layer
778 } else {
779 return false;
780 };
781
782 layer.advance();
783 if layer.next_capture.is_some() {
784 let key = layer.sort_key();
785 let i = 1 + self.layers[1..self.active_layer_count]
786 .iter()
787 .position(|later_layer| key < later_layer.sort_key())
788 .unwrap_or(self.active_layer_count - 1);
789 self.layers[0..i].rotate_left(1);
790 } else {
791 self.layers[0..self.active_layer_count].rotate_left(1);
792 self.active_layer_count -= 1;
793 }
794
795 true
796 }
797
798 pub fn set_byte_range(&mut self, range: Range<usize>) {
799 for layer in &mut self.layers {
800 layer.captures.set_byte_range(range.clone());
801 if let Some(capture) = &layer.next_capture {
802 if capture.node.end_byte() > range.start {
803 continue;
804 }
805 }
806 layer.advance();
807 }
808 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
809 self.active_layer_count = self
810 .layers
811 .iter()
812 .position(|layer| layer.next_capture.is_none())
813 .unwrap_or(self.layers.len());
814 }
815}
816
817impl<'a> SyntaxMapMatches<'a> {
818 fn new(
819 range: Range<usize>,
820 text: &'a Rope,
821 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
822 query: fn(&Grammar) -> Option<&Query>,
823 ) -> Self {
824 let mut result = Self::default();
825 for SyntaxLayerInfo {
826 language,
827 depth,
828 node,
829 } in layers
830 {
831 let grammar = match &language.grammar {
832 Some(grammer) => grammer,
833 None => continue,
834 };
835 let query = match query(&grammar) {
836 Some(query) => query,
837 None => continue,
838 };
839
840 let mut query_cursor = QueryCursorHandle::new();
841
842 // TODO - add a Tree-sitter API to remove the need for this.
843 let cursor = unsafe {
844 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
845 };
846
847 cursor.set_byte_range(range.clone());
848 let matches = cursor.matches(query, node, TextProvider(text));
849 let grammar_index = result
850 .grammars
851 .iter()
852 .position(|g| g.id == grammar.id())
853 .unwrap_or_else(|| {
854 result.grammars.push(grammar);
855 result.grammars.len() - 1
856 });
857 let mut layer = SyntaxMapMatchesLayer {
858 depth,
859 grammar_index,
860 matches,
861 next_pattern_index: 0,
862 next_captures: Vec::new(),
863 has_next: false,
864 _query_cursor: query_cursor,
865 };
866
867 layer.advance();
868 if layer.has_next {
869 let key = layer.sort_key();
870 let ix = match result.layers[..result.active_layer_count]
871 .binary_search_by_key(&key, |layer| layer.sort_key())
872 {
873 Ok(ix) | Err(ix) => ix,
874 };
875 result.layers.insert(ix, layer);
876 result.active_layer_count += 1;
877 } else {
878 result.layers.push(layer);
879 }
880 }
881 result
882 }
883
884 pub fn grammars(&self) -> &[&'a Grammar] {
885 &self.grammars
886 }
887
888 pub fn peek(&self) -> Option<SyntaxMapMatch> {
889 let layer = self.layers.first()?;
890 if !layer.has_next {
891 return None;
892 }
893 Some(SyntaxMapMatch {
894 depth: layer.depth,
895 grammar_index: layer.grammar_index,
896 pattern_index: layer.next_pattern_index,
897 captures: &layer.next_captures,
898 })
899 }
900
901 pub fn advance(&mut self) -> bool {
902 let layer = if let Some(layer) = self.layers.first_mut() {
903 layer
904 } else {
905 return false;
906 };
907
908 layer.advance();
909 if layer.has_next {
910 let key = layer.sort_key();
911 let i = 1 + self.layers[1..self.active_layer_count]
912 .iter()
913 .position(|later_layer| key < later_layer.sort_key())
914 .unwrap_or(self.active_layer_count - 1);
915 self.layers[0..i].rotate_left(1);
916 } else {
917 self.layers[0..self.active_layer_count].rotate_left(1);
918 self.active_layer_count -= 1;
919 }
920
921 true
922 }
923}
924
925impl<'a> SyntaxMapCapturesLayer<'a> {
926 fn advance(&mut self) {
927 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
928 }
929
930 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
931 if let Some(capture) = &self.next_capture {
932 let range = capture.node.byte_range();
933 (range.start, Reverse(range.end), self.depth)
934 } else {
935 (usize::MAX, Reverse(0), usize::MAX)
936 }
937 }
938}
939
940impl<'a> SyntaxMapMatchesLayer<'a> {
941 fn advance(&mut self) {
942 if let Some(mat) = self.matches.next() {
943 self.next_captures.clear();
944 self.next_captures.extend_from_slice(&mat.captures);
945 self.next_pattern_index = mat.pattern_index;
946 self.has_next = true;
947 } else {
948 self.has_next = false;
949 }
950 }
951
952 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
953 if self.has_next {
954 let captures = &self.next_captures;
955 if let Some((first, last)) = captures.first().zip(captures.last()) {
956 return (
957 first.node.start_byte(),
958 Reverse(last.node.end_byte()),
959 self.depth,
960 );
961 }
962 }
963 (usize::MAX, Reverse(0), usize::MAX)
964 }
965}
966
967impl<'a> Iterator for SyntaxMapCaptures<'a> {
968 type Item = SyntaxMapCapture<'a>;
969
970 fn next(&mut self) -> Option<Self::Item> {
971 let result = self.peek();
972 self.advance();
973 result
974 }
975}
976
977fn join_ranges(
978 a: impl Iterator<Item = Range<usize>>,
979 b: impl Iterator<Item = Range<usize>>,
980) -> Vec<Range<usize>> {
981 let mut result = Vec::<Range<usize>>::new();
982 let mut a = a.peekable();
983 let mut b = b.peekable();
984 loop {
985 let range = match (a.peek(), b.peek()) {
986 (Some(range_a), Some(range_b)) => {
987 if range_a.start < range_b.start {
988 a.next().unwrap()
989 } else {
990 b.next().unwrap()
991 }
992 }
993 (None, Some(_)) => b.next().unwrap(),
994 (Some(_), None) => a.next().unwrap(),
995 (None, None) => break,
996 };
997
998 if let Some(last) = result.last_mut() {
999 if range.start <= last.end {
1000 last.end = last.end.max(range.end);
1001 continue;
1002 }
1003 }
1004 result.push(range);
1005 }
1006 result
1007}
1008
1009fn parse_text(
1010 grammar: &Grammar,
1011 text: &Rope,
1012 start_byte: usize,
1013 start_point: Point,
1014 mut ranges: Vec<tree_sitter::Range>,
1015 old_tree: Option<Tree>,
1016) -> Tree {
1017 for range in &mut ranges {
1018 range.start_byte -= start_byte;
1019 range.end_byte -= start_byte;
1020 range.start_point = (Point::from_ts_point(range.start_point) - start_point).to_ts_point();
1021 range.end_point = (Point::from_ts_point(range.end_point) - start_point).to_ts_point();
1022 }
1023
1024 PARSER.with(|parser| {
1025 let mut parser = parser.borrow_mut();
1026 let mut chunks = text.chunks_in_range(start_byte..text.len());
1027 parser
1028 .set_included_ranges(&ranges)
1029 .expect("overlapping ranges");
1030 parser
1031 .set_language(grammar.ts_language)
1032 .expect("incompatible grammar");
1033 parser
1034 .parse_with(
1035 &mut move |offset, _| {
1036 chunks.seek(start_byte + offset);
1037 chunks.next().unwrap_or("").as_bytes()
1038 },
1039 old_tree.as_ref(),
1040 )
1041 .expect("invalid language")
1042 })
1043}
1044
1045fn get_injections(
1046 config: &InjectionConfig,
1047 text: &BufferSnapshot,
1048 node: Node,
1049 language_registry: &LanguageRegistry,
1050 depth: usize,
1051 changed_ranges: &[Range<usize>],
1052 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
1053 queue: &mut BinaryHeap<ParseStep>,
1054) {
1055 let mut query_cursor = QueryCursorHandle::new();
1056 let mut prev_match = None;
1057
1058 combined_injection_ranges.clear();
1059 for pattern in &config.patterns {
1060 if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
1061 if let Some(language) = language_registry.language_for_name(language_name) {
1062 combined_injection_ranges.insert(language, Vec::new());
1063 }
1064 }
1065 }
1066
1067 for query_range in changed_ranges {
1068 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
1069 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
1070 let content_ranges = mat
1071 .nodes_for_capture_index(config.content_capture_ix)
1072 .map(|node| node.range())
1073 .collect::<Vec<_>>();
1074 if content_ranges.is_empty() {
1075 continue;
1076 }
1077
1078 // Avoid duplicate matches if two changed ranges intersect the same injection.
1079 let content_range =
1080 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
1081 if let Some((last_pattern_ix, last_range)) = &prev_match {
1082 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1083 continue;
1084 }
1085 }
1086 prev_match = Some((mat.pattern_index, content_range.clone()));
1087
1088 let combined = config.patterns[mat.pattern_index].combined;
1089
1090 let mut language_name = None;
1091 let mut step_range = content_range.clone();
1092 if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {
1093 language_name = Some(Cow::Borrowed(name.as_ref()))
1094 } else if let Some(language_node) = config
1095 .language_capture_ix
1096 .and_then(|ix| mat.nodes_for_capture_index(ix).next())
1097 {
1098 step_range.start = cmp::min(content_range.start, language_node.start_byte());
1099 step_range.end = cmp::max(content_range.end, language_node.end_byte());
1100 language_name = Some(Cow::Owned(
1101 text.text_for_range(language_node.byte_range()).collect(),
1102 ))
1103 };
1104
1105 if let Some(language_name) = language_name {
1106 let language = language_registry
1107 .language_for_name(&language_name)
1108 .or_else(|| language_registry.language_for_extension(&language_name));
1109 let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
1110 if let Some(language) = language {
1111 if combined {
1112 combined_injection_ranges
1113 .get_mut(&language.clone())
1114 .unwrap()
1115 .extend(content_ranges);
1116 } else {
1117 queue.push(ParseStep {
1118 depth,
1119 language: ParseStepLanguage::Loaded { language },
1120 included_ranges: content_ranges,
1121 range,
1122 mode: ParseMode::Single,
1123 });
1124 }
1125 } else {
1126 queue.push(ParseStep {
1127 depth,
1128 language: ParseStepLanguage::Pending {
1129 name: language_name.into(),
1130 },
1131 included_ranges: content_ranges,
1132 range,
1133 mode: ParseMode::Single,
1134 });
1135 }
1136 }
1137 }
1138 }
1139
1140 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1141 included_ranges.sort_unstable();
1142 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1143 queue.push(ParseStep {
1144 depth,
1145 language: ParseStepLanguage::Loaded { language },
1146 range,
1147 included_ranges,
1148 mode: ParseMode::Combined {
1149 parent_layer_range: node.start_byte()..node.end_byte(),
1150 parent_layer_changed_ranges: changed_ranges.to_vec(),
1151 },
1152 })
1153 }
1154}
1155
1156fn splice_included_ranges(
1157 mut ranges: Vec<tree_sitter::Range>,
1158 changed_ranges: &[Range<usize>],
1159 new_ranges: &[tree_sitter::Range],
1160) -> Vec<tree_sitter::Range> {
1161 let mut changed_ranges = changed_ranges.into_iter().peekable();
1162 let mut new_ranges = new_ranges.into_iter().peekable();
1163 let mut ranges_ix = 0;
1164 loop {
1165 let new_range = new_ranges.peek();
1166 let mut changed_range = changed_ranges.peek();
1167
1168 // Remove ranges that have changed before inserting any new ranges
1169 // into those ranges.
1170 if let Some((changed, new)) = changed_range.zip(new_range) {
1171 if new.end_byte < changed.start {
1172 changed_range = None;
1173 }
1174 }
1175
1176 if let Some(changed) = changed_range {
1177 let mut start_ix = ranges_ix
1178 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1179 Ok(ix) | Err(ix) => ix,
1180 };
1181 let mut end_ix = ranges_ix
1182 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1183 Ok(ix) => ix + 1,
1184 Err(ix) => ix,
1185 };
1186
1187 // If there are empty ranges, then there may be multiple ranges with the same
1188 // start or end. Expand the splice to include any adjacent ranges that touch
1189 // the changed range.
1190 while start_ix > 0 {
1191 if ranges[start_ix - 1].end_byte == changed.start {
1192 start_ix -= 1;
1193 } else {
1194 break;
1195 }
1196 }
1197 while let Some(range) = ranges.get(end_ix) {
1198 if range.start_byte == changed.end {
1199 end_ix += 1;
1200 } else {
1201 break;
1202 }
1203 }
1204
1205 if end_ix > start_ix {
1206 ranges.splice(start_ix..end_ix, []);
1207 }
1208 changed_ranges.next();
1209 ranges_ix = start_ix;
1210 } else if let Some(new_range) = new_range {
1211 let ix = ranges_ix
1212 + match ranges[ranges_ix..]
1213 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1214 {
1215 Ok(ix) | Err(ix) => ix,
1216 };
1217 ranges.insert(ix, **new_range);
1218 new_ranges.next();
1219 ranges_ix = ix + 1;
1220 } else {
1221 break;
1222 }
1223 }
1224 ranges
1225}
1226
1227impl<'a> SyntaxLayerInfo<'a> {
1228 pub(crate) fn override_id(&self, offset: usize, text: &text::BufferSnapshot) -> Option<u32> {
1229 let text = TextProvider(text.as_rope());
1230 let config = self.language.grammar.as_ref()?.override_config.as_ref()?;
1231
1232 let mut query_cursor = QueryCursorHandle::new();
1233 query_cursor.set_byte_range(offset..offset);
1234
1235 let mut smallest_match: Option<(u32, Range<usize>)> = None;
1236 for mat in query_cursor.matches(&config.query, self.node, text) {
1237 for capture in mat.captures {
1238 if !config.values.contains_key(&capture.index) {
1239 continue;
1240 }
1241
1242 let range = capture.node.byte_range();
1243 if offset <= range.start || offset >= range.end {
1244 continue;
1245 }
1246
1247 if let Some((_, smallest_range)) = &smallest_match {
1248 if range.len() < smallest_range.len() {
1249 smallest_match = Some((capture.index, range))
1250 }
1251 continue;
1252 }
1253
1254 smallest_match = Some((capture.index, range));
1255 }
1256 }
1257
1258 smallest_match.map(|(index, _)| index)
1259 }
1260}
1261
1262impl std::ops::Deref for SyntaxMap {
1263 type Target = SyntaxSnapshot;
1264
1265 fn deref(&self) -> &Self::Target {
1266 &self.snapshot
1267 }
1268}
1269
1270impl PartialEq for ParseStep {
1271 fn eq(&self, _: &Self) -> bool {
1272 false
1273 }
1274}
1275
1276impl Eq for ParseStep {}
1277
1278impl PartialOrd for ParseStep {
1279 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1280 Some(self.cmp(&other))
1281 }
1282}
1283
1284impl Ord for ParseStep {
1285 fn cmp(&self, other: &Self) -> Ordering {
1286 let range_a = self.range();
1287 let range_b = other.range();
1288 Ord::cmp(&other.depth, &self.depth)
1289 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1290 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1291 .then_with(|| self.language.id().cmp(&other.language.id()))
1292 }
1293}
1294
1295impl ParseStep {
1296 fn range(&self) -> Range<usize> {
1297 if let ParseMode::Combined {
1298 parent_layer_range, ..
1299 } = &self.mode
1300 {
1301 parent_layer_range.clone()
1302 } else {
1303 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1304 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1305 start..end
1306 }
1307 }
1308}
1309
1310impl ChangedRegion {
1311 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1312 let range_a = &self.range;
1313 let range_b = &other.range;
1314 Ord::cmp(&self.depth, &other.depth)
1315 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1316 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1317 }
1318}
1319
1320impl ChangeRegionSet {
1321 fn start_position(&self) -> ChangeStartPosition {
1322 self.0.first().map_or(
1323 ChangeStartPosition {
1324 depth: usize::MAX,
1325 position: Anchor::MAX,
1326 },
1327 |region| ChangeStartPosition {
1328 depth: region.depth,
1329 position: region.range.start,
1330 },
1331 )
1332 }
1333
1334 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1335 for region in &self.0 {
1336 if region.depth < layer.depth {
1337 continue;
1338 }
1339 if region.depth > layer.depth {
1340 break;
1341 }
1342 if region.range.end.cmp(&layer.range.start, text).is_le() {
1343 continue;
1344 }
1345 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1346 break;
1347 }
1348 return true;
1349 }
1350 false
1351 }
1352
1353 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1354 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1355 self.0.insert(ix, region);
1356 }
1357 }
1358
1359 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1360 let prev_len = self.0.len();
1361 self.0.retain(|region| {
1362 region.depth > summary.max_depth
1363 || (region.depth == summary.max_depth
1364 && region
1365 .range
1366 .end
1367 .cmp(&summary.last_layer_range.start, text)
1368 .is_gt())
1369 });
1370 self.0.len() < prev_len
1371 }
1372}
1373
1374impl Default for SyntaxLayerSummary {
1375 fn default() -> Self {
1376 Self {
1377 max_depth: 0,
1378 min_depth: 0,
1379 range: Anchor::MAX..Anchor::MIN,
1380 last_layer_range: Anchor::MIN..Anchor::MAX,
1381 last_layer_language: None,
1382 contains_pending_layer: false,
1383 }
1384 }
1385}
1386
1387impl sum_tree::Summary for SyntaxLayerSummary {
1388 type Context = BufferSnapshot;
1389
1390 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1391 if other.max_depth > self.max_depth {
1392 self.max_depth = other.max_depth;
1393 self.range = other.range.clone();
1394 } else {
1395 if other.range.start.cmp(&self.range.start, buffer).is_lt() {
1396 self.range.start = other.range.start;
1397 }
1398 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1399 self.range.end = other.range.end;
1400 }
1401 }
1402 self.last_layer_range = other.last_layer_range.clone();
1403 self.last_layer_language = other.last_layer_language;
1404 self.contains_pending_layer |= other.contains_pending_layer;
1405 }
1406}
1407
1408impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1409 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1410 Ord::cmp(&self.depth, &cursor_location.max_depth)
1411 .then_with(|| {
1412 self.range
1413 .start
1414 .cmp(&cursor_location.last_layer_range.start, buffer)
1415 })
1416 .then_with(|| {
1417 cursor_location
1418 .last_layer_range
1419 .end
1420 .cmp(&self.range.end, buffer)
1421 })
1422 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1423 }
1424}
1425
1426impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for ChangeStartPosition {
1427 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1428 Ord::cmp(&self.depth, &cursor_location.max_depth)
1429 .then_with(|| self.position.cmp(&cursor_location.range.end, text))
1430 }
1431}
1432
1433impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1434 for SyntaxLayerPositionBeforeChange
1435{
1436 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1437 if self.change.cmp(cursor_location, buffer).is_le() {
1438 return Ordering::Less;
1439 } else {
1440 self.position.cmp(cursor_location, buffer)
1441 }
1442 }
1443}
1444
1445impl sum_tree::Item for SyntaxLayer {
1446 type Summary = SyntaxLayerSummary;
1447
1448 fn summary(&self) -> Self::Summary {
1449 SyntaxLayerSummary {
1450 min_depth: self.depth,
1451 max_depth: self.depth,
1452 range: self.range.clone(),
1453 last_layer_range: self.range.clone(),
1454 last_layer_language: self.content.language_id(),
1455 contains_pending_layer: matches!(self.content, SyntaxLayerContent::Pending { .. }),
1456 }
1457 }
1458}
1459
1460impl std::fmt::Debug for SyntaxLayer {
1461 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1462 f.debug_struct("SyntaxLayer")
1463 .field("depth", &self.depth)
1464 .field("range", &self.range)
1465 .field("tree", &self.content.tree())
1466 .finish()
1467 }
1468}
1469
1470impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1471 type I = ByteChunks<'a>;
1472
1473 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1474 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1475 }
1476}
1477
1478impl<'a> Iterator for ByteChunks<'a> {
1479 type Item = &'a [u8];
1480
1481 fn next(&mut self) -> Option<Self::Item> {
1482 self.0.next().map(str::as_bytes)
1483 }
1484}
1485
1486impl QueryCursorHandle {
1487 pub(crate) fn new() -> Self {
1488 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1489 cursor.set_match_limit(64);
1490 QueryCursorHandle(Some(cursor))
1491 }
1492}
1493
1494impl Deref for QueryCursorHandle {
1495 type Target = QueryCursor;
1496
1497 fn deref(&self) -> &Self::Target {
1498 self.0.as_ref().unwrap()
1499 }
1500}
1501
1502impl DerefMut for QueryCursorHandle {
1503 fn deref_mut(&mut self) -> &mut Self::Target {
1504 self.0.as_mut().unwrap()
1505 }
1506}
1507
1508impl Drop for QueryCursorHandle {
1509 fn drop(&mut self) {
1510 let mut cursor = self.0.take().unwrap();
1511 cursor.set_byte_range(0..usize::MAX);
1512 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1513 QUERY_CURSORS.lock().push(cursor)
1514 }
1515}
1516
1517pub(crate) trait ToTreeSitterPoint {
1518 fn to_ts_point(self) -> tree_sitter::Point;
1519 fn from_ts_point(point: tree_sitter::Point) -> Self;
1520}
1521
1522impl ToTreeSitterPoint for Point {
1523 fn to_ts_point(self) -> tree_sitter::Point {
1524 tree_sitter::Point::new(self.row as usize, self.column as usize)
1525 }
1526
1527 fn from_ts_point(point: tree_sitter::Point) -> Self {
1528 Point::new(point.row as u32, point.column as u32)
1529 }
1530}
1531
1532#[cfg(test)]
1533mod tests {
1534 use super::*;
1535 use crate::LanguageConfig;
1536 use rand::rngs::StdRng;
1537 use std::env;
1538 use text::Buffer;
1539 use unindent::Unindent as _;
1540 use util::test::marked_text_ranges;
1541
1542 #[test]
1543 fn test_splice_included_ranges() {
1544 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1545
1546 let new_ranges = splice_included_ranges(
1547 ranges.clone(),
1548 &[54..56, 58..68],
1549 &[ts_range(50..54), ts_range(59..67)],
1550 );
1551 assert_eq!(
1552 new_ranges,
1553 &[
1554 ts_range(20..30),
1555 ts_range(50..54),
1556 ts_range(59..67),
1557 ts_range(80..90),
1558 ]
1559 );
1560
1561 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1562 assert_eq!(
1563 new_ranges,
1564 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1565 );
1566
1567 let new_ranges =
1568 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1569 assert_eq!(
1570 new_ranges,
1571 &[
1572 ts_range(0..2),
1573 ts_range(20..30),
1574 ts_range(50..60),
1575 ts_range(70..75),
1576 ts_range(80..90)
1577 ]
1578 );
1579
1580 let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
1581 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
1582
1583 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1584 tree_sitter::Range {
1585 start_byte: range.start,
1586 start_point: tree_sitter::Point {
1587 row: 0,
1588 column: range.start,
1589 },
1590 end_byte: range.end,
1591 end_point: tree_sitter::Point {
1592 row: 0,
1593 column: range.end,
1594 },
1595 }
1596 }
1597 }
1598
1599 #[gpui::test]
1600 fn test_syntax_map_layers_for_range() {
1601 let registry = Arc::new(LanguageRegistry::test());
1602 let language = Arc::new(rust_lang());
1603 registry.add(language.clone());
1604
1605 let mut buffer = Buffer::new(
1606 0,
1607 0,
1608 r#"
1609 fn a() {
1610 assert_eq!(
1611 b(vec![C {}]),
1612 vec![d.e],
1613 );
1614 println!("{}", f(|_| true));
1615 }
1616 "#
1617 .unindent(),
1618 );
1619
1620 let mut syntax_map = SyntaxMap::new();
1621 syntax_map.set_language_registry(registry.clone());
1622 syntax_map.reparse(language.clone(), &buffer);
1623
1624 assert_layers_for_range(
1625 &syntax_map,
1626 &buffer,
1627 Point::new(2, 0)..Point::new(2, 0),
1628 &[
1629 "...(function_item ... (block (expression_statement (macro_invocation...",
1630 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1631 ],
1632 );
1633 assert_layers_for_range(
1634 &syntax_map,
1635 &buffer,
1636 Point::new(2, 14)..Point::new(2, 16),
1637 &[
1638 "...(function_item ...",
1639 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1640 "...(array_expression (struct_expression ...",
1641 ],
1642 );
1643 assert_layers_for_range(
1644 &syntax_map,
1645 &buffer,
1646 Point::new(3, 14)..Point::new(3, 16),
1647 &[
1648 "...(function_item ...",
1649 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1650 "...(array_expression (field_expression ...",
1651 ],
1652 );
1653 assert_layers_for_range(
1654 &syntax_map,
1655 &buffer,
1656 Point::new(5, 12)..Point::new(5, 16),
1657 &[
1658 "...(function_item ...",
1659 "...(call_expression ... (arguments (closure_expression ...",
1660 ],
1661 );
1662
1663 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1664 let macro_name_range = range_for_text(&buffer, "vec!");
1665 buffer.edit([(macro_name_range, "&")]);
1666 syntax_map.interpolate(&buffer);
1667 syntax_map.reparse(language.clone(), &buffer);
1668
1669 assert_layers_for_range(
1670 &syntax_map,
1671 &buffer,
1672 Point::new(2, 14)..Point::new(2, 16),
1673 &[
1674 "...(function_item ...",
1675 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1676 ],
1677 );
1678
1679 // Put the vec! macro back, adding back the syntactic layer.
1680 buffer.undo();
1681 syntax_map.interpolate(&buffer);
1682 syntax_map.reparse(language.clone(), &buffer);
1683
1684 assert_layers_for_range(
1685 &syntax_map,
1686 &buffer,
1687 Point::new(2, 14)..Point::new(2, 16),
1688 &[
1689 "...(function_item ...",
1690 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1691 "...(array_expression (struct_expression ...",
1692 ],
1693 );
1694 }
1695
1696 #[gpui::test]
1697 fn test_dynamic_language_injection() {
1698 let registry = Arc::new(LanguageRegistry::test());
1699 let markdown = Arc::new(markdown_lang());
1700 registry.add(markdown.clone());
1701 registry.add(Arc::new(rust_lang()));
1702 registry.add(Arc::new(ruby_lang()));
1703
1704 let mut buffer = Buffer::new(
1705 0,
1706 0,
1707 r#"
1708 This is a code block:
1709
1710 ```rs
1711 fn foo() {}
1712 ```
1713 "#
1714 .unindent(),
1715 );
1716
1717 let mut syntax_map = SyntaxMap::new();
1718 syntax_map.set_language_registry(registry.clone());
1719 syntax_map.reparse(markdown.clone(), &buffer);
1720 assert_layers_for_range(
1721 &syntax_map,
1722 &buffer,
1723 Point::new(3, 0)..Point::new(3, 0),
1724 &[
1725 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1726 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
1727 ],
1728 );
1729
1730 // Replace Rust with Ruby in code block.
1731 let macro_name_range = range_for_text(&buffer, "rs");
1732 buffer.edit([(macro_name_range, "ruby")]);
1733 syntax_map.interpolate(&buffer);
1734 syntax_map.reparse(markdown.clone(), &buffer);
1735 assert_layers_for_range(
1736 &syntax_map,
1737 &buffer,
1738 Point::new(3, 0)..Point::new(3, 0),
1739 &[
1740 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1741 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
1742 ],
1743 );
1744
1745 // Replace Ruby with a language that hasn't been loaded yet.
1746 let macro_name_range = range_for_text(&buffer, "ruby");
1747 buffer.edit([(macro_name_range, "erb")]);
1748 syntax_map.interpolate(&buffer);
1749 syntax_map.reparse(markdown.clone(), &buffer);
1750 assert_layers_for_range(
1751 &syntax_map,
1752 &buffer,
1753 Point::new(3, 0)..Point::new(3, 0),
1754 &[
1755 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
1756 ],
1757 );
1758 assert_eq!(
1759 syntax_map
1760 .unknown_injection_languages(&buffer)
1761 .collect::<Vec<_>>(),
1762 vec![&Arc::from("erb")]
1763 );
1764
1765 registry.add(Arc::new(erb_lang()));
1766 }
1767
1768 #[gpui::test]
1769 fn test_typing_multiple_new_injections() {
1770 let (buffer, syntax_map) = test_edit_sequence(
1771 "Rust",
1772 &[
1773 "fn a() { dbg }",
1774 "fn a() { dbg«!» }",
1775 "fn a() { dbg!«()» }",
1776 "fn a() { dbg!(«b») }",
1777 "fn a() { dbg!(b«.») }",
1778 "fn a() { dbg!(b.«c») }",
1779 "fn a() { dbg!(b.c«()») }",
1780 "fn a() { dbg!(b.c(«vec»)) }",
1781 "fn a() { dbg!(b.c(vec«!»)) }",
1782 "fn a() { dbg!(b.c(vec!«[]»)) }",
1783 "fn a() { dbg!(b.c(vec![«d»])) }",
1784 "fn a() { dbg!(b.c(vec![d«.»])) }",
1785 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1786 ],
1787 );
1788
1789 assert_capture_ranges(
1790 &syntax_map,
1791 &buffer,
1792 &["field"],
1793 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1794 );
1795 }
1796
1797 #[gpui::test]
1798 fn test_pasting_new_injection_line_between_others() {
1799 let (buffer, syntax_map) = test_edit_sequence(
1800 "Rust",
1801 &[
1802 "
1803 fn a() {
1804 b!(B {});
1805 c!(C {});
1806 d!(D {});
1807 e!(E {});
1808 f!(F {});
1809 g!(G {});
1810 }
1811 ",
1812 "
1813 fn a() {
1814 b!(B {});
1815 c!(C {});
1816 d!(D {});
1817 « h!(H {});
1818 » e!(E {});
1819 f!(F {});
1820 g!(G {});
1821 }
1822 ",
1823 ],
1824 );
1825
1826 assert_capture_ranges(
1827 &syntax_map,
1828 &buffer,
1829 &["struct"],
1830 "
1831 fn a() {
1832 b!(«B {}»);
1833 c!(«C {}»);
1834 d!(«D {}»);
1835 h!(«H {}»);
1836 e!(«E {}»);
1837 f!(«F {}»);
1838 g!(«G {}»);
1839 }
1840 ",
1841 );
1842 }
1843
1844 #[gpui::test]
1845 fn test_joining_injections_with_child_injections() {
1846 let (buffer, syntax_map) = test_edit_sequence(
1847 "Rust",
1848 &[
1849 "
1850 fn a() {
1851 b!(
1852 c![one.two.three],
1853 d![four.five.six],
1854 );
1855 e!(
1856 f![seven.eight],
1857 );
1858 }
1859 ",
1860 "
1861 fn a() {
1862 b!(
1863 c![one.two.three],
1864 d![four.five.six],
1865 ˇ f![seven.eight],
1866 );
1867 }
1868 ",
1869 ],
1870 );
1871
1872 assert_capture_ranges(
1873 &syntax_map,
1874 &buffer,
1875 &["field"],
1876 "
1877 fn a() {
1878 b!(
1879 c![one.«two».«three»],
1880 d![four.«five».«six»],
1881 f![seven.«eight»],
1882 );
1883 }
1884 ",
1885 );
1886 }
1887
1888 #[gpui::test]
1889 fn test_editing_edges_of_injection() {
1890 test_edit_sequence(
1891 "Rust",
1892 &[
1893 "
1894 fn a() {
1895 b!(c!())
1896 }
1897 ",
1898 "
1899 fn a() {
1900 «d»!(c!())
1901 }
1902 ",
1903 "
1904 fn a() {
1905 «e»d!(c!())
1906 }
1907 ",
1908 "
1909 fn a() {
1910 ed!«[»c!()«]»
1911 }
1912 ",
1913 ],
1914 );
1915 }
1916
1917 #[gpui::test]
1918 fn test_edits_preceding_and_intersecting_injection() {
1919 test_edit_sequence(
1920 "Rust",
1921 &[
1922 //
1923 "const aaaaaaaaaaaa: B = c!(d(e.f));",
1924 "const aˇa: B = c!(d(eˇ));",
1925 ],
1926 );
1927 }
1928
1929 #[gpui::test]
1930 fn test_non_local_changes_create_injections() {
1931 test_edit_sequence(
1932 "Rust",
1933 &[
1934 "
1935 // a! {
1936 static B: C = d;
1937 // }
1938 ",
1939 "
1940 ˇa! {
1941 static B: C = d;
1942 ˇ}
1943 ",
1944 ],
1945 );
1946 }
1947
1948 #[gpui::test]
1949 fn test_creating_many_injections_in_one_edit() {
1950 test_edit_sequence(
1951 "Rust",
1952 &[
1953 "
1954 fn a() {
1955 one(Two::three(3));
1956 four(Five::six(6));
1957 seven(Eight::nine(9));
1958 }
1959 ",
1960 "
1961 fn a() {
1962 one«!»(Two::three(3));
1963 four«!»(Five::six(6));
1964 seven«!»(Eight::nine(9));
1965 }
1966 ",
1967 "
1968 fn a() {
1969 one!(Two::three«!»(3));
1970 four!(Five::six«!»(6));
1971 seven!(Eight::nine«!»(9));
1972 }
1973 ",
1974 ],
1975 );
1976 }
1977
1978 #[gpui::test]
1979 fn test_editing_across_injection_boundary() {
1980 test_edit_sequence(
1981 "Rust",
1982 &[
1983 "
1984 fn one() {
1985 two();
1986 three!(
1987 three.four,
1988 five.six,
1989 );
1990 }
1991 ",
1992 "
1993 fn one() {
1994 two();
1995 th«irty_five![»
1996 three.four,
1997 five.six,
1998 « seven.eight,
1999 ];»
2000 }
2001 ",
2002 ],
2003 );
2004 }
2005
2006 #[gpui::test]
2007 fn test_removing_injection_by_replacing_across_boundary() {
2008 test_edit_sequence(
2009 "Rust",
2010 &[
2011 "
2012 fn one() {
2013 two!(
2014 three.four,
2015 );
2016 }
2017 ",
2018 "
2019 fn one() {
2020 t«en
2021 .eleven(
2022 twelve,
2023 »
2024 three.four,
2025 );
2026 }
2027 ",
2028 ],
2029 );
2030 }
2031
2032 #[gpui::test]
2033 fn test_combined_injections() {
2034 let (buffer, syntax_map) = test_edit_sequence(
2035 "ERB",
2036 &[
2037 "
2038 <body>
2039 <% if @one %>
2040 <div class=one>
2041 <% else %>
2042 <div class=two>
2043 <% end %>
2044 </div>
2045 </body>
2046 ",
2047 "
2048 <body>
2049 <% if @one %>
2050 <div class=one>
2051 ˇ else ˇ
2052 <div class=two>
2053 <% end %>
2054 </div>
2055 </body>
2056 ",
2057 "
2058 <body>
2059 <% if @one «;» end %>
2060 </div>
2061 </body>
2062 ",
2063 ],
2064 );
2065
2066 assert_capture_ranges(
2067 &syntax_map,
2068 &buffer,
2069 &["tag", "ivar"],
2070 "
2071 <«body»>
2072 <% if «@one» ; end %>
2073 </«div»>
2074 </«body»>
2075 ",
2076 );
2077 }
2078
2079 #[gpui::test]
2080 fn test_combined_injections_empty_ranges() {
2081 test_edit_sequence(
2082 "ERB",
2083 &[
2084 "
2085 <% if @one %>
2086 <% else %>
2087 <% end %>
2088 ",
2089 "
2090 <% if @one %>
2091 ˇ<% end %>
2092 ",
2093 ],
2094 );
2095 }
2096
2097 #[gpui::test]
2098 fn test_combined_injections_edit_edges_of_ranges() {
2099 let (buffer, syntax_map) = test_edit_sequence(
2100 "ERB",
2101 &[
2102 "
2103 <%= one @two %>
2104 <%= three @four %>
2105 ",
2106 "
2107 <%= one @two %ˇ
2108 <%= three @four %>
2109 ",
2110 "
2111 <%= one @two %«>»
2112 <%= three @four %>
2113 ",
2114 ],
2115 );
2116
2117 assert_capture_ranges(
2118 &syntax_map,
2119 &buffer,
2120 &["tag", "ivar"],
2121 "
2122 <%= one «@two» %>
2123 <%= three «@four» %>
2124 ",
2125 );
2126 }
2127
2128 #[gpui::test]
2129 fn test_combined_injections_splitting_some_injections() {
2130 let (_buffer, _syntax_map) = test_edit_sequence(
2131 "ERB",
2132 &[
2133 r#"
2134 <%A if b(:c) %>
2135 d
2136 <% end %>
2137 eee
2138 <% f %>
2139 "#,
2140 r#"
2141 <%« AAAAAAA %>
2142 hhhhhhh
2143 <%=» if b(:c) %>
2144 d
2145 <% end %>
2146 eee
2147 <% f %>
2148 "#,
2149 ],
2150 );
2151 }
2152
2153 #[gpui::test(iterations = 50)]
2154 fn test_random_syntax_map_edits(mut rng: StdRng) {
2155 let operations = env::var("OPERATIONS")
2156 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2157 .unwrap_or(10);
2158
2159 let text = r#"
2160 fn test_something() {
2161 let vec = vec![5, 1, 3, 8];
2162 assert_eq!(
2163 vec
2164 .into_iter()
2165 .map(|i| i * 2)
2166 .collect::<Vec<usize>>(),
2167 vec![
2168 5 * 2, 1 * 2, 3 * 2, 8 * 2
2169 ],
2170 );
2171 }
2172 "#
2173 .unindent()
2174 .repeat(2);
2175
2176 let registry = Arc::new(LanguageRegistry::test());
2177 let language = Arc::new(rust_lang());
2178 registry.add(language.clone());
2179 let mut buffer = Buffer::new(0, 0, text);
2180
2181 let mut syntax_map = SyntaxMap::new();
2182 syntax_map.set_language_registry(registry.clone());
2183 syntax_map.reparse(language.clone(), &buffer);
2184
2185 let mut reference_syntax_map = SyntaxMap::new();
2186 reference_syntax_map.set_language_registry(registry.clone());
2187
2188 log::info!("initial text:\n{}", buffer.text());
2189
2190 for _ in 0..operations {
2191 let prev_buffer = buffer.snapshot();
2192 let prev_syntax_map = syntax_map.snapshot();
2193
2194 buffer.randomly_edit(&mut rng, 3);
2195 log::info!("text:\n{}", buffer.text());
2196
2197 syntax_map.interpolate(&buffer);
2198 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2199
2200 syntax_map.reparse(language.clone(), &buffer);
2201
2202 reference_syntax_map.clear();
2203 reference_syntax_map.reparse(language.clone(), &buffer);
2204 }
2205
2206 for i in 0..operations {
2207 let i = operations - i - 1;
2208 buffer.undo();
2209 log::info!("undoing operation {}", i);
2210 log::info!("text:\n{}", buffer.text());
2211
2212 syntax_map.interpolate(&buffer);
2213 syntax_map.reparse(language.clone(), &buffer);
2214
2215 reference_syntax_map.clear();
2216 reference_syntax_map.reparse(language.clone(), &buffer);
2217 assert_eq!(
2218 syntax_map.layers(&buffer).len(),
2219 reference_syntax_map.layers(&buffer).len(),
2220 "wrong number of layers after undoing edit {i}"
2221 );
2222 }
2223
2224 let layers = syntax_map.layers(&buffer);
2225 let reference_layers = reference_syntax_map.layers(&buffer);
2226 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2227 {
2228 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2229 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2230 }
2231 }
2232
2233 #[gpui::test(iterations = 50)]
2234 fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) {
2235 let operations = env::var("OPERATIONS")
2236 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2237 .unwrap_or(10);
2238
2239 let text = r#"
2240 <div id="main">
2241 <% if one?(:two) %>
2242 <p class="three" four>
2243 <%= yield :five %>
2244 </p>
2245 <% elsif Six.seven(8) %>
2246 <p id="three" four>
2247 <%= yield :five %>
2248 </p>
2249 <% else %>
2250 <span>Ok</span>
2251 <% end %>
2252 </div>
2253 "#
2254 .unindent()
2255 .repeat(8);
2256
2257 let registry = Arc::new(LanguageRegistry::test());
2258 let language = Arc::new(erb_lang());
2259 registry.add(language.clone());
2260 registry.add(Arc::new(ruby_lang()));
2261 registry.add(Arc::new(html_lang()));
2262 let mut buffer = Buffer::new(0, 0, text);
2263
2264 let mut syntax_map = SyntaxMap::new();
2265 syntax_map.set_language_registry(registry.clone());
2266 syntax_map.reparse(language.clone(), &buffer);
2267
2268 let mut reference_syntax_map = SyntaxMap::new();
2269 reference_syntax_map.set_language_registry(registry.clone());
2270
2271 log::info!("initial text:\n{}", buffer.text());
2272
2273 for _ in 0..operations {
2274 let prev_buffer = buffer.snapshot();
2275 let prev_syntax_map = syntax_map.snapshot();
2276
2277 buffer.randomly_edit(&mut rng, 3);
2278 log::info!("text:\n{}", buffer.text());
2279
2280 syntax_map.interpolate(&buffer);
2281 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2282
2283 syntax_map.reparse(language.clone(), &buffer);
2284
2285 reference_syntax_map.clear();
2286 reference_syntax_map.reparse(language.clone(), &buffer);
2287 }
2288
2289 for i in 0..operations {
2290 let i = operations - i - 1;
2291 buffer.undo();
2292 log::info!("undoing operation {}", i);
2293 log::info!("text:\n{}", buffer.text());
2294
2295 syntax_map.interpolate(&buffer);
2296 syntax_map.reparse(language.clone(), &buffer);
2297
2298 reference_syntax_map.clear();
2299 reference_syntax_map.reparse(language.clone(), &buffer);
2300 assert_eq!(
2301 syntax_map.layers(&buffer).len(),
2302 reference_syntax_map.layers(&buffer).len(),
2303 "wrong number of layers after undoing edit {i}"
2304 );
2305 }
2306
2307 let layers = syntax_map.layers(&buffer);
2308 let reference_layers = reference_syntax_map.layers(&buffer);
2309 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2310 {
2311 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2312 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2313 }
2314 }
2315
2316 fn check_interpolation(
2317 old_syntax_map: &SyntaxSnapshot,
2318 new_syntax_map: &SyntaxSnapshot,
2319 old_buffer: &BufferSnapshot,
2320 new_buffer: &BufferSnapshot,
2321 ) {
2322 let edits = new_buffer
2323 .edits_since::<usize>(&old_buffer.version())
2324 .collect::<Vec<_>>();
2325
2326 for (old_layer, new_layer) in old_syntax_map
2327 .layers
2328 .iter()
2329 .zip(new_syntax_map.layers.iter())
2330 {
2331 assert_eq!(old_layer.range, new_layer.range);
2332 let Some(old_tree) = old_layer.content.tree() else { continue };
2333 let Some(new_tree) = new_layer.content.tree() else { continue };
2334 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
2335 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
2336 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
2337 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
2338 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
2339 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
2340 check_node_edits(
2341 old_layer.depth,
2342 &old_layer.range,
2343 old_node,
2344 new_node,
2345 old_buffer,
2346 new_buffer,
2347 &edits,
2348 );
2349 }
2350
2351 fn check_node_edits(
2352 depth: usize,
2353 range: &Range<Anchor>,
2354 old_node: Node,
2355 new_node: Node,
2356 old_buffer: &BufferSnapshot,
2357 new_buffer: &BufferSnapshot,
2358 edits: &[text::Edit<usize>],
2359 ) {
2360 assert_eq!(old_node.kind(), new_node.kind());
2361
2362 let old_range = old_node.byte_range();
2363 let new_range = new_node.byte_range();
2364
2365 let is_edited = edits
2366 .iter()
2367 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
2368 if is_edited {
2369 assert!(
2370 new_node.has_changes(),
2371 concat!(
2372 "failed to mark node as edited.\n",
2373 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2374 "node kind: {}, old node range: {:?}, new node range: {:?}",
2375 ),
2376 depth,
2377 range.to_offset(old_buffer),
2378 range.to_offset(new_buffer),
2379 new_node.kind(),
2380 old_range,
2381 new_range,
2382 );
2383 }
2384
2385 if !new_node.has_changes() {
2386 assert_eq!(
2387 old_buffer
2388 .text_for_range(old_range.clone())
2389 .collect::<String>(),
2390 new_buffer
2391 .text_for_range(new_range.clone())
2392 .collect::<String>(),
2393 concat!(
2394 "mismatched text for node\n",
2395 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2396 "node kind: {}, old node range:{:?}, new node range:{:?}",
2397 ),
2398 depth,
2399 range.to_offset(old_buffer),
2400 range.to_offset(new_buffer),
2401 new_node.kind(),
2402 old_range,
2403 new_range,
2404 );
2405 }
2406
2407 for i in 0..new_node.child_count() {
2408 check_node_edits(
2409 depth,
2410 range,
2411 old_node.child(i).unwrap(),
2412 new_node.child(i).unwrap(),
2413 old_buffer,
2414 new_buffer,
2415 edits,
2416 )
2417 }
2418 }
2419 }
2420
2421 fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
2422 let registry = Arc::new(LanguageRegistry::test());
2423 registry.add(Arc::new(rust_lang()));
2424 registry.add(Arc::new(ruby_lang()));
2425 registry.add(Arc::new(html_lang()));
2426 registry.add(Arc::new(erb_lang()));
2427 registry.add(Arc::new(markdown_lang()));
2428 let language = registry.language_for_name(language_name).unwrap();
2429 let mut buffer = Buffer::new(0, 0, Default::default());
2430
2431 let mut mutated_syntax_map = SyntaxMap::new();
2432 mutated_syntax_map.set_language_registry(registry.clone());
2433 mutated_syntax_map.reparse(language.clone(), &buffer);
2434
2435 for (i, marked_string) in steps.into_iter().enumerate() {
2436 buffer.edit_via_marked_text(&marked_string.unindent());
2437
2438 // Reparse the syntax map
2439 mutated_syntax_map.interpolate(&buffer);
2440 mutated_syntax_map.reparse(language.clone(), &buffer);
2441
2442 // Create a second syntax map from scratch
2443 let mut reference_syntax_map = SyntaxMap::new();
2444 reference_syntax_map.set_language_registry(registry.clone());
2445 reference_syntax_map.reparse(language.clone(), &buffer);
2446
2447 // Compare the mutated syntax map to the new syntax map
2448 let mutated_layers = mutated_syntax_map.layers(&buffer);
2449 let reference_layers = reference_syntax_map.layers(&buffer);
2450 assert_eq!(
2451 mutated_layers.len(),
2452 reference_layers.len(),
2453 "wrong number of layers at step {i}"
2454 );
2455 for (edited_layer, reference_layer) in
2456 mutated_layers.into_iter().zip(reference_layers.into_iter())
2457 {
2458 assert_eq!(
2459 edited_layer.node.to_sexp(),
2460 reference_layer.node.to_sexp(),
2461 "different layer at step {i}"
2462 );
2463 assert_eq!(
2464 edited_layer.node.range(),
2465 reference_layer.node.range(),
2466 "different layer at step {i}"
2467 );
2468 }
2469 }
2470
2471 (buffer, mutated_syntax_map)
2472 }
2473
2474 fn html_lang() -> Language {
2475 Language::new(
2476 LanguageConfig {
2477 name: "HTML".into(),
2478 path_suffixes: vec!["html".to_string()],
2479 ..Default::default()
2480 },
2481 Some(tree_sitter_html::language()),
2482 )
2483 .with_highlights_query(
2484 r#"
2485 (tag_name) @tag
2486 (erroneous_end_tag_name) @tag
2487 (attribute_name) @property
2488 "#,
2489 )
2490 .unwrap()
2491 }
2492
2493 fn ruby_lang() -> Language {
2494 Language::new(
2495 LanguageConfig {
2496 name: "Ruby".into(),
2497 path_suffixes: vec!["rb".to_string()],
2498 ..Default::default()
2499 },
2500 Some(tree_sitter_ruby::language()),
2501 )
2502 .with_highlights_query(
2503 r#"
2504 ["if" "do" "else" "end"] @keyword
2505 (instance_variable) @ivar
2506 "#,
2507 )
2508 .unwrap()
2509 }
2510
2511 fn erb_lang() -> Language {
2512 Language::new(
2513 LanguageConfig {
2514 name: "ERB".into(),
2515 path_suffixes: vec!["erb".to_string()],
2516 ..Default::default()
2517 },
2518 Some(tree_sitter_embedded_template::language()),
2519 )
2520 .with_highlights_query(
2521 r#"
2522 ["<%" "%>"] @keyword
2523 "#,
2524 )
2525 .unwrap()
2526 .with_injection_query(
2527 r#"
2528 ((code) @content
2529 (#set! "language" "ruby")
2530 (#set! "combined"))
2531
2532 ((content) @content
2533 (#set! "language" "html")
2534 (#set! "combined"))
2535 "#,
2536 )
2537 .unwrap()
2538 }
2539
2540 fn rust_lang() -> Language {
2541 Language::new(
2542 LanguageConfig {
2543 name: "Rust".into(),
2544 path_suffixes: vec!["rs".to_string()],
2545 ..Default::default()
2546 },
2547 Some(tree_sitter_rust::language()),
2548 )
2549 .with_highlights_query(
2550 r#"
2551 (field_identifier) @field
2552 (struct_expression) @struct
2553 "#,
2554 )
2555 .unwrap()
2556 .with_injection_query(
2557 r#"
2558 (macro_invocation
2559 (token_tree) @content
2560 (#set! "language" "rust"))
2561 "#,
2562 )
2563 .unwrap()
2564 }
2565
2566 fn markdown_lang() -> Language {
2567 Language::new(
2568 LanguageConfig {
2569 name: "Markdown".into(),
2570 path_suffixes: vec!["md".into()],
2571 ..Default::default()
2572 },
2573 Some(tree_sitter_markdown::language()),
2574 )
2575 .with_injection_query(
2576 r#"
2577 (fenced_code_block
2578 (info_string
2579 (language) @language)
2580 (code_fence_content) @content)
2581 "#,
2582 )
2583 .unwrap()
2584 }
2585
2586 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2587 let start = buffer.as_rope().to_string().find(text).unwrap();
2588 start..start + text.len()
2589 }
2590
2591 fn assert_layers_for_range(
2592 syntax_map: &SyntaxMap,
2593 buffer: &BufferSnapshot,
2594 range: Range<Point>,
2595 expected_layers: &[&str],
2596 ) {
2597 let layers = syntax_map
2598 .layers_for_range(range, &buffer)
2599 .collect::<Vec<_>>();
2600 assert_eq!(
2601 layers.len(),
2602 expected_layers.len(),
2603 "wrong number of layers"
2604 );
2605 for (i, (SyntaxLayerInfo { node, .. }, expected_s_exp)) in
2606 layers.iter().zip(expected_layers.iter()).enumerate()
2607 {
2608 let actual_s_exp = node.to_sexp();
2609 assert!(
2610 string_contains_sequence(
2611 &actual_s_exp,
2612 &expected_s_exp.split("...").collect::<Vec<_>>()
2613 ),
2614 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2615 );
2616 }
2617 }
2618
2619 fn assert_capture_ranges(
2620 syntax_map: &SyntaxMap,
2621 buffer: &BufferSnapshot,
2622 highlight_query_capture_names: &[&str],
2623 marked_string: &str,
2624 ) {
2625 let mut actual_ranges = Vec::<Range<usize>>::new();
2626 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2627 grammar.highlights_query.as_ref()
2628 });
2629 let queries = captures
2630 .grammars()
2631 .iter()
2632 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2633 .collect::<Vec<_>>();
2634 for capture in captures {
2635 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2636 if highlight_query_capture_names.contains(&name.as_str()) {
2637 actual_ranges.push(capture.node.byte_range());
2638 }
2639 }
2640
2641 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2642 assert_eq!(text, buffer.text());
2643 assert_eq!(actual_ranges, expected_ranges);
2644 }
2645
2646 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2647 let mut last_part_end = 0;
2648 for part in parts {
2649 if let Some(start_ix) = text[last_part_end..].find(part) {
2650 last_part_end = start_ix + part.len();
2651 } else {
2652 return false;
2653 }
2654 }
2655 true
2656 }
2657}