1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use lazy_static::lazy_static;
4use parking_lot::Mutex;
5use std::{
6 borrow::Cow,
7 cell::RefCell,
8 cmp::{self, Ordering, Reverse},
9 collections::BinaryHeap,
10 iter,
11 ops::{Deref, DerefMut, Range},
12 sync::Arc,
13};
14use sum_tree::{Bias, SeekTarget, SumTree};
15use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
16use tree_sitter::{
17 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
18};
19
20thread_local! {
21 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
22}
23
24lazy_static! {
25 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
26}
27
28#[derive(Default)]
29pub struct SyntaxMap {
30 snapshot: SyntaxSnapshot,
31 language_registry: Option<Arc<LanguageRegistry>>,
32}
33
34#[derive(Clone, Default)]
35pub struct SyntaxSnapshot {
36 layers: SumTree<SyntaxLayer>,
37 parsed_version: clock::Global,
38 interpolated_version: clock::Global,
39 language_registry_version: usize,
40}
41
42#[derive(Default)]
43pub struct SyntaxMapCaptures<'a> {
44 layers: Vec<SyntaxMapCapturesLayer<'a>>,
45 active_layer_count: usize,
46 grammars: Vec<&'a Grammar>,
47}
48
49#[derive(Default)]
50pub struct SyntaxMapMatches<'a> {
51 layers: Vec<SyntaxMapMatchesLayer<'a>>,
52 active_layer_count: usize,
53 grammars: Vec<&'a Grammar>,
54}
55
56#[derive(Debug)]
57pub struct SyntaxMapCapture<'a> {
58 pub depth: usize,
59 pub node: Node<'a>,
60 pub index: u32,
61 pub grammar_index: usize,
62}
63
64#[derive(Debug)]
65pub struct SyntaxMapMatch<'a> {
66 pub depth: usize,
67 pub pattern_index: usize,
68 pub captures: &'a [QueryCapture<'a>],
69 pub grammar_index: usize,
70}
71
72struct SyntaxMapCapturesLayer<'a> {
73 depth: usize,
74 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
75 next_capture: Option<QueryCapture<'a>>,
76 grammar_index: usize,
77 _query_cursor: QueryCursorHandle,
78}
79
80struct SyntaxMapMatchesLayer<'a> {
81 depth: usize,
82 next_pattern_index: usize,
83 next_captures: Vec<QueryCapture<'a>>,
84 has_next: bool,
85 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
86 grammar_index: usize,
87 _query_cursor: QueryCursorHandle,
88}
89
90#[derive(Clone)]
91struct SyntaxLayer {
92 depth: usize,
93 range: Range<Anchor>,
94 content: SyntaxLayerContent,
95}
96
97#[derive(Clone)]
98enum SyntaxLayerContent {
99 Parsed {
100 tree: tree_sitter::Tree,
101 language: Arc<Language>,
102 },
103 Pending {
104 language_name: Arc<str>,
105 },
106}
107
108impl SyntaxLayerContent {
109 fn language_id(&self) -> Option<usize> {
110 match self {
111 SyntaxLayerContent::Parsed { language, .. } => language.id(),
112 SyntaxLayerContent::Pending { .. } => None,
113 }
114 }
115
116 fn tree(&self) -> Option<&Tree> {
117 match self {
118 SyntaxLayerContent::Parsed { tree, .. } => Some(tree),
119 SyntaxLayerContent::Pending { .. } => None,
120 }
121 }
122}
123
124#[derive(Debug)]
125pub struct SyntaxLayerInfo<'a> {
126 pub depth: usize,
127 pub node: Node<'a>,
128 pub language: &'a Arc<Language>,
129}
130
131#[derive(Debug, Clone)]
132struct SyntaxLayerSummary {
133 min_depth: usize,
134 max_depth: usize,
135 range: Range<Anchor>,
136 last_layer_range: Range<Anchor>,
137 last_layer_language: Option<usize>,
138 contains_unknown_injections: bool,
139}
140
141#[derive(Clone, Debug)]
142struct SyntaxLayerPosition {
143 depth: usize,
144 range: Range<Anchor>,
145 language: Option<usize>,
146}
147
148#[derive(Clone, Debug)]
149struct ChangeStartPosition {
150 depth: usize,
151 position: Anchor,
152}
153
154#[derive(Clone, Debug)]
155struct SyntaxLayerPositionBeforeChange {
156 position: SyntaxLayerPosition,
157 change: ChangeStartPosition,
158}
159
160struct ParseStep {
161 depth: usize,
162 language: ParseStepLanguage,
163 range: Range<Anchor>,
164 included_ranges: Vec<tree_sitter::Range>,
165 mode: ParseMode,
166}
167
168#[derive(Debug)]
169enum ParseStepLanguage {
170 Loaded { language: Arc<Language> },
171 Pending { name: Arc<str> },
172}
173
174impl ParseStepLanguage {
175 fn id(&self) -> Option<usize> {
176 match self {
177 ParseStepLanguage::Loaded { language } => language.id(),
178 ParseStepLanguage::Pending { .. } => None,
179 }
180 }
181}
182
183enum ParseMode {
184 Single,
185 Combined {
186 parent_layer_range: Range<usize>,
187 parent_layer_changed_ranges: Vec<Range<usize>>,
188 },
189}
190
191#[derive(Debug, PartialEq, Eq)]
192struct ChangedRegion {
193 depth: usize,
194 range: Range<Anchor>,
195}
196
197#[derive(Default)]
198struct ChangeRegionSet(Vec<ChangedRegion>);
199
200struct TextProvider<'a>(&'a Rope);
201
202struct ByteChunks<'a>(text::Chunks<'a>);
203
204struct QueryCursorHandle(Option<QueryCursor>);
205
206impl SyntaxMap {
207 pub fn new() -> Self {
208 Self::default()
209 }
210
211 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
212 self.language_registry = Some(registry);
213 }
214
215 pub fn snapshot(&self) -> SyntaxSnapshot {
216 self.snapshot.clone()
217 }
218
219 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
220 self.language_registry.clone()
221 }
222
223 pub fn interpolate(&mut self, text: &BufferSnapshot) {
224 self.snapshot.interpolate(text);
225 }
226
227 #[cfg(test)]
228 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
229 self.snapshot
230 .reparse(text, self.language_registry.clone(), language);
231 }
232
233 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot) {
234 self.snapshot = snapshot;
235 }
236
237 pub fn clear(&mut self) {
238 self.snapshot = SyntaxSnapshot::default();
239 }
240}
241
242impl SyntaxSnapshot {
243 pub fn is_empty(&self) -> bool {
244 self.layers.is_empty()
245 }
246
247 fn interpolate(&mut self, text: &BufferSnapshot) {
248 let edits = text
249 .anchored_edits_since::<(usize, Point)>(&self.interpolated_version)
250 .collect::<Vec<_>>();
251 self.interpolated_version = text.version().clone();
252
253 if edits.is_empty() {
254 return;
255 }
256
257 let mut layers = SumTree::new();
258 let mut first_edit_ix_for_depth = 0;
259 let mut prev_depth = 0;
260 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
261 cursor.next(text);
262
263 'outer: loop {
264 let depth = cursor.end(text).max_depth;
265 if depth > prev_depth {
266 first_edit_ix_for_depth = 0;
267 prev_depth = depth;
268 }
269
270 // Preserve any layers at this depth that precede the first edit.
271 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
272 let target = ChangeStartPosition {
273 depth,
274 position: edit_range.start,
275 };
276 if target.cmp(&cursor.start(), text).is_gt() {
277 let slice = cursor.slice(&target, Bias::Left, text);
278 layers.push_tree(slice, text);
279 }
280 }
281 // If this layer follows all of the edits, then preserve it and any
282 // subsequent layers at this same depth.
283 else if cursor.item().is_some() {
284 let slice = cursor.slice(
285 &SyntaxLayerPosition {
286 depth: depth + 1,
287 range: Anchor::MIN..Anchor::MAX,
288 language: None,
289 },
290 Bias::Left,
291 text,
292 );
293 layers.push_tree(slice, text);
294 continue;
295 };
296
297 let Some(layer) = cursor.item() else { break };
298 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
299
300 // Ignore edits that end before the start of this layer, and don't consider them
301 // for any subsequent layers at this same depth.
302 loop {
303 let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) else { continue 'outer };
304 if edit_range.end.cmp(&layer.range.start, text).is_le() {
305 first_edit_ix_for_depth += 1;
306 } else {
307 break;
308 }
309 }
310
311 let mut layer = layer.clone();
312 if let SyntaxLayerContent::Parsed { tree, .. } = &mut layer.content {
313 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
314 // Ignore any edits that follow this layer.
315 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
316 break;
317 }
318
319 // Apply any edits that intersect this layer to the layer's syntax tree.
320 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
321 tree_sitter::InputEdit {
322 start_byte: edit.new.start.0 - start_byte,
323 old_end_byte: edit.new.start.0 - start_byte
324 + (edit.old.end.0 - edit.old.start.0),
325 new_end_byte: edit.new.end.0 - start_byte,
326 start_position: (edit.new.start.1 - start_point).to_ts_point(),
327 old_end_position: (edit.new.start.1 - start_point
328 + (edit.old.end.1 - edit.old.start.1))
329 .to_ts_point(),
330 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
331 }
332 } else {
333 let node = tree.root_node();
334 tree_sitter::InputEdit {
335 start_byte: 0,
336 old_end_byte: node.end_byte(),
337 new_end_byte: 0,
338 start_position: Default::default(),
339 old_end_position: node.end_position(),
340 new_end_position: Default::default(),
341 }
342 };
343
344 tree.edit(&tree_edit);
345 }
346
347 debug_assert!(
348 tree.root_node().end_byte() <= text.len(),
349 "tree's size {}, is larger than text size {}",
350 tree.root_node().end_byte(),
351 text.len(),
352 );
353 }
354
355 layers.push(layer, text);
356 cursor.next(text);
357 }
358
359 layers.push_tree(cursor.suffix(&text), &text);
360 drop(cursor);
361 self.layers = layers;
362 }
363
364 pub fn reparse(
365 &mut self,
366 text: &BufferSnapshot,
367 registry: Option<Arc<LanguageRegistry>>,
368 root_language: Arc<Language>,
369 ) {
370 let edit_ranges = text
371 .edits_since::<usize>(&self.parsed_version)
372 .map(|edit| edit.new)
373 .collect::<Vec<_>>();
374 self.reparse_with_ranges(text, root_language.clone(), edit_ranges, registry.as_ref());
375
376 if let Some(registry) = registry {
377 if registry.version() != self.language_registry_version {
378 let mut resolved_injection_ranges = Vec::new();
379 let mut cursor = self
380 .layers
381 .filter::<_, ()>(|summary| summary.contains_unknown_injections);
382 cursor.next(text);
383 while let Some(layer) = cursor.item() {
384 let SyntaxLayerContent::Pending { language_name } = &layer.content else { unreachable!() };
385 if {
386 let language_registry = ®istry;
387 language_registry.language_for_name_or_extension(language_name)
388 }
389 .is_some()
390 {
391 resolved_injection_ranges.push(layer.range.to_offset(text));
392 }
393
394 cursor.next(text);
395 }
396 drop(cursor);
397
398 if !resolved_injection_ranges.is_empty() {
399 self.reparse_with_ranges(
400 text,
401 root_language,
402 resolved_injection_ranges,
403 Some(®istry),
404 );
405 }
406 self.language_registry_version = registry.version();
407 }
408 }
409 }
410
411 fn reparse_with_ranges(
412 &mut self,
413 text: &BufferSnapshot,
414 root_language: Arc<Language>,
415 invalidated_ranges: Vec<Range<usize>>,
416 registry: Option<&Arc<LanguageRegistry>>,
417 ) {
418 let max_depth = self.layers.summary().max_depth;
419 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
420 cursor.next(&text);
421 let mut layers = SumTree::new();
422
423 let mut changed_regions = ChangeRegionSet::default();
424 let mut queue = BinaryHeap::new();
425 let mut combined_injection_ranges = HashMap::default();
426 queue.push(ParseStep {
427 depth: 0,
428 language: ParseStepLanguage::Loaded {
429 language: root_language,
430 },
431 included_ranges: vec![tree_sitter::Range {
432 start_byte: 0,
433 end_byte: text.len(),
434 start_point: Point::zero().to_ts_point(),
435 end_point: text.max_point().to_ts_point(),
436 }],
437 range: Anchor::MIN..Anchor::MAX,
438 mode: ParseMode::Single,
439 });
440
441 loop {
442 let step = queue.pop();
443 let position = if let Some(step) = &step {
444 SyntaxLayerPosition {
445 depth: step.depth,
446 range: step.range.clone(),
447 language: step.language.id(),
448 }
449 } else {
450 SyntaxLayerPosition {
451 depth: max_depth + 1,
452 range: Anchor::MAX..Anchor::MAX,
453 language: None,
454 }
455 };
456
457 let mut done = cursor.item().is_none();
458 while !done && position.cmp(&cursor.end(text), &text).is_gt() {
459 done = true;
460
461 let bounded_position = SyntaxLayerPositionBeforeChange {
462 position: position.clone(),
463 change: changed_regions.start_position(),
464 };
465 if bounded_position.cmp(&cursor.start(), &text).is_gt() {
466 let slice = cursor.slice(&bounded_position, Bias::Left, text);
467 if !slice.is_empty() {
468 layers.push_tree(slice, &text);
469 if changed_regions.prune(cursor.end(text), text) {
470 done = false;
471 }
472 }
473 }
474
475 while position.cmp(&cursor.end(text), text).is_gt() {
476 let Some(layer) = cursor.item() else { break };
477
478 if changed_regions.intersects(&layer, text) {
479 changed_regions.insert(
480 ChangedRegion {
481 depth: layer.depth + 1,
482 range: layer.range.clone(),
483 },
484 text,
485 );
486 } else {
487 layers.push(layer.clone(), text);
488 }
489
490 cursor.next(text);
491 if changed_regions.prune(cursor.end(text), text) {
492 done = false;
493 }
494 }
495 }
496
497 let Some(step) = step else { break };
498 let (step_start_byte, step_start_point) =
499 step.range.start.summary::<(usize, Point)>(text);
500 let step_end_byte = step.range.end.to_offset(text);
501
502 let mut old_layer = cursor.item();
503 if let Some(layer) = old_layer {
504 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
505 && layer.content.language_id() == step.language.id()
506 {
507 cursor.next(&text);
508 } else {
509 old_layer = None;
510 }
511 }
512
513 let content = match step.language {
514 ParseStepLanguage::Loaded { language } => {
515 let Some(grammar) = language.grammar() else { continue };
516 let tree;
517 let changed_ranges;
518
519 let mut included_ranges = step.included_ranges;
520 for range in &mut included_ranges {
521 range.start_byte -= step_start_byte;
522 range.end_byte -= step_start_byte;
523 range.start_point = (Point::from_ts_point(range.start_point)
524 - step_start_point)
525 .to_ts_point();
526 range.end_point = (Point::from_ts_point(range.end_point)
527 - step_start_point)
528 .to_ts_point();
529 }
530
531 if let Some(SyntaxLayerContent::Parsed { tree: old_tree, .. }) =
532 old_layer.map(|layer| &layer.content)
533 {
534 if let ParseMode::Combined {
535 mut parent_layer_changed_ranges,
536 ..
537 } = step.mode
538 {
539 for range in &mut parent_layer_changed_ranges {
540 range.start -= step_start_byte;
541 range.end -= step_start_byte;
542 }
543
544 included_ranges = splice_included_ranges(
545 old_tree.included_ranges(),
546 &parent_layer_changed_ranges,
547 &included_ranges,
548 );
549 }
550
551 tree = parse_text(
552 grammar,
553 text.as_rope(),
554 step_start_byte,
555 included_ranges,
556 Some(old_tree.clone()),
557 );
558 changed_ranges = join_ranges(
559 invalidated_ranges.iter().cloned().filter(|range| {
560 range.start <= step_end_byte && range.end >= step_start_byte
561 }),
562 old_tree.changed_ranges(&tree).map(|r| {
563 step_start_byte + r.start_byte..step_start_byte + r.end_byte
564 }),
565 );
566 } else {
567 tree = parse_text(
568 grammar,
569 text.as_rope(),
570 step_start_byte,
571 included_ranges,
572 None,
573 );
574 changed_ranges = vec![step_start_byte..step_end_byte];
575 }
576
577 if let (Some((config, registry)), false) = (
578 grammar.injection_config.as_ref().zip(registry.as_ref()),
579 changed_ranges.is_empty(),
580 ) {
581 for range in &changed_ranges {
582 changed_regions.insert(
583 ChangedRegion {
584 depth: step.depth + 1,
585 range: text.anchor_before(range.start)
586 ..text.anchor_after(range.end),
587 },
588 text,
589 );
590 }
591 get_injections(
592 config,
593 text,
594 tree.root_node_with_offset(
595 step_start_byte,
596 step_start_point.to_ts_point(),
597 ),
598 registry,
599 step.depth + 1,
600 &changed_ranges,
601 &mut combined_injection_ranges,
602 &mut queue,
603 );
604 }
605
606 SyntaxLayerContent::Parsed { tree, language }
607 }
608 ParseStepLanguage::Pending { name } => SyntaxLayerContent::Pending {
609 language_name: name,
610 },
611 };
612
613 layers.push(
614 SyntaxLayer {
615 depth: step.depth,
616 range: step.range,
617 content,
618 },
619 &text,
620 );
621 }
622
623 drop(cursor);
624 self.layers = layers;
625 self.interpolated_version = text.version.clone();
626 self.parsed_version = text.version.clone();
627 #[cfg(debug_assertions)]
628 self.check_invariants(text);
629 }
630
631 #[cfg(debug_assertions)]
632 fn check_invariants(&self, text: &BufferSnapshot) {
633 let mut max_depth = 0;
634 let mut prev_range: Option<Range<Anchor>> = None;
635 for layer in self.layers.iter() {
636 if layer.depth == max_depth {
637 if let Some(prev_range) = prev_range {
638 match layer.range.start.cmp(&prev_range.start, text) {
639 Ordering::Less => panic!("layers out of order"),
640 Ordering::Equal => {
641 assert!(layer.range.end.cmp(&prev_range.end, text).is_ge())
642 }
643 Ordering::Greater => {}
644 }
645 }
646 } else if layer.depth < max_depth {
647 panic!("layers out of order")
648 }
649 max_depth = layer.depth;
650 prev_range = Some(layer.range.clone());
651 }
652 }
653
654 pub fn single_tree_captures<'a>(
655 range: Range<usize>,
656 text: &'a Rope,
657 tree: &'a Tree,
658 language: &'a Arc<Language>,
659 query: fn(&Grammar) -> Option<&Query>,
660 ) -> SyntaxMapCaptures<'a> {
661 SyntaxMapCaptures::new(
662 range.clone(),
663 text,
664 [SyntaxLayerInfo {
665 language,
666 depth: 0,
667 node: tree.root_node(),
668 }]
669 .into_iter(),
670 query,
671 )
672 }
673
674 pub fn captures<'a>(
675 &'a self,
676 range: Range<usize>,
677 buffer: &'a BufferSnapshot,
678 query: fn(&Grammar) -> Option<&Query>,
679 ) -> SyntaxMapCaptures {
680 SyntaxMapCaptures::new(
681 range.clone(),
682 buffer.as_rope(),
683 self.layers_for_range(range, buffer).into_iter(),
684 query,
685 )
686 }
687
688 pub fn matches<'a>(
689 &'a self,
690 range: Range<usize>,
691 buffer: &'a BufferSnapshot,
692 query: fn(&Grammar) -> Option<&Query>,
693 ) -> SyntaxMapMatches {
694 SyntaxMapMatches::new(
695 range.clone(),
696 buffer.as_rope(),
697 self.layers_for_range(range, buffer).into_iter(),
698 query,
699 )
700 }
701
702 #[cfg(test)]
703 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
704 self.layers_for_range(0..buffer.len(), buffer).collect()
705 }
706
707 pub fn layers_for_range<'a, T: ToOffset>(
708 &'a self,
709 range: Range<T>,
710 buffer: &'a BufferSnapshot,
711 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
712 let start = buffer.anchor_before(range.start.to_offset(buffer));
713 let end = buffer.anchor_after(range.end.to_offset(buffer));
714
715 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
716 if summary.max_depth > summary.min_depth {
717 true
718 } else {
719 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
720 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
721 !is_before_start && !is_after_end
722 }
723 });
724
725 cursor.next(buffer);
726 iter::from_fn(move || {
727 while let Some(layer) = cursor.item() {
728 if let SyntaxLayerContent::Parsed { tree, language } = &layer.content {
729 let info = SyntaxLayerInfo {
730 language,
731 depth: layer.depth,
732 node: tree.root_node_with_offset(
733 layer.range.start.to_offset(buffer),
734 layer.range.start.to_point(buffer).to_ts_point(),
735 ),
736 };
737 cursor.next(buffer);
738 return Some(info);
739 } else {
740 cursor.next(buffer);
741 }
742 }
743 None
744 })
745 }
746
747 pub fn contains_unknown_injections(&self) -> bool {
748 self.layers.summary().contains_unknown_injections
749 }
750
751 pub fn language_registry_version(&self) -> usize {
752 self.language_registry_version
753 }
754}
755
756impl<'a> SyntaxMapCaptures<'a> {
757 fn new(
758 range: Range<usize>,
759 text: &'a Rope,
760 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
761 query: fn(&Grammar) -> Option<&Query>,
762 ) -> Self {
763 let mut result = Self {
764 layers: Vec::new(),
765 grammars: Vec::new(),
766 active_layer_count: 0,
767 };
768 for SyntaxLayerInfo {
769 language,
770 depth,
771 node,
772 } in layers
773 {
774 let grammar = match &language.grammar {
775 Some(grammer) => grammer,
776 None => continue,
777 };
778 let query = match query(&grammar) {
779 Some(query) => query,
780 None => continue,
781 };
782
783 let mut query_cursor = QueryCursorHandle::new();
784
785 // TODO - add a Tree-sitter API to remove the need for this.
786 let cursor = unsafe {
787 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
788 };
789
790 cursor.set_byte_range(range.clone());
791 let captures = cursor.captures(query, node, TextProvider(text));
792 let grammar_index = result
793 .grammars
794 .iter()
795 .position(|g| g.id == grammar.id())
796 .unwrap_or_else(|| {
797 result.grammars.push(grammar);
798 result.grammars.len() - 1
799 });
800 let mut layer = SyntaxMapCapturesLayer {
801 depth,
802 grammar_index,
803 next_capture: None,
804 captures,
805 _query_cursor: query_cursor,
806 };
807
808 layer.advance();
809 if layer.next_capture.is_some() {
810 let key = layer.sort_key();
811 let ix = match result.layers[..result.active_layer_count]
812 .binary_search_by_key(&key, |layer| layer.sort_key())
813 {
814 Ok(ix) | Err(ix) => ix,
815 };
816 result.layers.insert(ix, layer);
817 result.active_layer_count += 1;
818 } else {
819 result.layers.push(layer);
820 }
821 }
822
823 result
824 }
825
826 pub fn grammars(&self) -> &[&'a Grammar] {
827 &self.grammars
828 }
829
830 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
831 let layer = self.layers[..self.active_layer_count].first()?;
832 let capture = layer.next_capture?;
833 Some(SyntaxMapCapture {
834 depth: layer.depth,
835 grammar_index: layer.grammar_index,
836 index: capture.index,
837 node: capture.node,
838 })
839 }
840
841 pub fn advance(&mut self) -> bool {
842 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
843 layer
844 } else {
845 return false;
846 };
847
848 layer.advance();
849 if layer.next_capture.is_some() {
850 let key = layer.sort_key();
851 let i = 1 + self.layers[1..self.active_layer_count]
852 .iter()
853 .position(|later_layer| key < later_layer.sort_key())
854 .unwrap_or(self.active_layer_count - 1);
855 self.layers[0..i].rotate_left(1);
856 } else {
857 self.layers[0..self.active_layer_count].rotate_left(1);
858 self.active_layer_count -= 1;
859 }
860
861 true
862 }
863
864 pub fn set_byte_range(&mut self, range: Range<usize>) {
865 for layer in &mut self.layers {
866 layer.captures.set_byte_range(range.clone());
867 if let Some(capture) = &layer.next_capture {
868 if capture.node.end_byte() > range.start {
869 continue;
870 }
871 }
872 layer.advance();
873 }
874 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
875 self.active_layer_count = self
876 .layers
877 .iter()
878 .position(|layer| layer.next_capture.is_none())
879 .unwrap_or(self.layers.len());
880 }
881}
882
883impl<'a> SyntaxMapMatches<'a> {
884 fn new(
885 range: Range<usize>,
886 text: &'a Rope,
887 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
888 query: fn(&Grammar) -> Option<&Query>,
889 ) -> Self {
890 let mut result = Self::default();
891 for SyntaxLayerInfo {
892 language,
893 depth,
894 node,
895 } in layers
896 {
897 let grammar = match &language.grammar {
898 Some(grammer) => grammer,
899 None => continue,
900 };
901 let query = match query(&grammar) {
902 Some(query) => query,
903 None => continue,
904 };
905
906 let mut query_cursor = QueryCursorHandle::new();
907
908 // TODO - add a Tree-sitter API to remove the need for this.
909 let cursor = unsafe {
910 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
911 };
912
913 cursor.set_byte_range(range.clone());
914 let matches = cursor.matches(query, node, TextProvider(text));
915 let grammar_index = result
916 .grammars
917 .iter()
918 .position(|g| g.id == grammar.id())
919 .unwrap_or_else(|| {
920 result.grammars.push(grammar);
921 result.grammars.len() - 1
922 });
923 let mut layer = SyntaxMapMatchesLayer {
924 depth,
925 grammar_index,
926 matches,
927 next_pattern_index: 0,
928 next_captures: Vec::new(),
929 has_next: false,
930 _query_cursor: query_cursor,
931 };
932
933 layer.advance();
934 if layer.has_next {
935 let key = layer.sort_key();
936 let ix = match result.layers[..result.active_layer_count]
937 .binary_search_by_key(&key, |layer| layer.sort_key())
938 {
939 Ok(ix) | Err(ix) => ix,
940 };
941 result.layers.insert(ix, layer);
942 result.active_layer_count += 1;
943 } else {
944 result.layers.push(layer);
945 }
946 }
947 result
948 }
949
950 pub fn grammars(&self) -> &[&'a Grammar] {
951 &self.grammars
952 }
953
954 pub fn peek(&self) -> Option<SyntaxMapMatch> {
955 let layer = self.layers.first()?;
956 if !layer.has_next {
957 return None;
958 }
959 Some(SyntaxMapMatch {
960 depth: layer.depth,
961 grammar_index: layer.grammar_index,
962 pattern_index: layer.next_pattern_index,
963 captures: &layer.next_captures,
964 })
965 }
966
967 pub fn advance(&mut self) -> bool {
968 let layer = if let Some(layer) = self.layers.first_mut() {
969 layer
970 } else {
971 return false;
972 };
973
974 layer.advance();
975 if layer.has_next {
976 let key = layer.sort_key();
977 let i = 1 + self.layers[1..self.active_layer_count]
978 .iter()
979 .position(|later_layer| key < later_layer.sort_key())
980 .unwrap_or(self.active_layer_count - 1);
981 self.layers[0..i].rotate_left(1);
982 } else {
983 self.layers[0..self.active_layer_count].rotate_left(1);
984 self.active_layer_count -= 1;
985 }
986
987 true
988 }
989}
990
991impl<'a> SyntaxMapCapturesLayer<'a> {
992 fn advance(&mut self) {
993 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
994 }
995
996 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
997 if let Some(capture) = &self.next_capture {
998 let range = capture.node.byte_range();
999 (range.start, Reverse(range.end), self.depth)
1000 } else {
1001 (usize::MAX, Reverse(0), usize::MAX)
1002 }
1003 }
1004}
1005
1006impl<'a> SyntaxMapMatchesLayer<'a> {
1007 fn advance(&mut self) {
1008 if let Some(mat) = self.matches.next() {
1009 self.next_captures.clear();
1010 self.next_captures.extend_from_slice(&mat.captures);
1011 self.next_pattern_index = mat.pattern_index;
1012 self.has_next = true;
1013 } else {
1014 self.has_next = false;
1015 }
1016 }
1017
1018 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
1019 if self.has_next {
1020 let captures = &self.next_captures;
1021 if let Some((first, last)) = captures.first().zip(captures.last()) {
1022 return (
1023 first.node.start_byte(),
1024 Reverse(last.node.end_byte()),
1025 self.depth,
1026 );
1027 }
1028 }
1029 (usize::MAX, Reverse(0), usize::MAX)
1030 }
1031}
1032
1033impl<'a> Iterator for SyntaxMapCaptures<'a> {
1034 type Item = SyntaxMapCapture<'a>;
1035
1036 fn next(&mut self) -> Option<Self::Item> {
1037 let result = self.peek();
1038 self.advance();
1039 result
1040 }
1041}
1042
1043fn join_ranges(
1044 a: impl Iterator<Item = Range<usize>>,
1045 b: impl Iterator<Item = Range<usize>>,
1046) -> Vec<Range<usize>> {
1047 let mut result = Vec::<Range<usize>>::new();
1048 let mut a = a.peekable();
1049 let mut b = b.peekable();
1050 loop {
1051 let range = match (a.peek(), b.peek()) {
1052 (Some(range_a), Some(range_b)) => {
1053 if range_a.start < range_b.start {
1054 a.next().unwrap()
1055 } else {
1056 b.next().unwrap()
1057 }
1058 }
1059 (None, Some(_)) => b.next().unwrap(),
1060 (Some(_), None) => a.next().unwrap(),
1061 (None, None) => break,
1062 };
1063
1064 if let Some(last) = result.last_mut() {
1065 if range.start <= last.end {
1066 last.end = last.end.max(range.end);
1067 continue;
1068 }
1069 }
1070 result.push(range);
1071 }
1072 result
1073}
1074
1075fn parse_text(
1076 grammar: &Grammar,
1077 text: &Rope,
1078 start_byte: usize,
1079 ranges: Vec<tree_sitter::Range>,
1080 old_tree: Option<Tree>,
1081) -> Tree {
1082 PARSER.with(|parser| {
1083 let mut parser = parser.borrow_mut();
1084 let mut chunks = text.chunks_in_range(start_byte..text.len());
1085 parser
1086 .set_included_ranges(&ranges)
1087 .expect("overlapping ranges");
1088 parser
1089 .set_language(grammar.ts_language)
1090 .expect("incompatible grammar");
1091 parser
1092 .parse_with(
1093 &mut move |offset, _| {
1094 chunks.seek(start_byte + offset);
1095 chunks.next().unwrap_or("").as_bytes()
1096 },
1097 old_tree.as_ref(),
1098 )
1099 .expect("invalid language")
1100 })
1101}
1102
1103fn get_injections(
1104 config: &InjectionConfig,
1105 text: &BufferSnapshot,
1106 node: Node,
1107 language_registry: &Arc<LanguageRegistry>,
1108 depth: usize,
1109 changed_ranges: &[Range<usize>],
1110 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
1111 queue: &mut BinaryHeap<ParseStep>,
1112) {
1113 let mut query_cursor = QueryCursorHandle::new();
1114 let mut prev_match = None;
1115
1116 combined_injection_ranges.clear();
1117 for pattern in &config.patterns {
1118 if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
1119 if let Some(language) = language_registry.language_for_name_or_extension(language_name)
1120 {
1121 combined_injection_ranges.insert(language, Vec::new());
1122 }
1123 }
1124 }
1125
1126 for query_range in changed_ranges {
1127 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
1128 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
1129 let content_ranges = mat
1130 .nodes_for_capture_index(config.content_capture_ix)
1131 .map(|node| node.range())
1132 .collect::<Vec<_>>();
1133 if content_ranges.is_empty() {
1134 continue;
1135 }
1136
1137 // Avoid duplicate matches if two changed ranges intersect the same injection.
1138 let content_range =
1139 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
1140 if let Some((last_pattern_ix, last_range)) = &prev_match {
1141 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1142 continue;
1143 }
1144 }
1145 prev_match = Some((mat.pattern_index, content_range.clone()));
1146
1147 let combined = config.patterns[mat.pattern_index].combined;
1148
1149 let mut language_name = None;
1150 let mut step_range = content_range.clone();
1151 if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {
1152 language_name = Some(Cow::Borrowed(name.as_ref()))
1153 } else if let Some(language_node) = config
1154 .language_capture_ix
1155 .and_then(|ix| mat.nodes_for_capture_index(ix).next())
1156 {
1157 step_range.start = cmp::min(content_range.start, language_node.start_byte());
1158 step_range.end = cmp::max(content_range.end, language_node.end_byte());
1159 language_name = Some(Cow::Owned(
1160 text.text_for_range(language_node.byte_range()).collect(),
1161 ))
1162 };
1163
1164 if let Some(language_name) = language_name {
1165 let language = {
1166 let language_name: &str = &language_name;
1167 language_registry.language_for_name_or_extension(language_name)
1168 };
1169 let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
1170 if let Some(language) = language {
1171 if combined {
1172 combined_injection_ranges
1173 .get_mut(&language.clone())
1174 .unwrap()
1175 .extend(content_ranges);
1176 } else {
1177 queue.push(ParseStep {
1178 depth,
1179 language: ParseStepLanguage::Loaded { language },
1180 included_ranges: content_ranges,
1181 range,
1182 mode: ParseMode::Single,
1183 });
1184 }
1185 } else {
1186 queue.push(ParseStep {
1187 depth,
1188 language: ParseStepLanguage::Pending {
1189 name: language_name.into(),
1190 },
1191 included_ranges: content_ranges,
1192 range,
1193 mode: ParseMode::Single,
1194 });
1195 }
1196 }
1197 }
1198 }
1199
1200 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1201 included_ranges.sort_unstable();
1202 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1203 queue.push(ParseStep {
1204 depth,
1205 language: ParseStepLanguage::Loaded { language },
1206 range,
1207 included_ranges,
1208 mode: ParseMode::Combined {
1209 parent_layer_range: node.start_byte()..node.end_byte(),
1210 parent_layer_changed_ranges: changed_ranges.to_vec(),
1211 },
1212 })
1213 }
1214}
1215
1216fn splice_included_ranges(
1217 mut ranges: Vec<tree_sitter::Range>,
1218 changed_ranges: &[Range<usize>],
1219 new_ranges: &[tree_sitter::Range],
1220) -> Vec<tree_sitter::Range> {
1221 let mut changed_ranges = changed_ranges.into_iter().peekable();
1222 let mut new_ranges = new_ranges.into_iter().peekable();
1223 let mut ranges_ix = 0;
1224 loop {
1225 let new_range = new_ranges.peek();
1226 let mut changed_range = changed_ranges.peek();
1227
1228 // Remove ranges that have changed before inserting any new ranges
1229 // into those ranges.
1230 if let Some((changed, new)) = changed_range.zip(new_range) {
1231 if new.end_byte < changed.start {
1232 changed_range = None;
1233 }
1234 }
1235
1236 if let Some(changed) = changed_range {
1237 let mut start_ix = ranges_ix
1238 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1239 Ok(ix) | Err(ix) => ix,
1240 };
1241 let mut end_ix = ranges_ix
1242 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1243 Ok(ix) => ix + 1,
1244 Err(ix) => ix,
1245 };
1246
1247 // If there are empty ranges, then there may be multiple ranges with the same
1248 // start or end. Expand the splice to include any adjacent ranges that touch
1249 // the changed range.
1250 while start_ix > 0 {
1251 if ranges[start_ix - 1].end_byte == changed.start {
1252 start_ix -= 1;
1253 } else {
1254 break;
1255 }
1256 }
1257 while let Some(range) = ranges.get(end_ix) {
1258 if range.start_byte == changed.end {
1259 end_ix += 1;
1260 } else {
1261 break;
1262 }
1263 }
1264
1265 if end_ix > start_ix {
1266 ranges.splice(start_ix..end_ix, []);
1267 }
1268 changed_ranges.next();
1269 ranges_ix = start_ix;
1270 } else if let Some(new_range) = new_range {
1271 let ix = ranges_ix
1272 + match ranges[ranges_ix..]
1273 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1274 {
1275 Ok(ix) | Err(ix) => ix,
1276 };
1277 ranges.insert(ix, **new_range);
1278 new_ranges.next();
1279 ranges_ix = ix + 1;
1280 } else {
1281 break;
1282 }
1283 }
1284 ranges
1285}
1286
1287impl<'a> SyntaxLayerInfo<'a> {
1288 pub(crate) fn override_id(&self, offset: usize, text: &text::BufferSnapshot) -> Option<u32> {
1289 let text = TextProvider(text.as_rope());
1290 let config = self.language.grammar.as_ref()?.override_config.as_ref()?;
1291
1292 let mut query_cursor = QueryCursorHandle::new();
1293 query_cursor.set_byte_range(offset..offset);
1294
1295 let mut smallest_match: Option<(u32, Range<usize>)> = None;
1296 for mat in query_cursor.matches(&config.query, self.node, text) {
1297 for capture in mat.captures {
1298 if !config.values.contains_key(&capture.index) {
1299 continue;
1300 }
1301
1302 let range = capture.node.byte_range();
1303 if offset <= range.start || offset >= range.end {
1304 continue;
1305 }
1306
1307 if let Some((_, smallest_range)) = &smallest_match {
1308 if range.len() < smallest_range.len() {
1309 smallest_match = Some((capture.index, range))
1310 }
1311 continue;
1312 }
1313
1314 smallest_match = Some((capture.index, range));
1315 }
1316 }
1317
1318 smallest_match.map(|(index, _)| index)
1319 }
1320}
1321
1322impl std::ops::Deref for SyntaxMap {
1323 type Target = SyntaxSnapshot;
1324
1325 fn deref(&self) -> &Self::Target {
1326 &self.snapshot
1327 }
1328}
1329
1330impl PartialEq for ParseStep {
1331 fn eq(&self, _: &Self) -> bool {
1332 false
1333 }
1334}
1335
1336impl Eq for ParseStep {}
1337
1338impl PartialOrd for ParseStep {
1339 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1340 Some(self.cmp(&other))
1341 }
1342}
1343
1344impl Ord for ParseStep {
1345 fn cmp(&self, other: &Self) -> Ordering {
1346 let range_a = self.range();
1347 let range_b = other.range();
1348 Ord::cmp(&other.depth, &self.depth)
1349 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1350 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1351 .then_with(|| self.language.id().cmp(&other.language.id()))
1352 }
1353}
1354
1355impl ParseStep {
1356 fn range(&self) -> Range<usize> {
1357 if let ParseMode::Combined {
1358 parent_layer_range, ..
1359 } = &self.mode
1360 {
1361 parent_layer_range.clone()
1362 } else {
1363 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1364 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1365 start..end
1366 }
1367 }
1368}
1369
1370impl ChangedRegion {
1371 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1372 let range_a = &self.range;
1373 let range_b = &other.range;
1374 Ord::cmp(&self.depth, &other.depth)
1375 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1376 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1377 }
1378}
1379
1380impl ChangeRegionSet {
1381 fn start_position(&self) -> ChangeStartPosition {
1382 self.0.first().map_or(
1383 ChangeStartPosition {
1384 depth: usize::MAX,
1385 position: Anchor::MAX,
1386 },
1387 |region| ChangeStartPosition {
1388 depth: region.depth,
1389 position: region.range.start,
1390 },
1391 )
1392 }
1393
1394 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1395 for region in &self.0 {
1396 if region.depth < layer.depth {
1397 continue;
1398 }
1399 if region.depth > layer.depth {
1400 break;
1401 }
1402 if region.range.end.cmp(&layer.range.start, text).is_le() {
1403 continue;
1404 }
1405 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1406 break;
1407 }
1408 return true;
1409 }
1410 false
1411 }
1412
1413 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1414 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1415 self.0.insert(ix, region);
1416 }
1417 }
1418
1419 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1420 let prev_len = self.0.len();
1421 self.0.retain(|region| {
1422 region.depth > summary.max_depth
1423 || (region.depth == summary.max_depth
1424 && region
1425 .range
1426 .end
1427 .cmp(&summary.last_layer_range.start, text)
1428 .is_gt())
1429 });
1430 self.0.len() < prev_len
1431 }
1432}
1433
1434impl Default for SyntaxLayerSummary {
1435 fn default() -> Self {
1436 Self {
1437 max_depth: 0,
1438 min_depth: 0,
1439 range: Anchor::MAX..Anchor::MIN,
1440 last_layer_range: Anchor::MIN..Anchor::MAX,
1441 last_layer_language: None,
1442 contains_unknown_injections: false,
1443 }
1444 }
1445}
1446
1447impl sum_tree::Summary for SyntaxLayerSummary {
1448 type Context = BufferSnapshot;
1449
1450 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1451 if other.max_depth > self.max_depth {
1452 self.max_depth = other.max_depth;
1453 self.range = other.range.clone();
1454 } else {
1455 if self.range == (Anchor::MAX..Anchor::MAX) {
1456 self.range.start = other.range.start;
1457 }
1458 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1459 self.range.end = other.range.end;
1460 }
1461 }
1462 self.last_layer_range = other.last_layer_range.clone();
1463 self.last_layer_language = other.last_layer_language;
1464 self.contains_unknown_injections |= other.contains_unknown_injections;
1465 }
1466}
1467
1468impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1469 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1470 Ord::cmp(&self.depth, &cursor_location.max_depth)
1471 .then_with(|| {
1472 self.range
1473 .start
1474 .cmp(&cursor_location.last_layer_range.start, buffer)
1475 })
1476 .then_with(|| {
1477 cursor_location
1478 .last_layer_range
1479 .end
1480 .cmp(&self.range.end, buffer)
1481 })
1482 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1483 }
1484}
1485
1486impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for ChangeStartPosition {
1487 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1488 Ord::cmp(&self.depth, &cursor_location.max_depth)
1489 .then_with(|| self.position.cmp(&cursor_location.range.end, text))
1490 }
1491}
1492
1493impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1494 for SyntaxLayerPositionBeforeChange
1495{
1496 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1497 if self.change.cmp(cursor_location, buffer).is_le() {
1498 return Ordering::Less;
1499 } else {
1500 self.position.cmp(cursor_location, buffer)
1501 }
1502 }
1503}
1504
1505impl sum_tree::Item for SyntaxLayer {
1506 type Summary = SyntaxLayerSummary;
1507
1508 fn summary(&self) -> Self::Summary {
1509 SyntaxLayerSummary {
1510 min_depth: self.depth,
1511 max_depth: self.depth,
1512 range: self.range.clone(),
1513 last_layer_range: self.range.clone(),
1514 last_layer_language: self.content.language_id(),
1515 contains_unknown_injections: matches!(self.content, SyntaxLayerContent::Pending { .. }),
1516 }
1517 }
1518}
1519
1520impl std::fmt::Debug for SyntaxLayer {
1521 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1522 f.debug_struct("SyntaxLayer")
1523 .field("depth", &self.depth)
1524 .field("range", &self.range)
1525 .field("tree", &self.content.tree())
1526 .finish()
1527 }
1528}
1529
1530impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1531 type I = ByteChunks<'a>;
1532
1533 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1534 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1535 }
1536}
1537
1538impl<'a> Iterator for ByteChunks<'a> {
1539 type Item = &'a [u8];
1540
1541 fn next(&mut self) -> Option<Self::Item> {
1542 self.0.next().map(str::as_bytes)
1543 }
1544}
1545
1546impl QueryCursorHandle {
1547 pub(crate) fn new() -> Self {
1548 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1549 cursor.set_match_limit(64);
1550 QueryCursorHandle(Some(cursor))
1551 }
1552}
1553
1554impl Deref for QueryCursorHandle {
1555 type Target = QueryCursor;
1556
1557 fn deref(&self) -> &Self::Target {
1558 self.0.as_ref().unwrap()
1559 }
1560}
1561
1562impl DerefMut for QueryCursorHandle {
1563 fn deref_mut(&mut self) -> &mut Self::Target {
1564 self.0.as_mut().unwrap()
1565 }
1566}
1567
1568impl Drop for QueryCursorHandle {
1569 fn drop(&mut self) {
1570 let mut cursor = self.0.take().unwrap();
1571 cursor.set_byte_range(0..usize::MAX);
1572 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1573 QUERY_CURSORS.lock().push(cursor)
1574 }
1575}
1576
1577pub(crate) trait ToTreeSitterPoint {
1578 fn to_ts_point(self) -> tree_sitter::Point;
1579 fn from_ts_point(point: tree_sitter::Point) -> Self;
1580}
1581
1582impl ToTreeSitterPoint for Point {
1583 fn to_ts_point(self) -> tree_sitter::Point {
1584 tree_sitter::Point::new(self.row as usize, self.column as usize)
1585 }
1586
1587 fn from_ts_point(point: tree_sitter::Point) -> Self {
1588 Point::new(point.row as u32, point.column as u32)
1589 }
1590}
1591
1592#[cfg(test)]
1593mod tests {
1594 use super::*;
1595 use crate::LanguageConfig;
1596 use rand::rngs::StdRng;
1597 use std::env;
1598 use text::Buffer;
1599 use unindent::Unindent as _;
1600 use util::test::marked_text_ranges;
1601
1602 #[test]
1603 fn test_splice_included_ranges() {
1604 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1605
1606 let new_ranges = splice_included_ranges(
1607 ranges.clone(),
1608 &[54..56, 58..68],
1609 &[ts_range(50..54), ts_range(59..67)],
1610 );
1611 assert_eq!(
1612 new_ranges,
1613 &[
1614 ts_range(20..30),
1615 ts_range(50..54),
1616 ts_range(59..67),
1617 ts_range(80..90),
1618 ]
1619 );
1620
1621 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1622 assert_eq!(
1623 new_ranges,
1624 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1625 );
1626
1627 let new_ranges =
1628 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1629 assert_eq!(
1630 new_ranges,
1631 &[
1632 ts_range(0..2),
1633 ts_range(20..30),
1634 ts_range(50..60),
1635 ts_range(70..75),
1636 ts_range(80..90)
1637 ]
1638 );
1639
1640 let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
1641 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
1642
1643 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1644 tree_sitter::Range {
1645 start_byte: range.start,
1646 start_point: tree_sitter::Point {
1647 row: 0,
1648 column: range.start,
1649 },
1650 end_byte: range.end,
1651 end_point: tree_sitter::Point {
1652 row: 0,
1653 column: range.end,
1654 },
1655 }
1656 }
1657 }
1658
1659 #[gpui::test]
1660 fn test_syntax_map_layers_for_range() {
1661 let registry = Arc::new(LanguageRegistry::test());
1662 let language = Arc::new(rust_lang());
1663 registry.add(language.clone());
1664
1665 let mut buffer = Buffer::new(
1666 0,
1667 0,
1668 r#"
1669 fn a() {
1670 assert_eq!(
1671 b(vec![C {}]),
1672 vec![d.e],
1673 );
1674 println!("{}", f(|_| true));
1675 }
1676 "#
1677 .unindent(),
1678 );
1679
1680 let mut syntax_map = SyntaxMap::new();
1681 syntax_map.set_language_registry(registry.clone());
1682 syntax_map.reparse(language.clone(), &buffer);
1683
1684 assert_layers_for_range(
1685 &syntax_map,
1686 &buffer,
1687 Point::new(2, 0)..Point::new(2, 0),
1688 &[
1689 "...(function_item ... (block (expression_statement (macro_invocation...",
1690 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1691 ],
1692 );
1693 assert_layers_for_range(
1694 &syntax_map,
1695 &buffer,
1696 Point::new(2, 14)..Point::new(2, 16),
1697 &[
1698 "...(function_item ...",
1699 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1700 "...(array_expression (struct_expression ...",
1701 ],
1702 );
1703 assert_layers_for_range(
1704 &syntax_map,
1705 &buffer,
1706 Point::new(3, 14)..Point::new(3, 16),
1707 &[
1708 "...(function_item ...",
1709 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1710 "...(array_expression (field_expression ...",
1711 ],
1712 );
1713 assert_layers_for_range(
1714 &syntax_map,
1715 &buffer,
1716 Point::new(5, 12)..Point::new(5, 16),
1717 &[
1718 "...(function_item ...",
1719 "...(call_expression ... (arguments (closure_expression ...",
1720 ],
1721 );
1722
1723 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1724 let macro_name_range = range_for_text(&buffer, "vec!");
1725 buffer.edit([(macro_name_range, "&")]);
1726 syntax_map.interpolate(&buffer);
1727 syntax_map.reparse(language.clone(), &buffer);
1728
1729 assert_layers_for_range(
1730 &syntax_map,
1731 &buffer,
1732 Point::new(2, 14)..Point::new(2, 16),
1733 &[
1734 "...(function_item ...",
1735 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1736 ],
1737 );
1738
1739 // Put the vec! macro back, adding back the syntactic layer.
1740 buffer.undo();
1741 syntax_map.interpolate(&buffer);
1742 syntax_map.reparse(language.clone(), &buffer);
1743
1744 assert_layers_for_range(
1745 &syntax_map,
1746 &buffer,
1747 Point::new(2, 14)..Point::new(2, 16),
1748 &[
1749 "...(function_item ...",
1750 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1751 "...(array_expression (struct_expression ...",
1752 ],
1753 );
1754 }
1755
1756 #[gpui::test]
1757 fn test_dynamic_language_injection() {
1758 let registry = Arc::new(LanguageRegistry::test());
1759 let markdown = Arc::new(markdown_lang());
1760 registry.add(markdown.clone());
1761 registry.add(Arc::new(rust_lang()));
1762 registry.add(Arc::new(ruby_lang()));
1763
1764 let mut buffer = Buffer::new(
1765 0,
1766 0,
1767 r#"
1768 This is a code block:
1769
1770 ```rs
1771 fn foo() {}
1772 ```
1773 "#
1774 .unindent(),
1775 );
1776
1777 let mut syntax_map = SyntaxMap::new();
1778 syntax_map.set_language_registry(registry.clone());
1779 syntax_map.reparse(markdown.clone(), &buffer);
1780 assert_layers_for_range(
1781 &syntax_map,
1782 &buffer,
1783 Point::new(3, 0)..Point::new(3, 0),
1784 &[
1785 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1786 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
1787 ],
1788 );
1789
1790 // Replace Rust with Ruby in code block.
1791 let macro_name_range = range_for_text(&buffer, "rs");
1792 buffer.edit([(macro_name_range, "ruby")]);
1793 syntax_map.interpolate(&buffer);
1794 syntax_map.reparse(markdown.clone(), &buffer);
1795 assert_layers_for_range(
1796 &syntax_map,
1797 &buffer,
1798 Point::new(3, 0)..Point::new(3, 0),
1799 &[
1800 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1801 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
1802 ],
1803 );
1804
1805 // Replace Ruby with a language that hasn't been loaded yet.
1806 let macro_name_range = range_for_text(&buffer, "ruby");
1807 buffer.edit([(macro_name_range, "html")]);
1808 syntax_map.interpolate(&buffer);
1809 syntax_map.reparse(markdown.clone(), &buffer);
1810 assert_layers_for_range(
1811 &syntax_map,
1812 &buffer,
1813 Point::new(3, 0)..Point::new(3, 0),
1814 &[
1815 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
1816 ],
1817 );
1818 assert!(syntax_map.contains_unknown_injections());
1819
1820 registry.add(Arc::new(html_lang()));
1821 syntax_map.reparse(markdown.clone(), &buffer);
1822 assert_layers_for_range(
1823 &syntax_map,
1824 &buffer,
1825 Point::new(3, 0)..Point::new(3, 0),
1826 &[
1827 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1828 "(fragment (text))",
1829 ],
1830 );
1831 assert!(!syntax_map.contains_unknown_injections());
1832 }
1833
1834 #[gpui::test]
1835 fn test_typing_multiple_new_injections() {
1836 let (buffer, syntax_map) = test_edit_sequence(
1837 "Rust",
1838 &[
1839 "fn a() { dbg }",
1840 "fn a() { dbg«!» }",
1841 "fn a() { dbg!«()» }",
1842 "fn a() { dbg!(«b») }",
1843 "fn a() { dbg!(b«.») }",
1844 "fn a() { dbg!(b.«c») }",
1845 "fn a() { dbg!(b.c«()») }",
1846 "fn a() { dbg!(b.c(«vec»)) }",
1847 "fn a() { dbg!(b.c(vec«!»)) }",
1848 "fn a() { dbg!(b.c(vec!«[]»)) }",
1849 "fn a() { dbg!(b.c(vec![«d»])) }",
1850 "fn a() { dbg!(b.c(vec![d«.»])) }",
1851 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1852 ],
1853 );
1854
1855 assert_capture_ranges(
1856 &syntax_map,
1857 &buffer,
1858 &["field"],
1859 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1860 );
1861 }
1862
1863 #[gpui::test]
1864 fn test_pasting_new_injection_line_between_others() {
1865 let (buffer, syntax_map) = test_edit_sequence(
1866 "Rust",
1867 &[
1868 "
1869 fn a() {
1870 b!(B {});
1871 c!(C {});
1872 d!(D {});
1873 e!(E {});
1874 f!(F {});
1875 g!(G {});
1876 }
1877 ",
1878 "
1879 fn a() {
1880 b!(B {});
1881 c!(C {});
1882 d!(D {});
1883 « h!(H {});
1884 » e!(E {});
1885 f!(F {});
1886 g!(G {});
1887 }
1888 ",
1889 ],
1890 );
1891
1892 assert_capture_ranges(
1893 &syntax_map,
1894 &buffer,
1895 &["struct"],
1896 "
1897 fn a() {
1898 b!(«B {}»);
1899 c!(«C {}»);
1900 d!(«D {}»);
1901 h!(«H {}»);
1902 e!(«E {}»);
1903 f!(«F {}»);
1904 g!(«G {}»);
1905 }
1906 ",
1907 );
1908 }
1909
1910 #[gpui::test]
1911 fn test_joining_injections_with_child_injections() {
1912 let (buffer, syntax_map) = test_edit_sequence(
1913 "Rust",
1914 &[
1915 "
1916 fn a() {
1917 b!(
1918 c![one.two.three],
1919 d![four.five.six],
1920 );
1921 e!(
1922 f![seven.eight],
1923 );
1924 }
1925 ",
1926 "
1927 fn a() {
1928 b!(
1929 c![one.two.three],
1930 d![four.five.six],
1931 ˇ f![seven.eight],
1932 );
1933 }
1934 ",
1935 ],
1936 );
1937
1938 assert_capture_ranges(
1939 &syntax_map,
1940 &buffer,
1941 &["field"],
1942 "
1943 fn a() {
1944 b!(
1945 c![one.«two».«three»],
1946 d![four.«five».«six»],
1947 f![seven.«eight»],
1948 );
1949 }
1950 ",
1951 );
1952 }
1953
1954 #[gpui::test]
1955 fn test_editing_edges_of_injection() {
1956 test_edit_sequence(
1957 "Rust",
1958 &[
1959 "
1960 fn a() {
1961 b!(c!())
1962 }
1963 ",
1964 "
1965 fn a() {
1966 «d»!(c!())
1967 }
1968 ",
1969 "
1970 fn a() {
1971 «e»d!(c!())
1972 }
1973 ",
1974 "
1975 fn a() {
1976 ed!«[»c!()«]»
1977 }
1978 ",
1979 ],
1980 );
1981 }
1982
1983 #[gpui::test]
1984 fn test_edits_preceding_and_intersecting_injection() {
1985 test_edit_sequence(
1986 "Rust",
1987 &[
1988 //
1989 "const aaaaaaaaaaaa: B = c!(d(e.f));",
1990 "const aˇa: B = c!(d(eˇ));",
1991 ],
1992 );
1993 }
1994
1995 #[gpui::test]
1996 fn test_non_local_changes_create_injections() {
1997 test_edit_sequence(
1998 "Rust",
1999 &[
2000 "
2001 // a! {
2002 static B: C = d;
2003 // }
2004 ",
2005 "
2006 ˇa! {
2007 static B: C = d;
2008 ˇ}
2009 ",
2010 ],
2011 );
2012 }
2013
2014 #[gpui::test]
2015 fn test_creating_many_injections_in_one_edit() {
2016 test_edit_sequence(
2017 "Rust",
2018 &[
2019 "
2020 fn a() {
2021 one(Two::three(3));
2022 four(Five::six(6));
2023 seven(Eight::nine(9));
2024 }
2025 ",
2026 "
2027 fn a() {
2028 one«!»(Two::three(3));
2029 four«!»(Five::six(6));
2030 seven«!»(Eight::nine(9));
2031 }
2032 ",
2033 "
2034 fn a() {
2035 one!(Two::three«!»(3));
2036 four!(Five::six«!»(6));
2037 seven!(Eight::nine«!»(9));
2038 }
2039 ",
2040 ],
2041 );
2042 }
2043
2044 #[gpui::test]
2045 fn test_editing_across_injection_boundary() {
2046 test_edit_sequence(
2047 "Rust",
2048 &[
2049 "
2050 fn one() {
2051 two();
2052 three!(
2053 three.four,
2054 five.six,
2055 );
2056 }
2057 ",
2058 "
2059 fn one() {
2060 two();
2061 th«irty_five![»
2062 three.four,
2063 five.six,
2064 « seven.eight,
2065 ];»
2066 }
2067 ",
2068 ],
2069 );
2070 }
2071
2072 #[gpui::test]
2073 fn test_removing_injection_by_replacing_across_boundary() {
2074 test_edit_sequence(
2075 "Rust",
2076 &[
2077 "
2078 fn one() {
2079 two!(
2080 three.four,
2081 );
2082 }
2083 ",
2084 "
2085 fn one() {
2086 t«en
2087 .eleven(
2088 twelve,
2089 »
2090 three.four,
2091 );
2092 }
2093 ",
2094 ],
2095 );
2096 }
2097
2098 #[gpui::test]
2099 fn test_combined_injections() {
2100 let (buffer, syntax_map) = test_edit_sequence(
2101 "ERB",
2102 &[
2103 "
2104 <body>
2105 <% if @one %>
2106 <div class=one>
2107 <% else %>
2108 <div class=two>
2109 <% end %>
2110 </div>
2111 </body>
2112 ",
2113 "
2114 <body>
2115 <% if @one %>
2116 <div class=one>
2117 ˇ else ˇ
2118 <div class=two>
2119 <% end %>
2120 </div>
2121 </body>
2122 ",
2123 "
2124 <body>
2125 <% if @one «;» end %>
2126 </div>
2127 </body>
2128 ",
2129 ],
2130 );
2131
2132 assert_capture_ranges(
2133 &syntax_map,
2134 &buffer,
2135 &["tag", "ivar"],
2136 "
2137 <«body»>
2138 <% if «@one» ; end %>
2139 </«div»>
2140 </«body»>
2141 ",
2142 );
2143 }
2144
2145 #[gpui::test]
2146 fn test_combined_injections_empty_ranges() {
2147 test_edit_sequence(
2148 "ERB",
2149 &[
2150 "
2151 <% if @one %>
2152 <% else %>
2153 <% end %>
2154 ",
2155 "
2156 <% if @one %>
2157 ˇ<% end %>
2158 ",
2159 ],
2160 );
2161 }
2162
2163 #[gpui::test]
2164 fn test_combined_injections_edit_edges_of_ranges() {
2165 let (buffer, syntax_map) = test_edit_sequence(
2166 "ERB",
2167 &[
2168 "
2169 <%= one @two %>
2170 <%= three @four %>
2171 ",
2172 "
2173 <%= one @two %ˇ
2174 <%= three @four %>
2175 ",
2176 "
2177 <%= one @two %«>»
2178 <%= three @four %>
2179 ",
2180 ],
2181 );
2182
2183 assert_capture_ranges(
2184 &syntax_map,
2185 &buffer,
2186 &["tag", "ivar"],
2187 "
2188 <%= one «@two» %>
2189 <%= three «@four» %>
2190 ",
2191 );
2192 }
2193
2194 #[gpui::test]
2195 fn test_combined_injections_splitting_some_injections() {
2196 let (_buffer, _syntax_map) = test_edit_sequence(
2197 "ERB",
2198 &[
2199 r#"
2200 <%A if b(:c) %>
2201 d
2202 <% end %>
2203 eee
2204 <% f %>
2205 "#,
2206 r#"
2207 <%« AAAAAAA %>
2208 hhhhhhh
2209 <%=» if b(:c) %>
2210 d
2211 <% end %>
2212 eee
2213 <% f %>
2214 "#,
2215 ],
2216 );
2217 }
2218
2219 #[gpui::test]
2220 fn test_combined_injections_inside_injections() {
2221 let (_buffer, _syntax_map) = test_edit_sequence(
2222 "Markdown",
2223 &[
2224 r#"
2225 here is some ERB code:
2226
2227 ```erb
2228 <ul>
2229 <% people.each do |person| %>
2230 <li><%= person.name %></li>
2231 <% end %>
2232 </ul>
2233 ```
2234 "#,
2235 r#"
2236 here is some ERB code:
2237
2238 ```erb
2239 <ul>
2240 <% people«2».each do |person| %>
2241 <li><%= person.name %></li>
2242 <% end %>
2243 </ul>
2244 ```
2245 "#,
2246 ],
2247 );
2248 }
2249
2250 #[gpui::test(iterations = 50)]
2251 fn test_random_syntax_map_edits(mut rng: StdRng) {
2252 let operations = env::var("OPERATIONS")
2253 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2254 .unwrap_or(10);
2255
2256 let text = r#"
2257 fn test_something() {
2258 let vec = vec![5, 1, 3, 8];
2259 assert_eq!(
2260 vec
2261 .into_iter()
2262 .map(|i| i * 2)
2263 .collect::<Vec<usize>>(),
2264 vec![
2265 5 * 2, 1 * 2, 3 * 2, 8 * 2
2266 ],
2267 );
2268 }
2269 "#
2270 .unindent()
2271 .repeat(2);
2272
2273 let registry = Arc::new(LanguageRegistry::test());
2274 let language = Arc::new(rust_lang());
2275 registry.add(language.clone());
2276 let mut buffer = Buffer::new(0, 0, text);
2277
2278 let mut syntax_map = SyntaxMap::new();
2279 syntax_map.set_language_registry(registry.clone());
2280 syntax_map.reparse(language.clone(), &buffer);
2281
2282 let mut reference_syntax_map = SyntaxMap::new();
2283 reference_syntax_map.set_language_registry(registry.clone());
2284
2285 log::info!("initial text:\n{}", buffer.text());
2286
2287 for _ in 0..operations {
2288 let prev_buffer = buffer.snapshot();
2289 let prev_syntax_map = syntax_map.snapshot();
2290
2291 buffer.randomly_edit(&mut rng, 3);
2292 log::info!("text:\n{}", buffer.text());
2293
2294 syntax_map.interpolate(&buffer);
2295 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2296
2297 syntax_map.reparse(language.clone(), &buffer);
2298
2299 reference_syntax_map.clear();
2300 reference_syntax_map.reparse(language.clone(), &buffer);
2301 }
2302
2303 for i in 0..operations {
2304 let i = operations - i - 1;
2305 buffer.undo();
2306 log::info!("undoing operation {}", i);
2307 log::info!("text:\n{}", buffer.text());
2308
2309 syntax_map.interpolate(&buffer);
2310 syntax_map.reparse(language.clone(), &buffer);
2311
2312 reference_syntax_map.clear();
2313 reference_syntax_map.reparse(language.clone(), &buffer);
2314 assert_eq!(
2315 syntax_map.layers(&buffer).len(),
2316 reference_syntax_map.layers(&buffer).len(),
2317 "wrong number of layers after undoing edit {i}"
2318 );
2319 }
2320
2321 let layers = syntax_map.layers(&buffer);
2322 let reference_layers = reference_syntax_map.layers(&buffer);
2323 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2324 {
2325 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2326 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2327 }
2328 }
2329
2330 #[gpui::test(iterations = 50)]
2331 fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) {
2332 let operations = env::var("OPERATIONS")
2333 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2334 .unwrap_or(10);
2335
2336 let text = r#"
2337 <div id="main">
2338 <% if one?(:two) %>
2339 <p class="three" four>
2340 <%= yield :five %>
2341 </p>
2342 <% elsif Six.seven(8) %>
2343 <p id="three" four>
2344 <%= yield :five %>
2345 </p>
2346 <% else %>
2347 <span>Ok</span>
2348 <% end %>
2349 </div>
2350 "#
2351 .unindent()
2352 .repeat(8);
2353
2354 let registry = Arc::new(LanguageRegistry::test());
2355 let language = Arc::new(erb_lang());
2356 registry.add(language.clone());
2357 registry.add(Arc::new(ruby_lang()));
2358 registry.add(Arc::new(html_lang()));
2359 let mut buffer = Buffer::new(0, 0, text);
2360
2361 let mut syntax_map = SyntaxMap::new();
2362 syntax_map.set_language_registry(registry.clone());
2363 syntax_map.reparse(language.clone(), &buffer);
2364
2365 let mut reference_syntax_map = SyntaxMap::new();
2366 reference_syntax_map.set_language_registry(registry.clone());
2367
2368 log::info!("initial text:\n{}", buffer.text());
2369
2370 for _ in 0..operations {
2371 let prev_buffer = buffer.snapshot();
2372 let prev_syntax_map = syntax_map.snapshot();
2373
2374 buffer.randomly_edit(&mut rng, 3);
2375 log::info!("text:\n{}", buffer.text());
2376
2377 syntax_map.interpolate(&buffer);
2378 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2379
2380 syntax_map.reparse(language.clone(), &buffer);
2381
2382 reference_syntax_map.clear();
2383 reference_syntax_map.reparse(language.clone(), &buffer);
2384 }
2385
2386 for i in 0..operations {
2387 let i = operations - i - 1;
2388 buffer.undo();
2389 log::info!("undoing operation {}", i);
2390 log::info!("text:\n{}", buffer.text());
2391
2392 syntax_map.interpolate(&buffer);
2393 syntax_map.reparse(language.clone(), &buffer);
2394
2395 reference_syntax_map.clear();
2396 reference_syntax_map.reparse(language.clone(), &buffer);
2397 assert_eq!(
2398 syntax_map.layers(&buffer).len(),
2399 reference_syntax_map.layers(&buffer).len(),
2400 "wrong number of layers after undoing edit {i}"
2401 );
2402 }
2403
2404 let layers = syntax_map.layers(&buffer);
2405 let reference_layers = reference_syntax_map.layers(&buffer);
2406 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2407 {
2408 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2409 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2410 }
2411 }
2412
2413 fn check_interpolation(
2414 old_syntax_map: &SyntaxSnapshot,
2415 new_syntax_map: &SyntaxSnapshot,
2416 old_buffer: &BufferSnapshot,
2417 new_buffer: &BufferSnapshot,
2418 ) {
2419 let edits = new_buffer
2420 .edits_since::<usize>(&old_buffer.version())
2421 .collect::<Vec<_>>();
2422
2423 for (old_layer, new_layer) in old_syntax_map
2424 .layers
2425 .iter()
2426 .zip(new_syntax_map.layers.iter())
2427 {
2428 assert_eq!(old_layer.range, new_layer.range);
2429 let Some(old_tree) = old_layer.content.tree() else { continue };
2430 let Some(new_tree) = new_layer.content.tree() else { continue };
2431 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
2432 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
2433 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
2434 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
2435 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
2436 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
2437 check_node_edits(
2438 old_layer.depth,
2439 &old_layer.range,
2440 old_node,
2441 new_node,
2442 old_buffer,
2443 new_buffer,
2444 &edits,
2445 );
2446 }
2447
2448 fn check_node_edits(
2449 depth: usize,
2450 range: &Range<Anchor>,
2451 old_node: Node,
2452 new_node: Node,
2453 old_buffer: &BufferSnapshot,
2454 new_buffer: &BufferSnapshot,
2455 edits: &[text::Edit<usize>],
2456 ) {
2457 assert_eq!(old_node.kind(), new_node.kind());
2458
2459 let old_range = old_node.byte_range();
2460 let new_range = new_node.byte_range();
2461
2462 let is_edited = edits
2463 .iter()
2464 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
2465 if is_edited {
2466 assert!(
2467 new_node.has_changes(),
2468 concat!(
2469 "failed to mark node as edited.\n",
2470 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2471 "node kind: {}, old node range: {:?}, new node range: {:?}",
2472 ),
2473 depth,
2474 range.to_offset(old_buffer),
2475 range.to_offset(new_buffer),
2476 new_node.kind(),
2477 old_range,
2478 new_range,
2479 );
2480 }
2481
2482 if !new_node.has_changes() {
2483 assert_eq!(
2484 old_buffer
2485 .text_for_range(old_range.clone())
2486 .collect::<String>(),
2487 new_buffer
2488 .text_for_range(new_range.clone())
2489 .collect::<String>(),
2490 concat!(
2491 "mismatched text for node\n",
2492 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2493 "node kind: {}, old node range:{:?}, new node range:{:?}",
2494 ),
2495 depth,
2496 range.to_offset(old_buffer),
2497 range.to_offset(new_buffer),
2498 new_node.kind(),
2499 old_range,
2500 new_range,
2501 );
2502 }
2503
2504 for i in 0..new_node.child_count() {
2505 check_node_edits(
2506 depth,
2507 range,
2508 old_node.child(i).unwrap(),
2509 new_node.child(i).unwrap(),
2510 old_buffer,
2511 new_buffer,
2512 edits,
2513 )
2514 }
2515 }
2516 }
2517
2518 fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
2519 let registry = Arc::new(LanguageRegistry::test());
2520 registry.add(Arc::new(rust_lang()));
2521 registry.add(Arc::new(ruby_lang()));
2522 registry.add(Arc::new(html_lang()));
2523 registry.add(Arc::new(erb_lang()));
2524 registry.add(Arc::new(markdown_lang()));
2525 let language = registry.language_for_name(language_name).unwrap();
2526 let mut buffer = Buffer::new(0, 0, Default::default());
2527
2528 let mut mutated_syntax_map = SyntaxMap::new();
2529 mutated_syntax_map.set_language_registry(registry.clone());
2530 mutated_syntax_map.reparse(language.clone(), &buffer);
2531
2532 for (i, marked_string) in steps.into_iter().enumerate() {
2533 buffer.edit_via_marked_text(&marked_string.unindent());
2534
2535 // Reparse the syntax map
2536 mutated_syntax_map.interpolate(&buffer);
2537 mutated_syntax_map.reparse(language.clone(), &buffer);
2538
2539 // Create a second syntax map from scratch
2540 let mut reference_syntax_map = SyntaxMap::new();
2541 reference_syntax_map.set_language_registry(registry.clone());
2542 reference_syntax_map.reparse(language.clone(), &buffer);
2543
2544 // Compare the mutated syntax map to the new syntax map
2545 let mutated_layers = mutated_syntax_map.layers(&buffer);
2546 let reference_layers = reference_syntax_map.layers(&buffer);
2547 assert_eq!(
2548 mutated_layers.len(),
2549 reference_layers.len(),
2550 "wrong number of layers at step {i}"
2551 );
2552 for (edited_layer, reference_layer) in
2553 mutated_layers.into_iter().zip(reference_layers.into_iter())
2554 {
2555 assert_eq!(
2556 edited_layer.node.to_sexp(),
2557 reference_layer.node.to_sexp(),
2558 "different layer at step {i}"
2559 );
2560 assert_eq!(
2561 edited_layer.node.range(),
2562 reference_layer.node.range(),
2563 "different layer at step {i}"
2564 );
2565 }
2566 }
2567
2568 (buffer, mutated_syntax_map)
2569 }
2570
2571 fn html_lang() -> Language {
2572 Language::new(
2573 LanguageConfig {
2574 name: "HTML".into(),
2575 path_suffixes: vec!["html".to_string()],
2576 ..Default::default()
2577 },
2578 Some(tree_sitter_html::language()),
2579 )
2580 .with_highlights_query(
2581 r#"
2582 (tag_name) @tag
2583 (erroneous_end_tag_name) @tag
2584 (attribute_name) @property
2585 "#,
2586 )
2587 .unwrap()
2588 }
2589
2590 fn ruby_lang() -> Language {
2591 Language::new(
2592 LanguageConfig {
2593 name: "Ruby".into(),
2594 path_suffixes: vec!["rb".to_string()],
2595 ..Default::default()
2596 },
2597 Some(tree_sitter_ruby::language()),
2598 )
2599 .with_highlights_query(
2600 r#"
2601 ["if" "do" "else" "end"] @keyword
2602 (instance_variable) @ivar
2603 "#,
2604 )
2605 .unwrap()
2606 }
2607
2608 fn erb_lang() -> Language {
2609 Language::new(
2610 LanguageConfig {
2611 name: "ERB".into(),
2612 path_suffixes: vec!["erb".to_string()],
2613 ..Default::default()
2614 },
2615 Some(tree_sitter_embedded_template::language()),
2616 )
2617 .with_highlights_query(
2618 r#"
2619 ["<%" "%>"] @keyword
2620 "#,
2621 )
2622 .unwrap()
2623 .with_injection_query(
2624 r#"
2625 ((code) @content
2626 (#set! "language" "ruby")
2627 (#set! "combined"))
2628
2629 ((content) @content
2630 (#set! "language" "html")
2631 (#set! "combined"))
2632 "#,
2633 )
2634 .unwrap()
2635 }
2636
2637 fn rust_lang() -> Language {
2638 Language::new(
2639 LanguageConfig {
2640 name: "Rust".into(),
2641 path_suffixes: vec!["rs".to_string()],
2642 ..Default::default()
2643 },
2644 Some(tree_sitter_rust::language()),
2645 )
2646 .with_highlights_query(
2647 r#"
2648 (field_identifier) @field
2649 (struct_expression) @struct
2650 "#,
2651 )
2652 .unwrap()
2653 .with_injection_query(
2654 r#"
2655 (macro_invocation
2656 (token_tree) @content
2657 (#set! "language" "rust"))
2658 "#,
2659 )
2660 .unwrap()
2661 }
2662
2663 fn markdown_lang() -> Language {
2664 Language::new(
2665 LanguageConfig {
2666 name: "Markdown".into(),
2667 path_suffixes: vec!["md".into()],
2668 ..Default::default()
2669 },
2670 Some(tree_sitter_markdown::language()),
2671 )
2672 .with_injection_query(
2673 r#"
2674 (fenced_code_block
2675 (info_string
2676 (language) @language)
2677 (code_fence_content) @content)
2678 "#,
2679 )
2680 .unwrap()
2681 }
2682
2683 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2684 let start = buffer.as_rope().to_string().find(text).unwrap();
2685 start..start + text.len()
2686 }
2687
2688 fn assert_layers_for_range(
2689 syntax_map: &SyntaxMap,
2690 buffer: &BufferSnapshot,
2691 range: Range<Point>,
2692 expected_layers: &[&str],
2693 ) {
2694 let layers = syntax_map
2695 .layers_for_range(range, &buffer)
2696 .collect::<Vec<_>>();
2697 assert_eq!(
2698 layers.len(),
2699 expected_layers.len(),
2700 "wrong number of layers"
2701 );
2702 for (i, (SyntaxLayerInfo { node, .. }, expected_s_exp)) in
2703 layers.iter().zip(expected_layers.iter()).enumerate()
2704 {
2705 let actual_s_exp = node.to_sexp();
2706 assert!(
2707 string_contains_sequence(
2708 &actual_s_exp,
2709 &expected_s_exp.split("...").collect::<Vec<_>>()
2710 ),
2711 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2712 );
2713 }
2714 }
2715
2716 fn assert_capture_ranges(
2717 syntax_map: &SyntaxMap,
2718 buffer: &BufferSnapshot,
2719 highlight_query_capture_names: &[&str],
2720 marked_string: &str,
2721 ) {
2722 let mut actual_ranges = Vec::<Range<usize>>::new();
2723 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2724 grammar.highlights_query.as_ref()
2725 });
2726 let queries = captures
2727 .grammars()
2728 .iter()
2729 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2730 .collect::<Vec<_>>();
2731 for capture in captures {
2732 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2733 if highlight_query_capture_names.contains(&name.as_str()) {
2734 actual_ranges.push(capture.node.byte_range());
2735 }
2736 }
2737
2738 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2739 assert_eq!(text, buffer.text());
2740 assert_eq!(actual_ranges, expected_ranges);
2741 }
2742
2743 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2744 let mut last_part_end = 0;
2745 for part in parts {
2746 if let Some(start_ix) = text[last_part_end..].find(part) {
2747 last_part_end = start_ix + part.len();
2748 } else {
2749 return false;
2750 }
2751 }
2752 true
2753 }
2754}