1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use futures::FutureExt;
4use lazy_static::lazy_static;
5use parking_lot::Mutex;
6use std::{
7 borrow::Cow,
8 cell::RefCell,
9 cmp::{self, Ordering, Reverse},
10 collections::BinaryHeap,
11 iter,
12 ops::{Deref, DerefMut, Range},
13 sync::Arc,
14};
15use sum_tree::{Bias, SeekTarget, SumTree};
16use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
17use tree_sitter::{
18 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
19};
20
21thread_local! {
22 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
23}
24
25lazy_static! {
26 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
27}
28
29#[derive(Default)]
30pub struct SyntaxMap {
31 snapshot: SyntaxSnapshot,
32 language_registry: Option<Arc<LanguageRegistry>>,
33}
34
35#[derive(Clone, Default)]
36pub struct SyntaxSnapshot {
37 layers: SumTree<SyntaxLayer>,
38 parsed_version: clock::Global,
39 interpolated_version: clock::Global,
40 language_registry_version: usize,
41}
42
43#[derive(Default)]
44pub struct SyntaxMapCaptures<'a> {
45 layers: Vec<SyntaxMapCapturesLayer<'a>>,
46 active_layer_count: usize,
47 grammars: Vec<&'a Grammar>,
48}
49
50#[derive(Default)]
51pub struct SyntaxMapMatches<'a> {
52 layers: Vec<SyntaxMapMatchesLayer<'a>>,
53 active_layer_count: usize,
54 grammars: Vec<&'a Grammar>,
55}
56
57#[derive(Debug)]
58pub struct SyntaxMapCapture<'a> {
59 pub depth: usize,
60 pub node: Node<'a>,
61 pub index: u32,
62 pub grammar_index: usize,
63}
64
65#[derive(Debug)]
66pub struct SyntaxMapMatch<'a> {
67 pub depth: usize,
68 pub pattern_index: usize,
69 pub captures: &'a [QueryCapture<'a>],
70 pub grammar_index: usize,
71}
72
73struct SyntaxMapCapturesLayer<'a> {
74 depth: usize,
75 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
76 next_capture: Option<QueryCapture<'a>>,
77 grammar_index: usize,
78 _query_cursor: QueryCursorHandle,
79}
80
81struct SyntaxMapMatchesLayer<'a> {
82 depth: usize,
83 next_pattern_index: usize,
84 next_captures: Vec<QueryCapture<'a>>,
85 has_next: bool,
86 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
87 grammar_index: usize,
88 _query_cursor: QueryCursorHandle,
89}
90
91#[derive(Clone)]
92struct SyntaxLayer {
93 depth: usize,
94 range: Range<Anchor>,
95 content: SyntaxLayerContent,
96}
97
98#[derive(Clone)]
99enum SyntaxLayerContent {
100 Parsed {
101 tree: tree_sitter::Tree,
102 language: Arc<Language>,
103 },
104 Pending {
105 language_name: Arc<str>,
106 },
107}
108
109impl SyntaxLayerContent {
110 fn language_id(&self) -> Option<usize> {
111 match self {
112 SyntaxLayerContent::Parsed { language, .. } => language.id(),
113 SyntaxLayerContent::Pending { .. } => None,
114 }
115 }
116
117 fn tree(&self) -> Option<&Tree> {
118 match self {
119 SyntaxLayerContent::Parsed { tree, .. } => Some(tree),
120 SyntaxLayerContent::Pending { .. } => None,
121 }
122 }
123}
124
125#[derive(Debug)]
126pub struct SyntaxLayerInfo<'a> {
127 pub depth: usize,
128 pub node: Node<'a>,
129 pub language: &'a Arc<Language>,
130}
131
132#[derive(Debug, Clone)]
133struct SyntaxLayerSummary {
134 min_depth: usize,
135 max_depth: usize,
136 range: Range<Anchor>,
137 last_layer_range: Range<Anchor>,
138 last_layer_language: Option<usize>,
139 contains_unknown_injections: bool,
140}
141
142#[derive(Clone, Debug)]
143struct SyntaxLayerPosition {
144 depth: usize,
145 range: Range<Anchor>,
146 language: Option<usize>,
147}
148
149#[derive(Clone, Debug)]
150struct ChangeStartPosition {
151 depth: usize,
152 position: Anchor,
153}
154
155#[derive(Clone, Debug)]
156struct SyntaxLayerPositionBeforeChange {
157 position: SyntaxLayerPosition,
158 change: ChangeStartPosition,
159}
160
161struct ParseStep {
162 depth: usize,
163 language: ParseStepLanguage,
164 range: Range<Anchor>,
165 included_ranges: Vec<tree_sitter::Range>,
166 mode: ParseMode,
167}
168
169#[derive(Debug)]
170enum ParseStepLanguage {
171 Loaded { language: Arc<Language> },
172 Pending { name: Arc<str> },
173}
174
175impl ParseStepLanguage {
176 fn id(&self) -> Option<usize> {
177 match self {
178 ParseStepLanguage::Loaded { language } => language.id(),
179 ParseStepLanguage::Pending { .. } => None,
180 }
181 }
182}
183
184enum ParseMode {
185 Single,
186 Combined {
187 parent_layer_range: Range<usize>,
188 parent_layer_changed_ranges: Vec<Range<usize>>,
189 },
190}
191
192#[derive(Debug, PartialEq, Eq)]
193struct ChangedRegion {
194 depth: usize,
195 range: Range<Anchor>,
196}
197
198#[derive(Default)]
199struct ChangeRegionSet(Vec<ChangedRegion>);
200
201struct TextProvider<'a>(&'a Rope);
202
203struct ByteChunks<'a>(text::Chunks<'a>);
204
205struct QueryCursorHandle(Option<QueryCursor>);
206
207impl SyntaxMap {
208 pub fn new() -> Self {
209 Self::default()
210 }
211
212 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
213 self.language_registry = Some(registry);
214 }
215
216 pub fn snapshot(&self) -> SyntaxSnapshot {
217 self.snapshot.clone()
218 }
219
220 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
221 self.language_registry.clone()
222 }
223
224 pub fn interpolate(&mut self, text: &BufferSnapshot) {
225 self.snapshot.interpolate(text);
226 }
227
228 #[cfg(test)]
229 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
230 self.snapshot
231 .reparse(text, self.language_registry.clone(), language);
232 }
233
234 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot) {
235 self.snapshot = snapshot;
236 }
237
238 pub fn clear(&mut self) {
239 self.snapshot = SyntaxSnapshot::default();
240 }
241}
242
243impl SyntaxSnapshot {
244 pub fn is_empty(&self) -> bool {
245 self.layers.is_empty()
246 }
247
248 fn interpolate(&mut self, text: &BufferSnapshot) {
249 let edits = text
250 .anchored_edits_since::<(usize, Point)>(&self.interpolated_version)
251 .collect::<Vec<_>>();
252 self.interpolated_version = text.version().clone();
253
254 if edits.is_empty() {
255 return;
256 }
257
258 let mut layers = SumTree::new();
259 let mut first_edit_ix_for_depth = 0;
260 let mut prev_depth = 0;
261 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
262 cursor.next(text);
263
264 'outer: loop {
265 let depth = cursor.end(text).max_depth;
266 if depth > prev_depth {
267 first_edit_ix_for_depth = 0;
268 prev_depth = depth;
269 }
270
271 // Preserve any layers at this depth that precede the first edit.
272 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
273 let target = ChangeStartPosition {
274 depth,
275 position: edit_range.start,
276 };
277 if target.cmp(&cursor.start(), text).is_gt() {
278 let slice = cursor.slice(&target, Bias::Left, text);
279 layers.push_tree(slice, text);
280 }
281 }
282 // If this layer follows all of the edits, then preserve it and any
283 // subsequent layers at this same depth.
284 else if cursor.item().is_some() {
285 let slice = cursor.slice(
286 &SyntaxLayerPosition {
287 depth: depth + 1,
288 range: Anchor::MIN..Anchor::MAX,
289 language: None,
290 },
291 Bias::Left,
292 text,
293 );
294 layers.push_tree(slice, text);
295 continue;
296 };
297
298 let Some(layer) = cursor.item() else { break };
299 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
300
301 // Ignore edits that end before the start of this layer, and don't consider them
302 // for any subsequent layers at this same depth.
303 loop {
304 let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) else { continue 'outer };
305 if edit_range.end.cmp(&layer.range.start, text).is_le() {
306 first_edit_ix_for_depth += 1;
307 } else {
308 break;
309 }
310 }
311
312 let mut layer = layer.clone();
313 if let SyntaxLayerContent::Parsed { tree, .. } = &mut layer.content {
314 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
315 // Ignore any edits that follow this layer.
316 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
317 break;
318 }
319
320 // Apply any edits that intersect this layer to the layer's syntax tree.
321 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
322 tree_sitter::InputEdit {
323 start_byte: edit.new.start.0 - start_byte,
324 old_end_byte: edit.new.start.0 - start_byte
325 + (edit.old.end.0 - edit.old.start.0),
326 new_end_byte: edit.new.end.0 - start_byte,
327 start_position: (edit.new.start.1 - start_point).to_ts_point(),
328 old_end_position: (edit.new.start.1 - start_point
329 + (edit.old.end.1 - edit.old.start.1))
330 .to_ts_point(),
331 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
332 }
333 } else {
334 let node = tree.root_node();
335 tree_sitter::InputEdit {
336 start_byte: 0,
337 old_end_byte: node.end_byte(),
338 new_end_byte: 0,
339 start_position: Default::default(),
340 old_end_position: node.end_position(),
341 new_end_position: Default::default(),
342 }
343 };
344
345 tree.edit(&tree_edit);
346 }
347
348 debug_assert!(
349 tree.root_node().end_byte() <= text.len(),
350 "tree's size {}, is larger than text size {}",
351 tree.root_node().end_byte(),
352 text.len(),
353 );
354 }
355
356 layers.push(layer, text);
357 cursor.next(text);
358 }
359
360 layers.push_tree(cursor.suffix(&text), &text);
361 drop(cursor);
362 self.layers = layers;
363 }
364
365 pub fn reparse(
366 &mut self,
367 text: &BufferSnapshot,
368 registry: Option<Arc<LanguageRegistry>>,
369 root_language: Arc<Language>,
370 ) {
371 let edit_ranges = text
372 .edits_since::<usize>(&self.parsed_version)
373 .map(|edit| edit.new)
374 .collect::<Vec<_>>();
375 self.reparse_with_ranges(text, root_language.clone(), edit_ranges, registry.as_ref());
376
377 if let Some(registry) = registry {
378 if registry.version() != self.language_registry_version {
379 let mut resolved_injection_ranges = Vec::new();
380 let mut cursor = self
381 .layers
382 .filter::<_, ()>(|summary| summary.contains_unknown_injections);
383 cursor.next(text);
384 while let Some(layer) = cursor.item() {
385 let SyntaxLayerContent::Pending { language_name } = &layer.content else { unreachable!() };
386 if registry
387 .language_for_name_or_extension(language_name)
388 .now_or_never()
389 .and_then(|language| language.ok())
390 .is_some()
391 {
392 resolved_injection_ranges.push(layer.range.to_offset(text));
393 }
394
395 cursor.next(text);
396 }
397 drop(cursor);
398
399 if !resolved_injection_ranges.is_empty() {
400 self.reparse_with_ranges(
401 text,
402 root_language,
403 resolved_injection_ranges,
404 Some(®istry),
405 );
406 }
407 self.language_registry_version = registry.version();
408 }
409 }
410 }
411
412 fn reparse_with_ranges(
413 &mut self,
414 text: &BufferSnapshot,
415 root_language: Arc<Language>,
416 invalidated_ranges: Vec<Range<usize>>,
417 registry: Option<&Arc<LanguageRegistry>>,
418 ) {
419 let max_depth = self.layers.summary().max_depth;
420 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
421 cursor.next(&text);
422 let mut layers = SumTree::new();
423
424 let mut changed_regions = ChangeRegionSet::default();
425 let mut queue = BinaryHeap::new();
426 let mut combined_injection_ranges = HashMap::default();
427 queue.push(ParseStep {
428 depth: 0,
429 language: ParseStepLanguage::Loaded {
430 language: root_language,
431 },
432 included_ranges: vec![tree_sitter::Range {
433 start_byte: 0,
434 end_byte: text.len(),
435 start_point: Point::zero().to_ts_point(),
436 end_point: text.max_point().to_ts_point(),
437 }],
438 range: Anchor::MIN..Anchor::MAX,
439 mode: ParseMode::Single,
440 });
441
442 loop {
443 let step = queue.pop();
444 let position = if let Some(step) = &step {
445 SyntaxLayerPosition {
446 depth: step.depth,
447 range: step.range.clone(),
448 language: step.language.id(),
449 }
450 } else {
451 SyntaxLayerPosition {
452 depth: max_depth + 1,
453 range: Anchor::MAX..Anchor::MAX,
454 language: None,
455 }
456 };
457
458 let mut done = cursor.item().is_none();
459 while !done && position.cmp(&cursor.end(text), &text).is_gt() {
460 done = true;
461
462 let bounded_position = SyntaxLayerPositionBeforeChange {
463 position: position.clone(),
464 change: changed_regions.start_position(),
465 };
466 if bounded_position.cmp(&cursor.start(), &text).is_gt() {
467 let slice = cursor.slice(&bounded_position, Bias::Left, text);
468 if !slice.is_empty() {
469 layers.push_tree(slice, &text);
470 if changed_regions.prune(cursor.end(text), text) {
471 done = false;
472 }
473 }
474 }
475
476 while position.cmp(&cursor.end(text), text).is_gt() {
477 let Some(layer) = cursor.item() else { break };
478
479 if changed_regions.intersects(&layer, text) {
480 changed_regions.insert(
481 ChangedRegion {
482 depth: layer.depth + 1,
483 range: layer.range.clone(),
484 },
485 text,
486 );
487 } else {
488 layers.push(layer.clone(), text);
489 }
490
491 cursor.next(text);
492 if changed_regions.prune(cursor.end(text), text) {
493 done = false;
494 }
495 }
496 }
497
498 let Some(step) = step else { break };
499 let (step_start_byte, step_start_point) =
500 step.range.start.summary::<(usize, Point)>(text);
501 let step_end_byte = step.range.end.to_offset(text);
502
503 let mut old_layer = cursor.item();
504 if let Some(layer) = old_layer {
505 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
506 && layer.content.language_id() == step.language.id()
507 {
508 cursor.next(&text);
509 } else {
510 old_layer = None;
511 }
512 }
513
514 let content = match step.language {
515 ParseStepLanguage::Loaded { language } => {
516 let Some(grammar) = language.grammar() else { continue };
517 let tree;
518 let changed_ranges;
519
520 let mut included_ranges = step.included_ranges;
521 for range in &mut included_ranges {
522 range.start_byte -= step_start_byte;
523 range.end_byte -= step_start_byte;
524 range.start_point = (Point::from_ts_point(range.start_point)
525 - step_start_point)
526 .to_ts_point();
527 range.end_point = (Point::from_ts_point(range.end_point)
528 - step_start_point)
529 .to_ts_point();
530 }
531
532 if let Some(SyntaxLayerContent::Parsed { tree: old_tree, .. }) =
533 old_layer.map(|layer| &layer.content)
534 {
535 if let ParseMode::Combined {
536 mut parent_layer_changed_ranges,
537 ..
538 } = step.mode
539 {
540 for range in &mut parent_layer_changed_ranges {
541 range.start -= step_start_byte;
542 range.end -= step_start_byte;
543 }
544
545 included_ranges = splice_included_ranges(
546 old_tree.included_ranges(),
547 &parent_layer_changed_ranges,
548 &included_ranges,
549 );
550 }
551
552 tree = parse_text(
553 grammar,
554 text.as_rope(),
555 step_start_byte,
556 included_ranges,
557 Some(old_tree.clone()),
558 );
559 changed_ranges = join_ranges(
560 invalidated_ranges.iter().cloned().filter(|range| {
561 range.start <= step_end_byte && range.end >= step_start_byte
562 }),
563 old_tree.changed_ranges(&tree).map(|r| {
564 step_start_byte + r.start_byte..step_start_byte + r.end_byte
565 }),
566 );
567 } else {
568 tree = parse_text(
569 grammar,
570 text.as_rope(),
571 step_start_byte,
572 included_ranges,
573 None,
574 );
575 changed_ranges = vec![step_start_byte..step_end_byte];
576 }
577
578 if let (Some((config, registry)), false) = (
579 grammar.injection_config.as_ref().zip(registry.as_ref()),
580 changed_ranges.is_empty(),
581 ) {
582 for range in &changed_ranges {
583 changed_regions.insert(
584 ChangedRegion {
585 depth: step.depth + 1,
586 range: text.anchor_before(range.start)
587 ..text.anchor_after(range.end),
588 },
589 text,
590 );
591 }
592 get_injections(
593 config,
594 text,
595 tree.root_node_with_offset(
596 step_start_byte,
597 step_start_point.to_ts_point(),
598 ),
599 registry,
600 step.depth + 1,
601 &changed_ranges,
602 &mut combined_injection_ranges,
603 &mut queue,
604 );
605 }
606
607 SyntaxLayerContent::Parsed { tree, language }
608 }
609 ParseStepLanguage::Pending { name } => SyntaxLayerContent::Pending {
610 language_name: name,
611 },
612 };
613
614 layers.push(
615 SyntaxLayer {
616 depth: step.depth,
617 range: step.range,
618 content,
619 },
620 &text,
621 );
622 }
623
624 drop(cursor);
625 self.layers = layers;
626 self.interpolated_version = text.version.clone();
627 self.parsed_version = text.version.clone();
628 #[cfg(debug_assertions)]
629 self.check_invariants(text);
630 }
631
632 #[cfg(debug_assertions)]
633 fn check_invariants(&self, text: &BufferSnapshot) {
634 let mut max_depth = 0;
635 let mut prev_range: Option<Range<Anchor>> = None;
636 for layer in self.layers.iter() {
637 if layer.depth == max_depth {
638 if let Some(prev_range) = prev_range {
639 match layer.range.start.cmp(&prev_range.start, text) {
640 Ordering::Less => panic!("layers out of order"),
641 Ordering::Equal => {
642 assert!(layer.range.end.cmp(&prev_range.end, text).is_ge())
643 }
644 Ordering::Greater => {}
645 }
646 }
647 } else if layer.depth < max_depth {
648 panic!("layers out of order")
649 }
650 max_depth = layer.depth;
651 prev_range = Some(layer.range.clone());
652 }
653 }
654
655 pub fn single_tree_captures<'a>(
656 range: Range<usize>,
657 text: &'a Rope,
658 tree: &'a Tree,
659 language: &'a Arc<Language>,
660 query: fn(&Grammar) -> Option<&Query>,
661 ) -> SyntaxMapCaptures<'a> {
662 SyntaxMapCaptures::new(
663 range.clone(),
664 text,
665 [SyntaxLayerInfo {
666 language,
667 depth: 0,
668 node: tree.root_node(),
669 }]
670 .into_iter(),
671 query,
672 )
673 }
674
675 pub fn captures<'a>(
676 &'a self,
677 range: Range<usize>,
678 buffer: &'a BufferSnapshot,
679 query: fn(&Grammar) -> Option<&Query>,
680 ) -> SyntaxMapCaptures {
681 SyntaxMapCaptures::new(
682 range.clone(),
683 buffer.as_rope(),
684 self.layers_for_range(range, buffer).into_iter(),
685 query,
686 )
687 }
688
689 pub fn matches<'a>(
690 &'a self,
691 range: Range<usize>,
692 buffer: &'a BufferSnapshot,
693 query: fn(&Grammar) -> Option<&Query>,
694 ) -> SyntaxMapMatches {
695 SyntaxMapMatches::new(
696 range.clone(),
697 buffer.as_rope(),
698 self.layers_for_range(range, buffer).into_iter(),
699 query,
700 )
701 }
702
703 #[cfg(test)]
704 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
705 self.layers_for_range(0..buffer.len(), buffer).collect()
706 }
707
708 pub fn layers_for_range<'a, T: ToOffset>(
709 &'a self,
710 range: Range<T>,
711 buffer: &'a BufferSnapshot,
712 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
713 let start = buffer.anchor_before(range.start.to_offset(buffer));
714 let end = buffer.anchor_after(range.end.to_offset(buffer));
715
716 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
717 if summary.max_depth > summary.min_depth {
718 true
719 } else {
720 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
721 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
722 !is_before_start && !is_after_end
723 }
724 });
725
726 cursor.next(buffer);
727 iter::from_fn(move || {
728 while let Some(layer) = cursor.item() {
729 if let SyntaxLayerContent::Parsed { tree, language } = &layer.content {
730 let info = SyntaxLayerInfo {
731 language,
732 depth: layer.depth,
733 node: tree.root_node_with_offset(
734 layer.range.start.to_offset(buffer),
735 layer.range.start.to_point(buffer).to_ts_point(),
736 ),
737 };
738 cursor.next(buffer);
739 return Some(info);
740 } else {
741 cursor.next(buffer);
742 }
743 }
744 None
745 })
746 }
747
748 pub fn contains_unknown_injections(&self) -> bool {
749 self.layers.summary().contains_unknown_injections
750 }
751
752 pub fn language_registry_version(&self) -> usize {
753 self.language_registry_version
754 }
755}
756
757impl<'a> SyntaxMapCaptures<'a> {
758 fn new(
759 range: Range<usize>,
760 text: &'a Rope,
761 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
762 query: fn(&Grammar) -> Option<&Query>,
763 ) -> Self {
764 let mut result = Self {
765 layers: Vec::new(),
766 grammars: Vec::new(),
767 active_layer_count: 0,
768 };
769 for SyntaxLayerInfo {
770 language,
771 depth,
772 node,
773 } in layers
774 {
775 let grammar = match &language.grammar {
776 Some(grammer) => grammer,
777 None => continue,
778 };
779 let query = match query(&grammar) {
780 Some(query) => query,
781 None => continue,
782 };
783
784 let mut query_cursor = QueryCursorHandle::new();
785
786 // TODO - add a Tree-sitter API to remove the need for this.
787 let cursor = unsafe {
788 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
789 };
790
791 cursor.set_byte_range(range.clone());
792 let captures = cursor.captures(query, node, TextProvider(text));
793 let grammar_index = result
794 .grammars
795 .iter()
796 .position(|g| g.id == grammar.id())
797 .unwrap_or_else(|| {
798 result.grammars.push(grammar);
799 result.grammars.len() - 1
800 });
801 let mut layer = SyntaxMapCapturesLayer {
802 depth,
803 grammar_index,
804 next_capture: None,
805 captures,
806 _query_cursor: query_cursor,
807 };
808
809 layer.advance();
810 if layer.next_capture.is_some() {
811 let key = layer.sort_key();
812 let ix = match result.layers[..result.active_layer_count]
813 .binary_search_by_key(&key, |layer| layer.sort_key())
814 {
815 Ok(ix) | Err(ix) => ix,
816 };
817 result.layers.insert(ix, layer);
818 result.active_layer_count += 1;
819 } else {
820 result.layers.push(layer);
821 }
822 }
823
824 result
825 }
826
827 pub fn grammars(&self) -> &[&'a Grammar] {
828 &self.grammars
829 }
830
831 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
832 let layer = self.layers[..self.active_layer_count].first()?;
833 let capture = layer.next_capture?;
834 Some(SyntaxMapCapture {
835 depth: layer.depth,
836 grammar_index: layer.grammar_index,
837 index: capture.index,
838 node: capture.node,
839 })
840 }
841
842 pub fn advance(&mut self) -> bool {
843 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
844 layer
845 } else {
846 return false;
847 };
848
849 layer.advance();
850 if layer.next_capture.is_some() {
851 let key = layer.sort_key();
852 let i = 1 + self.layers[1..self.active_layer_count]
853 .iter()
854 .position(|later_layer| key < later_layer.sort_key())
855 .unwrap_or(self.active_layer_count - 1);
856 self.layers[0..i].rotate_left(1);
857 } else {
858 self.layers[0..self.active_layer_count].rotate_left(1);
859 self.active_layer_count -= 1;
860 }
861
862 true
863 }
864
865 pub fn set_byte_range(&mut self, range: Range<usize>) {
866 for layer in &mut self.layers {
867 layer.captures.set_byte_range(range.clone());
868 if let Some(capture) = &layer.next_capture {
869 if capture.node.end_byte() > range.start {
870 continue;
871 }
872 }
873 layer.advance();
874 }
875 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
876 self.active_layer_count = self
877 .layers
878 .iter()
879 .position(|layer| layer.next_capture.is_none())
880 .unwrap_or(self.layers.len());
881 }
882}
883
884impl<'a> SyntaxMapMatches<'a> {
885 fn new(
886 range: Range<usize>,
887 text: &'a Rope,
888 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
889 query: fn(&Grammar) -> Option<&Query>,
890 ) -> Self {
891 let mut result = Self::default();
892 for SyntaxLayerInfo {
893 language,
894 depth,
895 node,
896 } in layers
897 {
898 let grammar = match &language.grammar {
899 Some(grammer) => grammer,
900 None => continue,
901 };
902 let query = match query(&grammar) {
903 Some(query) => query,
904 None => continue,
905 };
906
907 let mut query_cursor = QueryCursorHandle::new();
908
909 // TODO - add a Tree-sitter API to remove the need for this.
910 let cursor = unsafe {
911 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
912 };
913
914 cursor.set_byte_range(range.clone());
915 let matches = cursor.matches(query, node, TextProvider(text));
916 let grammar_index = result
917 .grammars
918 .iter()
919 .position(|g| g.id == grammar.id())
920 .unwrap_or_else(|| {
921 result.grammars.push(grammar);
922 result.grammars.len() - 1
923 });
924 let mut layer = SyntaxMapMatchesLayer {
925 depth,
926 grammar_index,
927 matches,
928 next_pattern_index: 0,
929 next_captures: Vec::new(),
930 has_next: false,
931 _query_cursor: query_cursor,
932 };
933
934 layer.advance();
935 if layer.has_next {
936 let key = layer.sort_key();
937 let ix = match result.layers[..result.active_layer_count]
938 .binary_search_by_key(&key, |layer| layer.sort_key())
939 {
940 Ok(ix) | Err(ix) => ix,
941 };
942 result.layers.insert(ix, layer);
943 result.active_layer_count += 1;
944 } else {
945 result.layers.push(layer);
946 }
947 }
948 result
949 }
950
951 pub fn grammars(&self) -> &[&'a Grammar] {
952 &self.grammars
953 }
954
955 pub fn peek(&self) -> Option<SyntaxMapMatch> {
956 let layer = self.layers.first()?;
957 if !layer.has_next {
958 return None;
959 }
960 Some(SyntaxMapMatch {
961 depth: layer.depth,
962 grammar_index: layer.grammar_index,
963 pattern_index: layer.next_pattern_index,
964 captures: &layer.next_captures,
965 })
966 }
967
968 pub fn advance(&mut self) -> bool {
969 let layer = if let Some(layer) = self.layers.first_mut() {
970 layer
971 } else {
972 return false;
973 };
974
975 layer.advance();
976 if layer.has_next {
977 let key = layer.sort_key();
978 let i = 1 + self.layers[1..self.active_layer_count]
979 .iter()
980 .position(|later_layer| key < later_layer.sort_key())
981 .unwrap_or(self.active_layer_count - 1);
982 self.layers[0..i].rotate_left(1);
983 } else {
984 self.layers[0..self.active_layer_count].rotate_left(1);
985 self.active_layer_count -= 1;
986 }
987
988 true
989 }
990}
991
992impl<'a> SyntaxMapCapturesLayer<'a> {
993 fn advance(&mut self) {
994 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
995 }
996
997 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
998 if let Some(capture) = &self.next_capture {
999 let range = capture.node.byte_range();
1000 (range.start, Reverse(range.end), self.depth)
1001 } else {
1002 (usize::MAX, Reverse(0), usize::MAX)
1003 }
1004 }
1005}
1006
1007impl<'a> SyntaxMapMatchesLayer<'a> {
1008 fn advance(&mut self) {
1009 if let Some(mat) = self.matches.next() {
1010 self.next_captures.clear();
1011 self.next_captures.extend_from_slice(&mat.captures);
1012 self.next_pattern_index = mat.pattern_index;
1013 self.has_next = true;
1014 } else {
1015 self.has_next = false;
1016 }
1017 }
1018
1019 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
1020 if self.has_next {
1021 let captures = &self.next_captures;
1022 if let Some((first, last)) = captures.first().zip(captures.last()) {
1023 return (
1024 first.node.start_byte(),
1025 Reverse(last.node.end_byte()),
1026 self.depth,
1027 );
1028 }
1029 }
1030 (usize::MAX, Reverse(0), usize::MAX)
1031 }
1032}
1033
1034impl<'a> Iterator for SyntaxMapCaptures<'a> {
1035 type Item = SyntaxMapCapture<'a>;
1036
1037 fn next(&mut self) -> Option<Self::Item> {
1038 let result = self.peek();
1039 self.advance();
1040 result
1041 }
1042}
1043
1044fn join_ranges(
1045 a: impl Iterator<Item = Range<usize>>,
1046 b: impl Iterator<Item = Range<usize>>,
1047) -> Vec<Range<usize>> {
1048 let mut result = Vec::<Range<usize>>::new();
1049 let mut a = a.peekable();
1050 let mut b = b.peekable();
1051 loop {
1052 let range = match (a.peek(), b.peek()) {
1053 (Some(range_a), Some(range_b)) => {
1054 if range_a.start < range_b.start {
1055 a.next().unwrap()
1056 } else {
1057 b.next().unwrap()
1058 }
1059 }
1060 (None, Some(_)) => b.next().unwrap(),
1061 (Some(_), None) => a.next().unwrap(),
1062 (None, None) => break,
1063 };
1064
1065 if let Some(last) = result.last_mut() {
1066 if range.start <= last.end {
1067 last.end = last.end.max(range.end);
1068 continue;
1069 }
1070 }
1071 result.push(range);
1072 }
1073 result
1074}
1075
1076fn parse_text(
1077 grammar: &Grammar,
1078 text: &Rope,
1079 start_byte: usize,
1080 ranges: Vec<tree_sitter::Range>,
1081 old_tree: Option<Tree>,
1082) -> Tree {
1083 PARSER.with(|parser| {
1084 let mut parser = parser.borrow_mut();
1085 let mut chunks = text.chunks_in_range(start_byte..text.len());
1086 parser
1087 .set_included_ranges(&ranges)
1088 .expect("overlapping ranges");
1089 parser
1090 .set_language(grammar.ts_language)
1091 .expect("incompatible grammar");
1092 parser
1093 .parse_with(
1094 &mut move |offset, _| {
1095 chunks.seek(start_byte + offset);
1096 chunks.next().unwrap_or("").as_bytes()
1097 },
1098 old_tree.as_ref(),
1099 )
1100 .expect("invalid language")
1101 })
1102}
1103
1104fn get_injections(
1105 config: &InjectionConfig,
1106 text: &BufferSnapshot,
1107 node: Node,
1108 language_registry: &Arc<LanguageRegistry>,
1109 depth: usize,
1110 changed_ranges: &[Range<usize>],
1111 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
1112 queue: &mut BinaryHeap<ParseStep>,
1113) {
1114 let mut query_cursor = QueryCursorHandle::new();
1115 let mut prev_match = None;
1116
1117 combined_injection_ranges.clear();
1118 for pattern in &config.patterns {
1119 if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
1120 if let Some(language) = language_registry
1121 .language_for_name_or_extension(language_name)
1122 .now_or_never()
1123 .and_then(|language| language.ok())
1124 {
1125 combined_injection_ranges.insert(language, Vec::new());
1126 }
1127 }
1128 }
1129
1130 for query_range in changed_ranges {
1131 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
1132 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
1133 let content_ranges = mat
1134 .nodes_for_capture_index(config.content_capture_ix)
1135 .map(|node| node.range())
1136 .collect::<Vec<_>>();
1137 if content_ranges.is_empty() {
1138 continue;
1139 }
1140
1141 // Avoid duplicate matches if two changed ranges intersect the same injection.
1142 let content_range =
1143 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
1144 if let Some((last_pattern_ix, last_range)) = &prev_match {
1145 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1146 continue;
1147 }
1148 }
1149 prev_match = Some((mat.pattern_index, content_range.clone()));
1150
1151 let combined = config.patterns[mat.pattern_index].combined;
1152
1153 let mut language_name = None;
1154 let mut step_range = content_range.clone();
1155 if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {
1156 language_name = Some(Cow::Borrowed(name.as_ref()))
1157 } else if let Some(language_node) = config
1158 .language_capture_ix
1159 .and_then(|ix| mat.nodes_for_capture_index(ix).next())
1160 {
1161 step_range.start = cmp::min(content_range.start, language_node.start_byte());
1162 step_range.end = cmp::max(content_range.end, language_node.end_byte());
1163 language_name = Some(Cow::Owned(
1164 text.text_for_range(language_node.byte_range()).collect(),
1165 ))
1166 };
1167
1168 if let Some(language_name) = language_name {
1169 let language = language_registry
1170 .language_for_name_or_extension(&language_name)
1171 .now_or_never()
1172 .and_then(|language| language.ok());
1173 let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
1174 if let Some(language) = language {
1175 if combined {
1176 combined_injection_ranges
1177 .get_mut(&language.clone())
1178 .unwrap()
1179 .extend(content_ranges);
1180 } else {
1181 queue.push(ParseStep {
1182 depth,
1183 language: ParseStepLanguage::Loaded { language },
1184 included_ranges: content_ranges,
1185 range,
1186 mode: ParseMode::Single,
1187 });
1188 }
1189 } else {
1190 queue.push(ParseStep {
1191 depth,
1192 language: ParseStepLanguage::Pending {
1193 name: language_name.into(),
1194 },
1195 included_ranges: content_ranges,
1196 range,
1197 mode: ParseMode::Single,
1198 });
1199 }
1200 }
1201 }
1202 }
1203
1204 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1205 included_ranges.sort_unstable();
1206 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1207 queue.push(ParseStep {
1208 depth,
1209 language: ParseStepLanguage::Loaded { language },
1210 range,
1211 included_ranges,
1212 mode: ParseMode::Combined {
1213 parent_layer_range: node.start_byte()..node.end_byte(),
1214 parent_layer_changed_ranges: changed_ranges.to_vec(),
1215 },
1216 })
1217 }
1218}
1219
1220fn splice_included_ranges(
1221 mut ranges: Vec<tree_sitter::Range>,
1222 changed_ranges: &[Range<usize>],
1223 new_ranges: &[tree_sitter::Range],
1224) -> Vec<tree_sitter::Range> {
1225 let mut changed_ranges = changed_ranges.into_iter().peekable();
1226 let mut new_ranges = new_ranges.into_iter().peekable();
1227 let mut ranges_ix = 0;
1228 loop {
1229 let new_range = new_ranges.peek();
1230 let mut changed_range = changed_ranges.peek();
1231
1232 // Remove ranges that have changed before inserting any new ranges
1233 // into those ranges.
1234 if let Some((changed, new)) = changed_range.zip(new_range) {
1235 if new.end_byte < changed.start {
1236 changed_range = None;
1237 }
1238 }
1239
1240 if let Some(changed) = changed_range {
1241 let mut start_ix = ranges_ix
1242 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1243 Ok(ix) | Err(ix) => ix,
1244 };
1245 let mut end_ix = ranges_ix
1246 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1247 Ok(ix) => ix + 1,
1248 Err(ix) => ix,
1249 };
1250
1251 // If there are empty ranges, then there may be multiple ranges with the same
1252 // start or end. Expand the splice to include any adjacent ranges that touch
1253 // the changed range.
1254 while start_ix > 0 {
1255 if ranges[start_ix - 1].end_byte == changed.start {
1256 start_ix -= 1;
1257 } else {
1258 break;
1259 }
1260 }
1261 while let Some(range) = ranges.get(end_ix) {
1262 if range.start_byte == changed.end {
1263 end_ix += 1;
1264 } else {
1265 break;
1266 }
1267 }
1268
1269 if end_ix > start_ix {
1270 ranges.splice(start_ix..end_ix, []);
1271 }
1272 changed_ranges.next();
1273 ranges_ix = start_ix;
1274 } else if let Some(new_range) = new_range {
1275 let ix = ranges_ix
1276 + match ranges[ranges_ix..]
1277 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1278 {
1279 Ok(ix) | Err(ix) => ix,
1280 };
1281 ranges.insert(ix, **new_range);
1282 new_ranges.next();
1283 ranges_ix = ix + 1;
1284 } else {
1285 break;
1286 }
1287 }
1288 ranges
1289}
1290
1291impl<'a> SyntaxLayerInfo<'a> {
1292 pub(crate) fn override_id(&self, offset: usize, text: &text::BufferSnapshot) -> Option<u32> {
1293 let text = TextProvider(text.as_rope());
1294 let config = self.language.grammar.as_ref()?.override_config.as_ref()?;
1295
1296 let mut query_cursor = QueryCursorHandle::new();
1297 query_cursor.set_byte_range(offset..offset);
1298
1299 let mut smallest_match: Option<(u32, Range<usize>)> = None;
1300 for mat in query_cursor.matches(&config.query, self.node, text) {
1301 for capture in mat.captures {
1302 if !config.values.contains_key(&capture.index) {
1303 continue;
1304 }
1305
1306 let range = capture.node.byte_range();
1307 if offset <= range.start || offset >= range.end {
1308 continue;
1309 }
1310
1311 if let Some((_, smallest_range)) = &smallest_match {
1312 if range.len() < smallest_range.len() {
1313 smallest_match = Some((capture.index, range))
1314 }
1315 continue;
1316 }
1317
1318 smallest_match = Some((capture.index, range));
1319 }
1320 }
1321
1322 smallest_match.map(|(index, _)| index)
1323 }
1324}
1325
1326impl std::ops::Deref for SyntaxMap {
1327 type Target = SyntaxSnapshot;
1328
1329 fn deref(&self) -> &Self::Target {
1330 &self.snapshot
1331 }
1332}
1333
1334impl PartialEq for ParseStep {
1335 fn eq(&self, _: &Self) -> bool {
1336 false
1337 }
1338}
1339
1340impl Eq for ParseStep {}
1341
1342impl PartialOrd for ParseStep {
1343 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1344 Some(self.cmp(&other))
1345 }
1346}
1347
1348impl Ord for ParseStep {
1349 fn cmp(&self, other: &Self) -> Ordering {
1350 let range_a = self.range();
1351 let range_b = other.range();
1352 Ord::cmp(&other.depth, &self.depth)
1353 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1354 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1355 .then_with(|| self.language.id().cmp(&other.language.id()))
1356 }
1357}
1358
1359impl ParseStep {
1360 fn range(&self) -> Range<usize> {
1361 if let ParseMode::Combined {
1362 parent_layer_range, ..
1363 } = &self.mode
1364 {
1365 parent_layer_range.clone()
1366 } else {
1367 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1368 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1369 start..end
1370 }
1371 }
1372}
1373
1374impl ChangedRegion {
1375 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1376 let range_a = &self.range;
1377 let range_b = &other.range;
1378 Ord::cmp(&self.depth, &other.depth)
1379 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1380 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1381 }
1382}
1383
1384impl ChangeRegionSet {
1385 fn start_position(&self) -> ChangeStartPosition {
1386 self.0.first().map_or(
1387 ChangeStartPosition {
1388 depth: usize::MAX,
1389 position: Anchor::MAX,
1390 },
1391 |region| ChangeStartPosition {
1392 depth: region.depth,
1393 position: region.range.start,
1394 },
1395 )
1396 }
1397
1398 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1399 for region in &self.0 {
1400 if region.depth < layer.depth {
1401 continue;
1402 }
1403 if region.depth > layer.depth {
1404 break;
1405 }
1406 if region.range.end.cmp(&layer.range.start, text).is_le() {
1407 continue;
1408 }
1409 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1410 break;
1411 }
1412 return true;
1413 }
1414 false
1415 }
1416
1417 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1418 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1419 self.0.insert(ix, region);
1420 }
1421 }
1422
1423 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1424 let prev_len = self.0.len();
1425 self.0.retain(|region| {
1426 region.depth > summary.max_depth
1427 || (region.depth == summary.max_depth
1428 && region
1429 .range
1430 .end
1431 .cmp(&summary.last_layer_range.start, text)
1432 .is_gt())
1433 });
1434 self.0.len() < prev_len
1435 }
1436}
1437
1438impl Default for SyntaxLayerSummary {
1439 fn default() -> Self {
1440 Self {
1441 max_depth: 0,
1442 min_depth: 0,
1443 range: Anchor::MAX..Anchor::MIN,
1444 last_layer_range: Anchor::MIN..Anchor::MAX,
1445 last_layer_language: None,
1446 contains_unknown_injections: false,
1447 }
1448 }
1449}
1450
1451impl sum_tree::Summary for SyntaxLayerSummary {
1452 type Context = BufferSnapshot;
1453
1454 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1455 if other.max_depth > self.max_depth {
1456 self.max_depth = other.max_depth;
1457 self.range = other.range.clone();
1458 } else {
1459 if self.range == (Anchor::MAX..Anchor::MAX) {
1460 self.range.start = other.range.start;
1461 }
1462 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1463 self.range.end = other.range.end;
1464 }
1465 }
1466 self.last_layer_range = other.last_layer_range.clone();
1467 self.last_layer_language = other.last_layer_language;
1468 self.contains_unknown_injections |= other.contains_unknown_injections;
1469 }
1470}
1471
1472impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1473 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1474 Ord::cmp(&self.depth, &cursor_location.max_depth)
1475 .then_with(|| {
1476 self.range
1477 .start
1478 .cmp(&cursor_location.last_layer_range.start, buffer)
1479 })
1480 .then_with(|| {
1481 cursor_location
1482 .last_layer_range
1483 .end
1484 .cmp(&self.range.end, buffer)
1485 })
1486 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1487 }
1488}
1489
1490impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for ChangeStartPosition {
1491 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1492 Ord::cmp(&self.depth, &cursor_location.max_depth)
1493 .then_with(|| self.position.cmp(&cursor_location.range.end, text))
1494 }
1495}
1496
1497impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1498 for SyntaxLayerPositionBeforeChange
1499{
1500 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1501 if self.change.cmp(cursor_location, buffer).is_le() {
1502 return Ordering::Less;
1503 } else {
1504 self.position.cmp(cursor_location, buffer)
1505 }
1506 }
1507}
1508
1509impl sum_tree::Item for SyntaxLayer {
1510 type Summary = SyntaxLayerSummary;
1511
1512 fn summary(&self) -> Self::Summary {
1513 SyntaxLayerSummary {
1514 min_depth: self.depth,
1515 max_depth: self.depth,
1516 range: self.range.clone(),
1517 last_layer_range: self.range.clone(),
1518 last_layer_language: self.content.language_id(),
1519 contains_unknown_injections: matches!(self.content, SyntaxLayerContent::Pending { .. }),
1520 }
1521 }
1522}
1523
1524impl std::fmt::Debug for SyntaxLayer {
1525 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1526 f.debug_struct("SyntaxLayer")
1527 .field("depth", &self.depth)
1528 .field("range", &self.range)
1529 .field("tree", &self.content.tree())
1530 .finish()
1531 }
1532}
1533
1534impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1535 type I = ByteChunks<'a>;
1536
1537 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1538 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1539 }
1540}
1541
1542impl<'a> Iterator for ByteChunks<'a> {
1543 type Item = &'a [u8];
1544
1545 fn next(&mut self) -> Option<Self::Item> {
1546 self.0.next().map(str::as_bytes)
1547 }
1548}
1549
1550impl QueryCursorHandle {
1551 pub(crate) fn new() -> Self {
1552 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1553 cursor.set_match_limit(64);
1554 QueryCursorHandle(Some(cursor))
1555 }
1556}
1557
1558impl Deref for QueryCursorHandle {
1559 type Target = QueryCursor;
1560
1561 fn deref(&self) -> &Self::Target {
1562 self.0.as_ref().unwrap()
1563 }
1564}
1565
1566impl DerefMut for QueryCursorHandle {
1567 fn deref_mut(&mut self) -> &mut Self::Target {
1568 self.0.as_mut().unwrap()
1569 }
1570}
1571
1572impl Drop for QueryCursorHandle {
1573 fn drop(&mut self) {
1574 let mut cursor = self.0.take().unwrap();
1575 cursor.set_byte_range(0..usize::MAX);
1576 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1577 QUERY_CURSORS.lock().push(cursor)
1578 }
1579}
1580
1581pub(crate) trait ToTreeSitterPoint {
1582 fn to_ts_point(self) -> tree_sitter::Point;
1583 fn from_ts_point(point: tree_sitter::Point) -> Self;
1584}
1585
1586impl ToTreeSitterPoint for Point {
1587 fn to_ts_point(self) -> tree_sitter::Point {
1588 tree_sitter::Point::new(self.row as usize, self.column as usize)
1589 }
1590
1591 fn from_ts_point(point: tree_sitter::Point) -> Self {
1592 Point::new(point.row as u32, point.column as u32)
1593 }
1594}
1595
1596#[cfg(test)]
1597mod tests {
1598 use super::*;
1599 use crate::LanguageConfig;
1600 use rand::rngs::StdRng;
1601 use std::env;
1602 use text::Buffer;
1603 use unindent::Unindent as _;
1604 use util::test::marked_text_ranges;
1605
1606 #[test]
1607 fn test_splice_included_ranges() {
1608 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1609
1610 let new_ranges = splice_included_ranges(
1611 ranges.clone(),
1612 &[54..56, 58..68],
1613 &[ts_range(50..54), ts_range(59..67)],
1614 );
1615 assert_eq!(
1616 new_ranges,
1617 &[
1618 ts_range(20..30),
1619 ts_range(50..54),
1620 ts_range(59..67),
1621 ts_range(80..90),
1622 ]
1623 );
1624
1625 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1626 assert_eq!(
1627 new_ranges,
1628 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1629 );
1630
1631 let new_ranges =
1632 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1633 assert_eq!(
1634 new_ranges,
1635 &[
1636 ts_range(0..2),
1637 ts_range(20..30),
1638 ts_range(50..60),
1639 ts_range(70..75),
1640 ts_range(80..90)
1641 ]
1642 );
1643
1644 let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
1645 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
1646
1647 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1648 tree_sitter::Range {
1649 start_byte: range.start,
1650 start_point: tree_sitter::Point {
1651 row: 0,
1652 column: range.start,
1653 },
1654 end_byte: range.end,
1655 end_point: tree_sitter::Point {
1656 row: 0,
1657 column: range.end,
1658 },
1659 }
1660 }
1661 }
1662
1663 #[gpui::test]
1664 fn test_syntax_map_layers_for_range() {
1665 let registry = Arc::new(LanguageRegistry::test());
1666 let language = Arc::new(rust_lang());
1667 registry.add(language.clone());
1668
1669 let mut buffer = Buffer::new(
1670 0,
1671 0,
1672 r#"
1673 fn a() {
1674 assert_eq!(
1675 b(vec![C {}]),
1676 vec![d.e],
1677 );
1678 println!("{}", f(|_| true));
1679 }
1680 "#
1681 .unindent(),
1682 );
1683
1684 let mut syntax_map = SyntaxMap::new();
1685 syntax_map.set_language_registry(registry.clone());
1686 syntax_map.reparse(language.clone(), &buffer);
1687
1688 assert_layers_for_range(
1689 &syntax_map,
1690 &buffer,
1691 Point::new(2, 0)..Point::new(2, 0),
1692 &[
1693 "...(function_item ... (block (expression_statement (macro_invocation...",
1694 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1695 ],
1696 );
1697 assert_layers_for_range(
1698 &syntax_map,
1699 &buffer,
1700 Point::new(2, 14)..Point::new(2, 16),
1701 &[
1702 "...(function_item ...",
1703 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1704 "...(array_expression (struct_expression ...",
1705 ],
1706 );
1707 assert_layers_for_range(
1708 &syntax_map,
1709 &buffer,
1710 Point::new(3, 14)..Point::new(3, 16),
1711 &[
1712 "...(function_item ...",
1713 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1714 "...(array_expression (field_expression ...",
1715 ],
1716 );
1717 assert_layers_for_range(
1718 &syntax_map,
1719 &buffer,
1720 Point::new(5, 12)..Point::new(5, 16),
1721 &[
1722 "...(function_item ...",
1723 "...(call_expression ... (arguments (closure_expression ...",
1724 ],
1725 );
1726
1727 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1728 let macro_name_range = range_for_text(&buffer, "vec!");
1729 buffer.edit([(macro_name_range, "&")]);
1730 syntax_map.interpolate(&buffer);
1731 syntax_map.reparse(language.clone(), &buffer);
1732
1733 assert_layers_for_range(
1734 &syntax_map,
1735 &buffer,
1736 Point::new(2, 14)..Point::new(2, 16),
1737 &[
1738 "...(function_item ...",
1739 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1740 ],
1741 );
1742
1743 // Put the vec! macro back, adding back the syntactic layer.
1744 buffer.undo();
1745 syntax_map.interpolate(&buffer);
1746 syntax_map.reparse(language.clone(), &buffer);
1747
1748 assert_layers_for_range(
1749 &syntax_map,
1750 &buffer,
1751 Point::new(2, 14)..Point::new(2, 16),
1752 &[
1753 "...(function_item ...",
1754 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1755 "...(array_expression (struct_expression ...",
1756 ],
1757 );
1758 }
1759
1760 #[gpui::test]
1761 fn test_dynamic_language_injection() {
1762 let registry = Arc::new(LanguageRegistry::test());
1763 let markdown = Arc::new(markdown_lang());
1764 registry.add(markdown.clone());
1765 registry.add(Arc::new(rust_lang()));
1766 registry.add(Arc::new(ruby_lang()));
1767
1768 let mut buffer = Buffer::new(
1769 0,
1770 0,
1771 r#"
1772 This is a code block:
1773
1774 ```rs
1775 fn foo() {}
1776 ```
1777 "#
1778 .unindent(),
1779 );
1780
1781 let mut syntax_map = SyntaxMap::new();
1782 syntax_map.set_language_registry(registry.clone());
1783 syntax_map.reparse(markdown.clone(), &buffer);
1784 assert_layers_for_range(
1785 &syntax_map,
1786 &buffer,
1787 Point::new(3, 0)..Point::new(3, 0),
1788 &[
1789 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1790 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
1791 ],
1792 );
1793
1794 // Replace Rust with Ruby in code block.
1795 let macro_name_range = range_for_text(&buffer, "rs");
1796 buffer.edit([(macro_name_range, "ruby")]);
1797 syntax_map.interpolate(&buffer);
1798 syntax_map.reparse(markdown.clone(), &buffer);
1799 assert_layers_for_range(
1800 &syntax_map,
1801 &buffer,
1802 Point::new(3, 0)..Point::new(3, 0),
1803 &[
1804 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1805 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
1806 ],
1807 );
1808
1809 // Replace Ruby with a language that hasn't been loaded yet.
1810 let macro_name_range = range_for_text(&buffer, "ruby");
1811 buffer.edit([(macro_name_range, "html")]);
1812 syntax_map.interpolate(&buffer);
1813 syntax_map.reparse(markdown.clone(), &buffer);
1814 assert_layers_for_range(
1815 &syntax_map,
1816 &buffer,
1817 Point::new(3, 0)..Point::new(3, 0),
1818 &[
1819 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
1820 ],
1821 );
1822 assert!(syntax_map.contains_unknown_injections());
1823
1824 registry.add(Arc::new(html_lang()));
1825 syntax_map.reparse(markdown.clone(), &buffer);
1826 assert_layers_for_range(
1827 &syntax_map,
1828 &buffer,
1829 Point::new(3, 0)..Point::new(3, 0),
1830 &[
1831 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1832 "(fragment (text))",
1833 ],
1834 );
1835 assert!(!syntax_map.contains_unknown_injections());
1836 }
1837
1838 #[gpui::test]
1839 fn test_typing_multiple_new_injections() {
1840 let (buffer, syntax_map) = test_edit_sequence(
1841 "Rust",
1842 &[
1843 "fn a() { dbg }",
1844 "fn a() { dbg«!» }",
1845 "fn a() { dbg!«()» }",
1846 "fn a() { dbg!(«b») }",
1847 "fn a() { dbg!(b«.») }",
1848 "fn a() { dbg!(b.«c») }",
1849 "fn a() { dbg!(b.c«()») }",
1850 "fn a() { dbg!(b.c(«vec»)) }",
1851 "fn a() { dbg!(b.c(vec«!»)) }",
1852 "fn a() { dbg!(b.c(vec!«[]»)) }",
1853 "fn a() { dbg!(b.c(vec![«d»])) }",
1854 "fn a() { dbg!(b.c(vec![d«.»])) }",
1855 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1856 ],
1857 );
1858
1859 assert_capture_ranges(
1860 &syntax_map,
1861 &buffer,
1862 &["field"],
1863 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1864 );
1865 }
1866
1867 #[gpui::test]
1868 fn test_pasting_new_injection_line_between_others() {
1869 let (buffer, syntax_map) = test_edit_sequence(
1870 "Rust",
1871 &[
1872 "
1873 fn a() {
1874 b!(B {});
1875 c!(C {});
1876 d!(D {});
1877 e!(E {});
1878 f!(F {});
1879 g!(G {});
1880 }
1881 ",
1882 "
1883 fn a() {
1884 b!(B {});
1885 c!(C {});
1886 d!(D {});
1887 « h!(H {});
1888 » e!(E {});
1889 f!(F {});
1890 g!(G {});
1891 }
1892 ",
1893 ],
1894 );
1895
1896 assert_capture_ranges(
1897 &syntax_map,
1898 &buffer,
1899 &["struct"],
1900 "
1901 fn a() {
1902 b!(«B {}»);
1903 c!(«C {}»);
1904 d!(«D {}»);
1905 h!(«H {}»);
1906 e!(«E {}»);
1907 f!(«F {}»);
1908 g!(«G {}»);
1909 }
1910 ",
1911 );
1912 }
1913
1914 #[gpui::test]
1915 fn test_joining_injections_with_child_injections() {
1916 let (buffer, syntax_map) = test_edit_sequence(
1917 "Rust",
1918 &[
1919 "
1920 fn a() {
1921 b!(
1922 c![one.two.three],
1923 d![four.five.six],
1924 );
1925 e!(
1926 f![seven.eight],
1927 );
1928 }
1929 ",
1930 "
1931 fn a() {
1932 b!(
1933 c![one.two.three],
1934 d![four.five.six],
1935 ˇ f![seven.eight],
1936 );
1937 }
1938 ",
1939 ],
1940 );
1941
1942 assert_capture_ranges(
1943 &syntax_map,
1944 &buffer,
1945 &["field"],
1946 "
1947 fn a() {
1948 b!(
1949 c![one.«two».«three»],
1950 d![four.«five».«six»],
1951 f![seven.«eight»],
1952 );
1953 }
1954 ",
1955 );
1956 }
1957
1958 #[gpui::test]
1959 fn test_editing_edges_of_injection() {
1960 test_edit_sequence(
1961 "Rust",
1962 &[
1963 "
1964 fn a() {
1965 b!(c!())
1966 }
1967 ",
1968 "
1969 fn a() {
1970 «d»!(c!())
1971 }
1972 ",
1973 "
1974 fn a() {
1975 «e»d!(c!())
1976 }
1977 ",
1978 "
1979 fn a() {
1980 ed!«[»c!()«]»
1981 }
1982 ",
1983 ],
1984 );
1985 }
1986
1987 #[gpui::test]
1988 fn test_edits_preceding_and_intersecting_injection() {
1989 test_edit_sequence(
1990 "Rust",
1991 &[
1992 //
1993 "const aaaaaaaaaaaa: B = c!(d(e.f));",
1994 "const aˇa: B = c!(d(eˇ));",
1995 ],
1996 );
1997 }
1998
1999 #[gpui::test]
2000 fn test_non_local_changes_create_injections() {
2001 test_edit_sequence(
2002 "Rust",
2003 &[
2004 "
2005 // a! {
2006 static B: C = d;
2007 // }
2008 ",
2009 "
2010 ˇa! {
2011 static B: C = d;
2012 ˇ}
2013 ",
2014 ],
2015 );
2016 }
2017
2018 #[gpui::test]
2019 fn test_creating_many_injections_in_one_edit() {
2020 test_edit_sequence(
2021 "Rust",
2022 &[
2023 "
2024 fn a() {
2025 one(Two::three(3));
2026 four(Five::six(6));
2027 seven(Eight::nine(9));
2028 }
2029 ",
2030 "
2031 fn a() {
2032 one«!»(Two::three(3));
2033 four«!»(Five::six(6));
2034 seven«!»(Eight::nine(9));
2035 }
2036 ",
2037 "
2038 fn a() {
2039 one!(Two::three«!»(3));
2040 four!(Five::six«!»(6));
2041 seven!(Eight::nine«!»(9));
2042 }
2043 ",
2044 ],
2045 );
2046 }
2047
2048 #[gpui::test]
2049 fn test_editing_across_injection_boundary() {
2050 test_edit_sequence(
2051 "Rust",
2052 &[
2053 "
2054 fn one() {
2055 two();
2056 three!(
2057 three.four,
2058 five.six,
2059 );
2060 }
2061 ",
2062 "
2063 fn one() {
2064 two();
2065 th«irty_five![»
2066 three.four,
2067 five.six,
2068 « seven.eight,
2069 ];»
2070 }
2071 ",
2072 ],
2073 );
2074 }
2075
2076 #[gpui::test]
2077 fn test_removing_injection_by_replacing_across_boundary() {
2078 test_edit_sequence(
2079 "Rust",
2080 &[
2081 "
2082 fn one() {
2083 two!(
2084 three.four,
2085 );
2086 }
2087 ",
2088 "
2089 fn one() {
2090 t«en
2091 .eleven(
2092 twelve,
2093 »
2094 three.four,
2095 );
2096 }
2097 ",
2098 ],
2099 );
2100 }
2101
2102 #[gpui::test]
2103 fn test_combined_injections() {
2104 let (buffer, syntax_map) = test_edit_sequence(
2105 "ERB",
2106 &[
2107 "
2108 <body>
2109 <% if @one %>
2110 <div class=one>
2111 <% else %>
2112 <div class=two>
2113 <% end %>
2114 </div>
2115 </body>
2116 ",
2117 "
2118 <body>
2119 <% if @one %>
2120 <div class=one>
2121 ˇ else ˇ
2122 <div class=two>
2123 <% end %>
2124 </div>
2125 </body>
2126 ",
2127 "
2128 <body>
2129 <% if @one «;» end %>
2130 </div>
2131 </body>
2132 ",
2133 ],
2134 );
2135
2136 assert_capture_ranges(
2137 &syntax_map,
2138 &buffer,
2139 &["tag", "ivar"],
2140 "
2141 <«body»>
2142 <% if «@one» ; end %>
2143 </«div»>
2144 </«body»>
2145 ",
2146 );
2147 }
2148
2149 #[gpui::test]
2150 fn test_combined_injections_empty_ranges() {
2151 test_edit_sequence(
2152 "ERB",
2153 &[
2154 "
2155 <% if @one %>
2156 <% else %>
2157 <% end %>
2158 ",
2159 "
2160 <% if @one %>
2161 ˇ<% end %>
2162 ",
2163 ],
2164 );
2165 }
2166
2167 #[gpui::test]
2168 fn test_combined_injections_edit_edges_of_ranges() {
2169 let (buffer, syntax_map) = test_edit_sequence(
2170 "ERB",
2171 &[
2172 "
2173 <%= one @two %>
2174 <%= three @four %>
2175 ",
2176 "
2177 <%= one @two %ˇ
2178 <%= three @four %>
2179 ",
2180 "
2181 <%= one @two %«>»
2182 <%= three @four %>
2183 ",
2184 ],
2185 );
2186
2187 assert_capture_ranges(
2188 &syntax_map,
2189 &buffer,
2190 &["tag", "ivar"],
2191 "
2192 <%= one «@two» %>
2193 <%= three «@four» %>
2194 ",
2195 );
2196 }
2197
2198 #[gpui::test]
2199 fn test_combined_injections_splitting_some_injections() {
2200 let (_buffer, _syntax_map) = test_edit_sequence(
2201 "ERB",
2202 &[
2203 r#"
2204 <%A if b(:c) %>
2205 d
2206 <% end %>
2207 eee
2208 <% f %>
2209 "#,
2210 r#"
2211 <%« AAAAAAA %>
2212 hhhhhhh
2213 <%=» if b(:c) %>
2214 d
2215 <% end %>
2216 eee
2217 <% f %>
2218 "#,
2219 ],
2220 );
2221 }
2222
2223 #[gpui::test]
2224 fn test_combined_injections_inside_injections() {
2225 let (_buffer, _syntax_map) = test_edit_sequence(
2226 "Markdown",
2227 &[
2228 r#"
2229 here is some ERB code:
2230
2231 ```erb
2232 <ul>
2233 <% people.each do |person| %>
2234 <li><%= person.name %></li>
2235 <% end %>
2236 </ul>
2237 ```
2238 "#,
2239 r#"
2240 here is some ERB code:
2241
2242 ```erb
2243 <ul>
2244 <% people«2».each do |person| %>
2245 <li><%= person.name %></li>
2246 <% end %>
2247 </ul>
2248 ```
2249 "#,
2250 ],
2251 );
2252 }
2253
2254 #[gpui::test(iterations = 50)]
2255 fn test_random_syntax_map_edits(mut rng: StdRng) {
2256 let operations = env::var("OPERATIONS")
2257 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2258 .unwrap_or(10);
2259
2260 let text = r#"
2261 fn test_something() {
2262 let vec = vec![5, 1, 3, 8];
2263 assert_eq!(
2264 vec
2265 .into_iter()
2266 .map(|i| i * 2)
2267 .collect::<Vec<usize>>(),
2268 vec![
2269 5 * 2, 1 * 2, 3 * 2, 8 * 2
2270 ],
2271 );
2272 }
2273 "#
2274 .unindent()
2275 .repeat(2);
2276
2277 let registry = Arc::new(LanguageRegistry::test());
2278 let language = Arc::new(rust_lang());
2279 registry.add(language.clone());
2280 let mut buffer = Buffer::new(0, 0, text);
2281
2282 let mut syntax_map = SyntaxMap::new();
2283 syntax_map.set_language_registry(registry.clone());
2284 syntax_map.reparse(language.clone(), &buffer);
2285
2286 let mut reference_syntax_map = SyntaxMap::new();
2287 reference_syntax_map.set_language_registry(registry.clone());
2288
2289 log::info!("initial text:\n{}", buffer.text());
2290
2291 for _ in 0..operations {
2292 let prev_buffer = buffer.snapshot();
2293 let prev_syntax_map = syntax_map.snapshot();
2294
2295 buffer.randomly_edit(&mut rng, 3);
2296 log::info!("text:\n{}", buffer.text());
2297
2298 syntax_map.interpolate(&buffer);
2299 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2300
2301 syntax_map.reparse(language.clone(), &buffer);
2302
2303 reference_syntax_map.clear();
2304 reference_syntax_map.reparse(language.clone(), &buffer);
2305 }
2306
2307 for i in 0..operations {
2308 let i = operations - i - 1;
2309 buffer.undo();
2310 log::info!("undoing operation {}", i);
2311 log::info!("text:\n{}", buffer.text());
2312
2313 syntax_map.interpolate(&buffer);
2314 syntax_map.reparse(language.clone(), &buffer);
2315
2316 reference_syntax_map.clear();
2317 reference_syntax_map.reparse(language.clone(), &buffer);
2318 assert_eq!(
2319 syntax_map.layers(&buffer).len(),
2320 reference_syntax_map.layers(&buffer).len(),
2321 "wrong number of layers after undoing edit {i}"
2322 );
2323 }
2324
2325 let layers = syntax_map.layers(&buffer);
2326 let reference_layers = reference_syntax_map.layers(&buffer);
2327 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2328 {
2329 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2330 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2331 }
2332 }
2333
2334 #[gpui::test(iterations = 50)]
2335 fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) {
2336 let operations = env::var("OPERATIONS")
2337 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2338 .unwrap_or(10);
2339
2340 let text = r#"
2341 <div id="main">
2342 <% if one?(:two) %>
2343 <p class="three" four>
2344 <%= yield :five %>
2345 </p>
2346 <% elsif Six.seven(8) %>
2347 <p id="three" four>
2348 <%= yield :five %>
2349 </p>
2350 <% else %>
2351 <span>Ok</span>
2352 <% end %>
2353 </div>
2354 "#
2355 .unindent()
2356 .repeat(8);
2357
2358 let registry = Arc::new(LanguageRegistry::test());
2359 let language = Arc::new(erb_lang());
2360 registry.add(language.clone());
2361 registry.add(Arc::new(ruby_lang()));
2362 registry.add(Arc::new(html_lang()));
2363 let mut buffer = Buffer::new(0, 0, text);
2364
2365 let mut syntax_map = SyntaxMap::new();
2366 syntax_map.set_language_registry(registry.clone());
2367 syntax_map.reparse(language.clone(), &buffer);
2368
2369 let mut reference_syntax_map = SyntaxMap::new();
2370 reference_syntax_map.set_language_registry(registry.clone());
2371
2372 log::info!("initial text:\n{}", buffer.text());
2373
2374 for _ in 0..operations {
2375 let prev_buffer = buffer.snapshot();
2376 let prev_syntax_map = syntax_map.snapshot();
2377
2378 buffer.randomly_edit(&mut rng, 3);
2379 log::info!("text:\n{}", buffer.text());
2380
2381 syntax_map.interpolate(&buffer);
2382 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2383
2384 syntax_map.reparse(language.clone(), &buffer);
2385
2386 reference_syntax_map.clear();
2387 reference_syntax_map.reparse(language.clone(), &buffer);
2388 }
2389
2390 for i in 0..operations {
2391 let i = operations - i - 1;
2392 buffer.undo();
2393 log::info!("undoing operation {}", i);
2394 log::info!("text:\n{}", buffer.text());
2395
2396 syntax_map.interpolate(&buffer);
2397 syntax_map.reparse(language.clone(), &buffer);
2398
2399 reference_syntax_map.clear();
2400 reference_syntax_map.reparse(language.clone(), &buffer);
2401 assert_eq!(
2402 syntax_map.layers(&buffer).len(),
2403 reference_syntax_map.layers(&buffer).len(),
2404 "wrong number of layers after undoing edit {i}"
2405 );
2406 }
2407
2408 let layers = syntax_map.layers(&buffer);
2409 let reference_layers = reference_syntax_map.layers(&buffer);
2410 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2411 {
2412 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2413 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2414 }
2415 }
2416
2417 fn check_interpolation(
2418 old_syntax_map: &SyntaxSnapshot,
2419 new_syntax_map: &SyntaxSnapshot,
2420 old_buffer: &BufferSnapshot,
2421 new_buffer: &BufferSnapshot,
2422 ) {
2423 let edits = new_buffer
2424 .edits_since::<usize>(&old_buffer.version())
2425 .collect::<Vec<_>>();
2426
2427 for (old_layer, new_layer) in old_syntax_map
2428 .layers
2429 .iter()
2430 .zip(new_syntax_map.layers.iter())
2431 {
2432 assert_eq!(old_layer.range, new_layer.range);
2433 let Some(old_tree) = old_layer.content.tree() else { continue };
2434 let Some(new_tree) = new_layer.content.tree() else { continue };
2435 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
2436 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
2437 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
2438 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
2439 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
2440 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
2441 check_node_edits(
2442 old_layer.depth,
2443 &old_layer.range,
2444 old_node,
2445 new_node,
2446 old_buffer,
2447 new_buffer,
2448 &edits,
2449 );
2450 }
2451
2452 fn check_node_edits(
2453 depth: usize,
2454 range: &Range<Anchor>,
2455 old_node: Node,
2456 new_node: Node,
2457 old_buffer: &BufferSnapshot,
2458 new_buffer: &BufferSnapshot,
2459 edits: &[text::Edit<usize>],
2460 ) {
2461 assert_eq!(old_node.kind(), new_node.kind());
2462
2463 let old_range = old_node.byte_range();
2464 let new_range = new_node.byte_range();
2465
2466 let is_edited = edits
2467 .iter()
2468 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
2469 if is_edited {
2470 assert!(
2471 new_node.has_changes(),
2472 concat!(
2473 "failed to mark node as edited.\n",
2474 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2475 "node kind: {}, old node range: {:?}, new node range: {:?}",
2476 ),
2477 depth,
2478 range.to_offset(old_buffer),
2479 range.to_offset(new_buffer),
2480 new_node.kind(),
2481 old_range,
2482 new_range,
2483 );
2484 }
2485
2486 if !new_node.has_changes() {
2487 assert_eq!(
2488 old_buffer
2489 .text_for_range(old_range.clone())
2490 .collect::<String>(),
2491 new_buffer
2492 .text_for_range(new_range.clone())
2493 .collect::<String>(),
2494 concat!(
2495 "mismatched text for node\n",
2496 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2497 "node kind: {}, old node range:{:?}, new node range:{:?}",
2498 ),
2499 depth,
2500 range.to_offset(old_buffer),
2501 range.to_offset(new_buffer),
2502 new_node.kind(),
2503 old_range,
2504 new_range,
2505 );
2506 }
2507
2508 for i in 0..new_node.child_count() {
2509 check_node_edits(
2510 depth,
2511 range,
2512 old_node.child(i).unwrap(),
2513 new_node.child(i).unwrap(),
2514 old_buffer,
2515 new_buffer,
2516 edits,
2517 )
2518 }
2519 }
2520 }
2521
2522 fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
2523 let registry = Arc::new(LanguageRegistry::test());
2524 registry.add(Arc::new(rust_lang()));
2525 registry.add(Arc::new(ruby_lang()));
2526 registry.add(Arc::new(html_lang()));
2527 registry.add(Arc::new(erb_lang()));
2528 registry.add(Arc::new(markdown_lang()));
2529 let language = registry
2530 .language_for_name(language_name)
2531 .now_or_never()
2532 .unwrap()
2533 .unwrap();
2534 let mut buffer = Buffer::new(0, 0, Default::default());
2535
2536 let mut mutated_syntax_map = SyntaxMap::new();
2537 mutated_syntax_map.set_language_registry(registry.clone());
2538 mutated_syntax_map.reparse(language.clone(), &buffer);
2539
2540 for (i, marked_string) in steps.into_iter().enumerate() {
2541 buffer.edit_via_marked_text(&marked_string.unindent());
2542
2543 // Reparse the syntax map
2544 mutated_syntax_map.interpolate(&buffer);
2545 mutated_syntax_map.reparse(language.clone(), &buffer);
2546
2547 // Create a second syntax map from scratch
2548 let mut reference_syntax_map = SyntaxMap::new();
2549 reference_syntax_map.set_language_registry(registry.clone());
2550 reference_syntax_map.reparse(language.clone(), &buffer);
2551
2552 // Compare the mutated syntax map to the new syntax map
2553 let mutated_layers = mutated_syntax_map.layers(&buffer);
2554 let reference_layers = reference_syntax_map.layers(&buffer);
2555 assert_eq!(
2556 mutated_layers.len(),
2557 reference_layers.len(),
2558 "wrong number of layers at step {i}"
2559 );
2560 for (edited_layer, reference_layer) in
2561 mutated_layers.into_iter().zip(reference_layers.into_iter())
2562 {
2563 assert_eq!(
2564 edited_layer.node.to_sexp(),
2565 reference_layer.node.to_sexp(),
2566 "different layer at step {i}"
2567 );
2568 assert_eq!(
2569 edited_layer.node.range(),
2570 reference_layer.node.range(),
2571 "different layer at step {i}"
2572 );
2573 }
2574 }
2575
2576 (buffer, mutated_syntax_map)
2577 }
2578
2579 fn html_lang() -> Language {
2580 Language::new(
2581 LanguageConfig {
2582 name: "HTML".into(),
2583 path_suffixes: vec!["html".to_string()],
2584 ..Default::default()
2585 },
2586 Some(tree_sitter_html::language()),
2587 )
2588 .with_highlights_query(
2589 r#"
2590 (tag_name) @tag
2591 (erroneous_end_tag_name) @tag
2592 (attribute_name) @property
2593 "#,
2594 )
2595 .unwrap()
2596 }
2597
2598 fn ruby_lang() -> Language {
2599 Language::new(
2600 LanguageConfig {
2601 name: "Ruby".into(),
2602 path_suffixes: vec!["rb".to_string()],
2603 ..Default::default()
2604 },
2605 Some(tree_sitter_ruby::language()),
2606 )
2607 .with_highlights_query(
2608 r#"
2609 ["if" "do" "else" "end"] @keyword
2610 (instance_variable) @ivar
2611 "#,
2612 )
2613 .unwrap()
2614 }
2615
2616 fn erb_lang() -> Language {
2617 Language::new(
2618 LanguageConfig {
2619 name: "ERB".into(),
2620 path_suffixes: vec!["erb".to_string()],
2621 ..Default::default()
2622 },
2623 Some(tree_sitter_embedded_template::language()),
2624 )
2625 .with_highlights_query(
2626 r#"
2627 ["<%" "%>"] @keyword
2628 "#,
2629 )
2630 .unwrap()
2631 .with_injection_query(
2632 r#"
2633 ((code) @content
2634 (#set! "language" "ruby")
2635 (#set! "combined"))
2636
2637 ((content) @content
2638 (#set! "language" "html")
2639 (#set! "combined"))
2640 "#,
2641 )
2642 .unwrap()
2643 }
2644
2645 fn rust_lang() -> Language {
2646 Language::new(
2647 LanguageConfig {
2648 name: "Rust".into(),
2649 path_suffixes: vec!["rs".to_string()],
2650 ..Default::default()
2651 },
2652 Some(tree_sitter_rust::language()),
2653 )
2654 .with_highlights_query(
2655 r#"
2656 (field_identifier) @field
2657 (struct_expression) @struct
2658 "#,
2659 )
2660 .unwrap()
2661 .with_injection_query(
2662 r#"
2663 (macro_invocation
2664 (token_tree) @content
2665 (#set! "language" "rust"))
2666 "#,
2667 )
2668 .unwrap()
2669 }
2670
2671 fn markdown_lang() -> Language {
2672 Language::new(
2673 LanguageConfig {
2674 name: "Markdown".into(),
2675 path_suffixes: vec!["md".into()],
2676 ..Default::default()
2677 },
2678 Some(tree_sitter_markdown::language()),
2679 )
2680 .with_injection_query(
2681 r#"
2682 (fenced_code_block
2683 (info_string
2684 (language) @language)
2685 (code_fence_content) @content)
2686 "#,
2687 )
2688 .unwrap()
2689 }
2690
2691 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2692 let start = buffer.as_rope().to_string().find(text).unwrap();
2693 start..start + text.len()
2694 }
2695
2696 fn assert_layers_for_range(
2697 syntax_map: &SyntaxMap,
2698 buffer: &BufferSnapshot,
2699 range: Range<Point>,
2700 expected_layers: &[&str],
2701 ) {
2702 let layers = syntax_map
2703 .layers_for_range(range, &buffer)
2704 .collect::<Vec<_>>();
2705 assert_eq!(
2706 layers.len(),
2707 expected_layers.len(),
2708 "wrong number of layers"
2709 );
2710 for (i, (SyntaxLayerInfo { node, .. }, expected_s_exp)) in
2711 layers.iter().zip(expected_layers.iter()).enumerate()
2712 {
2713 let actual_s_exp = node.to_sexp();
2714 assert!(
2715 string_contains_sequence(
2716 &actual_s_exp,
2717 &expected_s_exp.split("...").collect::<Vec<_>>()
2718 ),
2719 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2720 );
2721 }
2722 }
2723
2724 fn assert_capture_ranges(
2725 syntax_map: &SyntaxMap,
2726 buffer: &BufferSnapshot,
2727 highlight_query_capture_names: &[&str],
2728 marked_string: &str,
2729 ) {
2730 let mut actual_ranges = Vec::<Range<usize>>::new();
2731 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2732 grammar.highlights_query.as_ref()
2733 });
2734 let queries = captures
2735 .grammars()
2736 .iter()
2737 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2738 .collect::<Vec<_>>();
2739 for capture in captures {
2740 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2741 if highlight_query_capture_names.contains(&name.as_str()) {
2742 actual_ranges.push(capture.node.byte_range());
2743 }
2744 }
2745
2746 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2747 assert_eq!(text, buffer.text());
2748 assert_eq!(actual_ranges, expected_ranges);
2749 }
2750
2751 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2752 let mut last_part_end = 0;
2753 for part in parts {
2754 if let Some(start_ix) = text[last_part_end..].find(part) {
2755 last_part_end = start_ix + part.len();
2756 } else {
2757 return false;
2758 }
2759 }
2760 true
2761 }
2762}