1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use futures::FutureExt;
4use lazy_static::lazy_static;
5use parking_lot::Mutex;
6use std::{
7 borrow::Cow,
8 cell::RefCell,
9 cmp::{self, Ordering, Reverse},
10 collections::BinaryHeap,
11 iter,
12 ops::{Deref, DerefMut, Range},
13 sync::Arc,
14};
15use sum_tree::{Bias, SeekTarget, SumTree};
16use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
17use tree_sitter::{
18 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
19};
20
21thread_local! {
22 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
23}
24
25lazy_static! {
26 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
27}
28
29#[derive(Default)]
30pub struct SyntaxMap {
31 snapshot: SyntaxSnapshot,
32 language_registry: Option<Arc<LanguageRegistry>>,
33}
34
35#[derive(Clone, Default)]
36pub struct SyntaxSnapshot {
37 layers: SumTree<SyntaxLayer>,
38 parsed_version: clock::Global,
39 interpolated_version: clock::Global,
40 language_registry_version: usize,
41}
42
43#[derive(Default)]
44pub struct SyntaxMapCaptures<'a> {
45 layers: Vec<SyntaxMapCapturesLayer<'a>>,
46 active_layer_count: usize,
47 grammars: Vec<&'a Grammar>,
48}
49
50#[derive(Default)]
51pub struct SyntaxMapMatches<'a> {
52 layers: Vec<SyntaxMapMatchesLayer<'a>>,
53 active_layer_count: usize,
54 grammars: Vec<&'a Grammar>,
55}
56
57#[derive(Debug)]
58pub struct SyntaxMapCapture<'a> {
59 pub depth: usize,
60 pub node: Node<'a>,
61 pub index: u32,
62 pub grammar_index: usize,
63}
64
65#[derive(Debug)]
66pub struct SyntaxMapMatch<'a> {
67 pub depth: usize,
68 pub pattern_index: usize,
69 pub captures: &'a [QueryCapture<'a>],
70 pub grammar_index: usize,
71}
72
73struct SyntaxMapCapturesLayer<'a> {
74 depth: usize,
75 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
76 next_capture: Option<QueryCapture<'a>>,
77 grammar_index: usize,
78 _query_cursor: QueryCursorHandle,
79}
80
81struct SyntaxMapMatchesLayer<'a> {
82 depth: usize,
83 next_pattern_index: usize,
84 next_captures: Vec<QueryCapture<'a>>,
85 has_next: bool,
86 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
87 grammar_index: usize,
88 _query_cursor: QueryCursorHandle,
89}
90
91#[derive(Clone)]
92struct SyntaxLayer {
93 depth: usize,
94 range: Range<Anchor>,
95 content: SyntaxLayerContent,
96}
97
98#[derive(Clone)]
99enum SyntaxLayerContent {
100 Parsed {
101 tree: tree_sitter::Tree,
102 language: Arc<Language>,
103 },
104 Pending {
105 language_name: Arc<str>,
106 },
107}
108
109impl SyntaxLayerContent {
110 fn language_id(&self) -> Option<usize> {
111 match self {
112 SyntaxLayerContent::Parsed { language, .. } => language.id(),
113 SyntaxLayerContent::Pending { .. } => None,
114 }
115 }
116
117 fn tree(&self) -> Option<&Tree> {
118 match self {
119 SyntaxLayerContent::Parsed { tree, .. } => Some(tree),
120 SyntaxLayerContent::Pending { .. } => None,
121 }
122 }
123}
124
125#[derive(Debug)]
126pub struct SyntaxLayerInfo<'a> {
127 pub depth: usize,
128 pub node: Node<'a>,
129 pub language: &'a Arc<Language>,
130}
131
132#[derive(Debug, Clone)]
133struct SyntaxLayerSummary {
134 min_depth: usize,
135 max_depth: usize,
136 range: Range<Anchor>,
137 last_layer_range: Range<Anchor>,
138 last_layer_language: Option<usize>,
139 contains_unknown_injections: bool,
140}
141
142#[derive(Clone, Debug)]
143struct SyntaxLayerPosition {
144 depth: usize,
145 range: Range<Anchor>,
146 language: Option<usize>,
147}
148
149#[derive(Clone, Debug)]
150struct ChangeStartPosition {
151 depth: usize,
152 position: Anchor,
153}
154
155#[derive(Clone, Debug)]
156struct SyntaxLayerPositionBeforeChange {
157 position: SyntaxLayerPosition,
158 change: ChangeStartPosition,
159}
160
161struct ParseStep {
162 depth: usize,
163 language: ParseStepLanguage,
164 range: Range<Anchor>,
165 included_ranges: Vec<tree_sitter::Range>,
166 mode: ParseMode,
167}
168
169#[derive(Debug)]
170enum ParseStepLanguage {
171 Loaded { language: Arc<Language> },
172 Pending { name: Arc<str> },
173}
174
175impl ParseStepLanguage {
176 fn id(&self) -> Option<usize> {
177 match self {
178 ParseStepLanguage::Loaded { language } => language.id(),
179 ParseStepLanguage::Pending { .. } => None,
180 }
181 }
182}
183
184enum ParseMode {
185 Single,
186 Combined {
187 parent_layer_range: Range<usize>,
188 parent_layer_changed_ranges: Vec<Range<usize>>,
189 },
190}
191
192#[derive(Debug, PartialEq, Eq)]
193struct ChangedRegion {
194 depth: usize,
195 range: Range<Anchor>,
196}
197
198#[derive(Default)]
199struct ChangeRegionSet(Vec<ChangedRegion>);
200
201struct TextProvider<'a>(&'a Rope);
202
203struct ByteChunks<'a>(text::Chunks<'a>);
204
205struct QueryCursorHandle(Option<QueryCursor>);
206
207impl SyntaxMap {
208 pub fn new() -> Self {
209 Self::default()
210 }
211
212 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
213 self.language_registry = Some(registry);
214 }
215
216 pub fn snapshot(&self) -> SyntaxSnapshot {
217 self.snapshot.clone()
218 }
219
220 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
221 self.language_registry.clone()
222 }
223
224 pub fn interpolate(&mut self, text: &BufferSnapshot) {
225 self.snapshot.interpolate(text);
226 }
227
228 #[cfg(test)]
229 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
230 self.snapshot
231 .reparse(text, self.language_registry.clone(), language);
232 }
233
234 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot) {
235 self.snapshot = snapshot;
236 }
237
238 pub fn clear(&mut self) {
239 self.snapshot = SyntaxSnapshot::default();
240 }
241}
242
243impl SyntaxSnapshot {
244 pub fn is_empty(&self) -> bool {
245 self.layers.is_empty()
246 }
247
248 fn interpolate(&mut self, text: &BufferSnapshot) {
249 let edits = text
250 .anchored_edits_since::<(usize, Point)>(&self.interpolated_version)
251 .collect::<Vec<_>>();
252 self.interpolated_version = text.version().clone();
253
254 if edits.is_empty() {
255 return;
256 }
257
258 let mut layers = SumTree::new();
259 let mut first_edit_ix_for_depth = 0;
260 let mut prev_depth = 0;
261 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
262 cursor.next(text);
263
264 'outer: loop {
265 let depth = cursor.end(text).max_depth;
266 if depth > prev_depth {
267 first_edit_ix_for_depth = 0;
268 prev_depth = depth;
269 }
270
271 // Preserve any layers at this depth that precede the first edit.
272 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
273 let target = ChangeStartPosition {
274 depth,
275 position: edit_range.start,
276 };
277 if target.cmp(&cursor.start(), text).is_gt() {
278 let slice = cursor.slice(&target, Bias::Left, text);
279 layers.push_tree(slice, text);
280 }
281 }
282 // If this layer follows all of the edits, then preserve it and any
283 // subsequent layers at this same depth.
284 else if cursor.item().is_some() {
285 let slice = cursor.slice(
286 &SyntaxLayerPosition {
287 depth: depth + 1,
288 range: Anchor::MIN..Anchor::MAX,
289 language: None,
290 },
291 Bias::Left,
292 text,
293 );
294 layers.push_tree(slice, text);
295 continue;
296 };
297
298 let Some(layer) = cursor.item() else { break };
299 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
300
301 // Ignore edits that end before the start of this layer, and don't consider them
302 // for any subsequent layers at this same depth.
303 loop {
304 let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) else { continue 'outer };
305 if edit_range.end.cmp(&layer.range.start, text).is_le() {
306 first_edit_ix_for_depth += 1;
307 } else {
308 break;
309 }
310 }
311
312 let mut layer = layer.clone();
313 if let SyntaxLayerContent::Parsed { tree, .. } = &mut layer.content {
314 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
315 // Ignore any edits that follow this layer.
316 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
317 break;
318 }
319
320 // Apply any edits that intersect this layer to the layer's syntax tree.
321 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
322 tree_sitter::InputEdit {
323 start_byte: edit.new.start.0 - start_byte,
324 old_end_byte: edit.new.start.0 - start_byte
325 + (edit.old.end.0 - edit.old.start.0),
326 new_end_byte: edit.new.end.0 - start_byte,
327 start_position: (edit.new.start.1 - start_point).to_ts_point(),
328 old_end_position: (edit.new.start.1 - start_point
329 + (edit.old.end.1 - edit.old.start.1))
330 .to_ts_point(),
331 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
332 }
333 } else {
334 let node = tree.root_node();
335 tree_sitter::InputEdit {
336 start_byte: 0,
337 old_end_byte: node.end_byte(),
338 new_end_byte: 0,
339 start_position: Default::default(),
340 old_end_position: node.end_position(),
341 new_end_position: Default::default(),
342 }
343 };
344
345 tree.edit(&tree_edit);
346 }
347
348 debug_assert!(
349 tree.root_node().end_byte() <= text.len(),
350 "tree's size {}, is larger than text size {}",
351 tree.root_node().end_byte(),
352 text.len(),
353 );
354 }
355
356 layers.push(layer, text);
357 cursor.next(text);
358 }
359
360 layers.push_tree(cursor.suffix(&text), &text);
361 drop(cursor);
362 self.layers = layers;
363 }
364
365 pub fn reparse(
366 &mut self,
367 text: &BufferSnapshot,
368 registry: Option<Arc<LanguageRegistry>>,
369 root_language: Arc<Language>,
370 ) {
371 let edit_ranges = text
372 .edits_since::<usize>(&self.parsed_version)
373 .map(|edit| edit.new)
374 .collect::<Vec<_>>();
375 self.reparse_with_ranges(text, root_language.clone(), edit_ranges, registry.as_ref());
376
377 if let Some(registry) = registry {
378 if registry.version() != self.language_registry_version {
379 let mut resolved_injection_ranges = Vec::new();
380 let mut cursor = self
381 .layers
382 .filter::<_, ()>(|summary| summary.contains_unknown_injections);
383 cursor.next(text);
384 while let Some(layer) = cursor.item() {
385 let SyntaxLayerContent::Pending { language_name } = &layer.content else { unreachable!() };
386 if registry
387 .language_for_name_or_extension(language_name)
388 .now_or_never()
389 .and_then(|language| language.ok())
390 .is_some()
391 {
392 resolved_injection_ranges.push(layer.range.to_offset(text));
393 }
394
395 cursor.next(text);
396 }
397 drop(cursor);
398
399 if !resolved_injection_ranges.is_empty() {
400 self.reparse_with_ranges(
401 text,
402 root_language,
403 resolved_injection_ranges,
404 Some(®istry),
405 );
406 }
407 self.language_registry_version = registry.version();
408 }
409 }
410 }
411
412 fn reparse_with_ranges(
413 &mut self,
414 text: &BufferSnapshot,
415 root_language: Arc<Language>,
416 invalidated_ranges: Vec<Range<usize>>,
417 registry: Option<&Arc<LanguageRegistry>>,
418 ) {
419 let max_depth = self.layers.summary().max_depth;
420 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
421 cursor.next(&text);
422 let mut layers = SumTree::new();
423
424 let mut changed_regions = ChangeRegionSet::default();
425 let mut queue = BinaryHeap::new();
426 let mut combined_injection_ranges = HashMap::default();
427 queue.push(ParseStep {
428 depth: 0,
429 language: ParseStepLanguage::Loaded {
430 language: root_language,
431 },
432 included_ranges: vec![tree_sitter::Range {
433 start_byte: 0,
434 end_byte: text.len(),
435 start_point: Point::zero().to_ts_point(),
436 end_point: text.max_point().to_ts_point(),
437 }],
438 range: Anchor::MIN..Anchor::MAX,
439 mode: ParseMode::Single,
440 });
441
442 loop {
443 let step = queue.pop();
444 let position = if let Some(step) = &step {
445 SyntaxLayerPosition {
446 depth: step.depth,
447 range: step.range.clone(),
448 language: step.language.id(),
449 }
450 } else {
451 SyntaxLayerPosition {
452 depth: max_depth + 1,
453 range: Anchor::MAX..Anchor::MAX,
454 language: None,
455 }
456 };
457
458 let mut done = cursor.item().is_none();
459 while !done && position.cmp(&cursor.end(text), &text).is_gt() {
460 done = true;
461
462 let bounded_position = SyntaxLayerPositionBeforeChange {
463 position: position.clone(),
464 change: changed_regions.start_position(),
465 };
466 if bounded_position.cmp(&cursor.start(), &text).is_gt() {
467 let slice = cursor.slice(&bounded_position, Bias::Left, text);
468 if !slice.is_empty() {
469 layers.push_tree(slice, &text);
470 if changed_regions.prune(cursor.end(text), text) {
471 done = false;
472 }
473 }
474 }
475
476 while position.cmp(&cursor.end(text), text).is_gt() {
477 let Some(layer) = cursor.item() else { break };
478
479 if changed_regions.intersects(&layer, text) {
480 changed_regions.insert(
481 ChangedRegion {
482 depth: layer.depth + 1,
483 range: layer.range.clone(),
484 },
485 text,
486 );
487 } else {
488 layers.push(layer.clone(), text);
489 }
490
491 cursor.next(text);
492 if changed_regions.prune(cursor.end(text), text) {
493 done = false;
494 }
495 }
496 }
497
498 let Some(step) = step else { break };
499 let (step_start_byte, step_start_point) =
500 step.range.start.summary::<(usize, Point)>(text);
501 let step_end_byte = step.range.end.to_offset(text);
502
503 let mut old_layer = cursor.item();
504 if let Some(layer) = old_layer {
505 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
506 && layer.content.language_id() == step.language.id()
507 {
508 cursor.next(&text);
509 } else {
510 old_layer = None;
511 }
512 }
513
514 let content = match step.language {
515 ParseStepLanguage::Loaded { language } => {
516 let Some(grammar) = language.grammar() else { continue };
517 let tree;
518 let changed_ranges;
519
520 let mut included_ranges = step.included_ranges;
521 for range in &mut included_ranges {
522 range.start_byte -= step_start_byte;
523 range.end_byte -= step_start_byte;
524 range.start_point = (Point::from_ts_point(range.start_point)
525 - step_start_point)
526 .to_ts_point();
527 range.end_point = (Point::from_ts_point(range.end_point)
528 - step_start_point)
529 .to_ts_point();
530 }
531
532 if let Some(SyntaxLayerContent::Parsed { tree: old_tree, .. }) =
533 old_layer.map(|layer| &layer.content)
534 {
535 if let ParseMode::Combined {
536 mut parent_layer_changed_ranges,
537 ..
538 } = step.mode
539 {
540 for range in &mut parent_layer_changed_ranges {
541 range.start -= step_start_byte;
542 range.end -= step_start_byte;
543 }
544
545 included_ranges = splice_included_ranges(
546 old_tree.included_ranges(),
547 &parent_layer_changed_ranges,
548 &included_ranges,
549 );
550 }
551
552 tree = parse_text(
553 grammar,
554 text.as_rope(),
555 step_start_byte,
556 included_ranges,
557 Some(old_tree.clone()),
558 );
559 changed_ranges = join_ranges(
560 invalidated_ranges.iter().cloned().filter(|range| {
561 range.start <= step_end_byte && range.end >= step_start_byte
562 }),
563 old_tree.changed_ranges(&tree).map(|r| {
564 step_start_byte + r.start_byte..step_start_byte + r.end_byte
565 }),
566 );
567 } else {
568 tree = parse_text(
569 grammar,
570 text.as_rope(),
571 step_start_byte,
572 included_ranges,
573 None,
574 );
575 changed_ranges = vec![step_start_byte..step_end_byte];
576 }
577
578 if let (Some((config, registry)), false) = (
579 grammar.injection_config.as_ref().zip(registry.as_ref()),
580 changed_ranges.is_empty(),
581 ) {
582 for range in &changed_ranges {
583 changed_regions.insert(
584 ChangedRegion {
585 depth: step.depth + 1,
586 range: text.anchor_before(range.start)
587 ..text.anchor_after(range.end),
588 },
589 text,
590 );
591 }
592 get_injections(
593 config,
594 text,
595 tree.root_node_with_offset(
596 step_start_byte,
597 step_start_point.to_ts_point(),
598 ),
599 registry,
600 step.depth + 1,
601 &changed_ranges,
602 &mut combined_injection_ranges,
603 &mut queue,
604 );
605 }
606
607 SyntaxLayerContent::Parsed { tree, language }
608 }
609 ParseStepLanguage::Pending { name } => SyntaxLayerContent::Pending {
610 language_name: name,
611 },
612 };
613
614 layers.push(
615 SyntaxLayer {
616 depth: step.depth,
617 range: step.range,
618 content,
619 },
620 &text,
621 );
622 }
623
624 drop(cursor);
625 self.layers = layers;
626 self.interpolated_version = text.version.clone();
627 self.parsed_version = text.version.clone();
628 #[cfg(debug_assertions)]
629 self.check_invariants(text);
630 }
631
632 #[cfg(debug_assertions)]
633 fn check_invariants(&self, text: &BufferSnapshot) {
634 let mut max_depth = 0;
635 let mut prev_range: Option<Range<Anchor>> = None;
636 for layer in self.layers.iter() {
637 if layer.depth == max_depth {
638 if let Some(prev_range) = prev_range {
639 match layer.range.start.cmp(&prev_range.start, text) {
640 Ordering::Less => panic!("layers out of order"),
641 Ordering::Equal => {
642 assert!(layer.range.end.cmp(&prev_range.end, text).is_ge())
643 }
644 Ordering::Greater => {}
645 }
646 }
647 } else if layer.depth < max_depth {
648 panic!("layers out of order")
649 }
650 max_depth = layer.depth;
651 prev_range = Some(layer.range.clone());
652 }
653 }
654
655 pub fn single_tree_captures<'a>(
656 range: Range<usize>,
657 text: &'a Rope,
658 tree: &'a Tree,
659 language: &'a Arc<Language>,
660 query: fn(&Grammar) -> Option<&Query>,
661 ) -> SyntaxMapCaptures<'a> {
662 SyntaxMapCaptures::new(
663 range.clone(),
664 text,
665 [SyntaxLayerInfo {
666 language,
667 depth: 0,
668 node: tree.root_node(),
669 }]
670 .into_iter(),
671 query,
672 )
673 }
674
675 pub fn captures<'a>(
676 &'a self,
677 range: Range<usize>,
678 buffer: &'a BufferSnapshot,
679 query: fn(&Grammar) -> Option<&Query>,
680 ) -> SyntaxMapCaptures {
681 SyntaxMapCaptures::new(
682 range.clone(),
683 buffer.as_rope(),
684 self.layers_for_range(range, buffer).into_iter(),
685 query,
686 )
687 }
688
689 pub fn matches<'a>(
690 &'a self,
691 range: Range<usize>,
692 buffer: &'a BufferSnapshot,
693 query: fn(&Grammar) -> Option<&Query>,
694 ) -> SyntaxMapMatches {
695 SyntaxMapMatches::new(
696 range.clone(),
697 buffer.as_rope(),
698 self.layers_for_range(range, buffer).into_iter(),
699 query,
700 )
701 }
702
703 #[cfg(test)]
704 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
705 self.layers_for_range(0..buffer.len(), buffer).collect()
706 }
707
708 pub fn layers_for_range<'a, T: ToOffset>(
709 &'a self,
710 range: Range<T>,
711 buffer: &'a BufferSnapshot,
712 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
713 let start = buffer.anchor_before(range.start.to_offset(buffer));
714 let end = buffer.anchor_after(range.end.to_offset(buffer));
715
716 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
717 if summary.max_depth > summary.min_depth {
718 true
719 } else {
720 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
721 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
722 !is_before_start && !is_after_end
723 }
724 });
725
726 cursor.next(buffer);
727 iter::from_fn(move || {
728 while let Some(layer) = cursor.item() {
729 if let SyntaxLayerContent::Parsed { tree, language } = &layer.content {
730 let info = SyntaxLayerInfo {
731 language,
732 depth: layer.depth,
733 node: tree.root_node_with_offset(
734 layer.range.start.to_offset(buffer),
735 layer.range.start.to_point(buffer).to_ts_point(),
736 ),
737 };
738 cursor.next(buffer);
739 return Some(info);
740 } else {
741 cursor.next(buffer);
742 }
743 }
744 None
745 })
746 }
747
748 pub fn contains_unknown_injections(&self) -> bool {
749 self.layers.summary().contains_unknown_injections
750 }
751
752 pub fn language_registry_version(&self) -> usize {
753 self.language_registry_version
754 }
755}
756
757impl<'a> SyntaxMapCaptures<'a> {
758 fn new(
759 range: Range<usize>,
760 text: &'a Rope,
761 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
762 query: fn(&Grammar) -> Option<&Query>,
763 ) -> Self {
764 let mut result = Self {
765 layers: Vec::new(),
766 grammars: Vec::new(),
767 active_layer_count: 0,
768 };
769 for SyntaxLayerInfo {
770 language,
771 depth,
772 node,
773 } in layers
774 {
775 let grammar = match &language.grammar {
776 Some(grammar) => grammar,
777 None => continue,
778 };
779 let query = match query(&grammar) {
780 Some(query) => query,
781 None => continue,
782 };
783
784 let mut query_cursor = QueryCursorHandle::new();
785
786 // TODO - add a Tree-sitter API to remove the need for this.
787 let cursor = unsafe {
788 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
789 };
790
791 cursor.set_byte_range(range.clone());
792 let captures = cursor.captures(query, node, TextProvider(text));
793 let grammar_index = result
794 .grammars
795 .iter()
796 .position(|g| g.id == grammar.id())
797 .unwrap_or_else(|| {
798 result.grammars.push(grammar);
799 result.grammars.len() - 1
800 });
801 let mut layer = SyntaxMapCapturesLayer {
802 depth,
803 grammar_index,
804 next_capture: None,
805 captures,
806 _query_cursor: query_cursor,
807 };
808
809 layer.advance();
810 if layer.next_capture.is_some() {
811 let key = layer.sort_key();
812 let ix = match result.layers[..result.active_layer_count]
813 .binary_search_by_key(&key, |layer| layer.sort_key())
814 {
815 Ok(ix) | Err(ix) => ix,
816 };
817 result.layers.insert(ix, layer);
818 result.active_layer_count += 1;
819 } else {
820 result.layers.push(layer);
821 }
822 }
823
824 result
825 }
826
827 pub fn grammars(&self) -> &[&'a Grammar] {
828 &self.grammars
829 }
830
831 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
832 let layer = self.layers[..self.active_layer_count].first()?;
833 let capture = layer.next_capture?;
834 Some(SyntaxMapCapture {
835 depth: layer.depth,
836 grammar_index: layer.grammar_index,
837 index: capture.index,
838 node: capture.node,
839 })
840 }
841
842 pub fn advance(&mut self) -> bool {
843 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
844 layer
845 } else {
846 return false;
847 };
848
849 layer.advance();
850 if layer.next_capture.is_some() {
851 let key = layer.sort_key();
852 let i = 1 + self.layers[1..self.active_layer_count]
853 .iter()
854 .position(|later_layer| key < later_layer.sort_key())
855 .unwrap_or(self.active_layer_count - 1);
856 self.layers[0..i].rotate_left(1);
857 } else {
858 self.layers[0..self.active_layer_count].rotate_left(1);
859 self.active_layer_count -= 1;
860 }
861
862 true
863 }
864
865 pub fn set_byte_range(&mut self, range: Range<usize>) {
866 for layer in &mut self.layers {
867 layer.captures.set_byte_range(range.clone());
868 if let Some(capture) = &layer.next_capture {
869 if capture.node.end_byte() > range.start {
870 continue;
871 }
872 }
873 layer.advance();
874 }
875 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
876 self.active_layer_count = self
877 .layers
878 .iter()
879 .position(|layer| layer.next_capture.is_none())
880 .unwrap_or(self.layers.len());
881 }
882}
883
884impl<'a> SyntaxMapMatches<'a> {
885 fn new(
886 range: Range<usize>,
887 text: &'a Rope,
888 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
889 query: fn(&Grammar) -> Option<&Query>,
890 ) -> Self {
891 let mut result = Self::default();
892 for SyntaxLayerInfo {
893 language,
894 depth,
895 node,
896 } in layers
897 {
898 let grammar = match &language.grammar {
899 Some(grammar) => grammar,
900 None => continue,
901 };
902 let query = match query(&grammar) {
903 Some(query) => query,
904 None => continue,
905 };
906
907 let mut query_cursor = QueryCursorHandle::new();
908
909 // TODO - add a Tree-sitter API to remove the need for this.
910 let cursor = unsafe {
911 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
912 };
913
914 cursor.set_byte_range(range.clone());
915 let matches = cursor.matches(query, node, TextProvider(text));
916 let grammar_index = result
917 .grammars
918 .iter()
919 .position(|g| g.id == grammar.id())
920 .unwrap_or_else(|| {
921 result.grammars.push(grammar);
922 result.grammars.len() - 1
923 });
924 let mut layer = SyntaxMapMatchesLayer {
925 depth,
926 grammar_index,
927 matches,
928 next_pattern_index: 0,
929 next_captures: Vec::new(),
930 has_next: false,
931 _query_cursor: query_cursor,
932 };
933
934 layer.advance();
935 if layer.has_next {
936 let key = layer.sort_key();
937 let ix = match result.layers[..result.active_layer_count]
938 .binary_search_by_key(&key, |layer| layer.sort_key())
939 {
940 Ok(ix) | Err(ix) => ix,
941 };
942 result.layers.insert(ix, layer);
943 result.active_layer_count += 1;
944 } else {
945 result.layers.push(layer);
946 }
947 }
948 result
949 }
950
951 pub fn grammars(&self) -> &[&'a Grammar] {
952 &self.grammars
953 }
954
955 pub fn peek(&self) -> Option<SyntaxMapMatch> {
956 let layer = self.layers.first()?;
957 if !layer.has_next {
958 return None;
959 }
960 Some(SyntaxMapMatch {
961 depth: layer.depth,
962 grammar_index: layer.grammar_index,
963 pattern_index: layer.next_pattern_index,
964 captures: &layer.next_captures,
965 })
966 }
967
968 pub fn advance(&mut self) -> bool {
969 let layer = if let Some(layer) = self.layers.first_mut() {
970 layer
971 } else {
972 return false;
973 };
974
975 layer.advance();
976 if layer.has_next {
977 let key = layer.sort_key();
978 let i = 1 + self.layers[1..self.active_layer_count]
979 .iter()
980 .position(|later_layer| key < later_layer.sort_key())
981 .unwrap_or(self.active_layer_count - 1);
982 self.layers[0..i].rotate_left(1);
983 } else {
984 self.layers[0..self.active_layer_count].rotate_left(1);
985 self.active_layer_count -= 1;
986 }
987
988 true
989 }
990}
991
992impl<'a> SyntaxMapCapturesLayer<'a> {
993 fn advance(&mut self) {
994 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
995 }
996
997 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
998 if let Some(capture) = &self.next_capture {
999 let range = capture.node.byte_range();
1000 (range.start, Reverse(range.end), self.depth)
1001 } else {
1002 (usize::MAX, Reverse(0), usize::MAX)
1003 }
1004 }
1005}
1006
1007impl<'a> SyntaxMapMatchesLayer<'a> {
1008 fn advance(&mut self) {
1009 if let Some(mat) = self.matches.next() {
1010 self.next_captures.clear();
1011 self.next_captures.extend_from_slice(&mat.captures);
1012 self.next_pattern_index = mat.pattern_index;
1013 self.has_next = true;
1014 } else {
1015 self.has_next = false;
1016 }
1017 }
1018
1019 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
1020 if self.has_next {
1021 let captures = &self.next_captures;
1022 if let Some((first, last)) = captures.first().zip(captures.last()) {
1023 return (
1024 first.node.start_byte(),
1025 Reverse(last.node.end_byte()),
1026 self.depth,
1027 );
1028 }
1029 }
1030 (usize::MAX, Reverse(0), usize::MAX)
1031 }
1032}
1033
1034impl<'a> Iterator for SyntaxMapCaptures<'a> {
1035 type Item = SyntaxMapCapture<'a>;
1036
1037 fn next(&mut self) -> Option<Self::Item> {
1038 let result = self.peek();
1039 self.advance();
1040 result
1041 }
1042}
1043
1044fn join_ranges(
1045 a: impl Iterator<Item = Range<usize>>,
1046 b: impl Iterator<Item = Range<usize>>,
1047) -> Vec<Range<usize>> {
1048 let mut result = Vec::<Range<usize>>::new();
1049 let mut a = a.peekable();
1050 let mut b = b.peekable();
1051 loop {
1052 let range = match (a.peek(), b.peek()) {
1053 (Some(range_a), Some(range_b)) => {
1054 if range_a.start < range_b.start {
1055 a.next().unwrap()
1056 } else {
1057 b.next().unwrap()
1058 }
1059 }
1060 (None, Some(_)) => b.next().unwrap(),
1061 (Some(_), None) => a.next().unwrap(),
1062 (None, None) => break,
1063 };
1064
1065 if let Some(last) = result.last_mut() {
1066 if range.start <= last.end {
1067 last.end = last.end.max(range.end);
1068 continue;
1069 }
1070 }
1071 result.push(range);
1072 }
1073 result
1074}
1075
1076fn parse_text(
1077 grammar: &Grammar,
1078 text: &Rope,
1079 start_byte: usize,
1080 ranges: Vec<tree_sitter::Range>,
1081 old_tree: Option<Tree>,
1082) -> Tree {
1083 PARSER.with(|parser| {
1084 let mut parser = parser.borrow_mut();
1085 let mut chunks = text.chunks_in_range(start_byte..text.len());
1086 parser
1087 .set_included_ranges(&ranges)
1088 .expect("overlapping ranges");
1089 parser
1090 .set_language(grammar.ts_language)
1091 .expect("incompatible grammar");
1092 parser
1093 .parse_with(
1094 &mut move |offset, _| {
1095 chunks.seek(start_byte + offset);
1096 chunks.next().unwrap_or("").as_bytes()
1097 },
1098 old_tree.as_ref(),
1099 )
1100 .expect("invalid language")
1101 })
1102}
1103
1104fn get_injections(
1105 config: &InjectionConfig,
1106 text: &BufferSnapshot,
1107 node: Node,
1108 language_registry: &Arc<LanguageRegistry>,
1109 depth: usize,
1110 changed_ranges: &[Range<usize>],
1111 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
1112 queue: &mut BinaryHeap<ParseStep>,
1113) {
1114 let mut query_cursor = QueryCursorHandle::new();
1115 let mut prev_match = None;
1116
1117 // Ensure that a `ParseStep` is created for every combined injection language, even
1118 // if there currently no matches for that injection.
1119 combined_injection_ranges.clear();
1120 for pattern in &config.patterns {
1121 if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
1122 if let Some(language) = language_registry
1123 .language_for_name_or_extension(language_name)
1124 .now_or_never()
1125 .and_then(|language| language.ok())
1126 {
1127 combined_injection_ranges.insert(language, Vec::new());
1128 }
1129 }
1130 }
1131
1132 for query_range in changed_ranges {
1133 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
1134 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
1135 let content_ranges = mat
1136 .nodes_for_capture_index(config.content_capture_ix)
1137 .map(|node| node.range())
1138 .collect::<Vec<_>>();
1139 if content_ranges.is_empty() {
1140 continue;
1141 }
1142
1143 // Avoid duplicate matches if two changed ranges intersect the same injection.
1144 let content_range =
1145 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
1146 if let Some((last_pattern_ix, last_range)) = &prev_match {
1147 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1148 continue;
1149 }
1150 }
1151 prev_match = Some((mat.pattern_index, content_range.clone()));
1152
1153 let combined = config.patterns[mat.pattern_index].combined;
1154
1155 let mut language_name = None;
1156 let mut step_range = content_range.clone();
1157 if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {
1158 language_name = Some(Cow::Borrowed(name.as_ref()))
1159 } else if let Some(language_node) = config
1160 .language_capture_ix
1161 .and_then(|ix| mat.nodes_for_capture_index(ix).next())
1162 {
1163 step_range.start = cmp::min(content_range.start, language_node.start_byte());
1164 step_range.end = cmp::max(content_range.end, language_node.end_byte());
1165 language_name = Some(Cow::Owned(
1166 text.text_for_range(language_node.byte_range()).collect(),
1167 ))
1168 };
1169
1170 if let Some(language_name) = language_name {
1171 let language = language_registry
1172 .language_for_name_or_extension(&language_name)
1173 .now_or_never()
1174 .and_then(|language| language.ok());
1175 let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
1176 if let Some(language) = language {
1177 if combined {
1178 combined_injection_ranges
1179 .entry(language.clone())
1180 .or_default()
1181 .extend(content_ranges);
1182 } else {
1183 queue.push(ParseStep {
1184 depth,
1185 language: ParseStepLanguage::Loaded { language },
1186 included_ranges: content_ranges,
1187 range,
1188 mode: ParseMode::Single,
1189 });
1190 }
1191 } else {
1192 queue.push(ParseStep {
1193 depth,
1194 language: ParseStepLanguage::Pending {
1195 name: language_name.into(),
1196 },
1197 included_ranges: content_ranges,
1198 range,
1199 mode: ParseMode::Single,
1200 });
1201 }
1202 }
1203 }
1204 }
1205
1206 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1207 included_ranges.sort_unstable();
1208 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1209 queue.push(ParseStep {
1210 depth,
1211 language: ParseStepLanguage::Loaded { language },
1212 range,
1213 included_ranges,
1214 mode: ParseMode::Combined {
1215 parent_layer_range: node.start_byte()..node.end_byte(),
1216 parent_layer_changed_ranges: changed_ranges.to_vec(),
1217 },
1218 })
1219 }
1220}
1221
1222fn splice_included_ranges(
1223 mut ranges: Vec<tree_sitter::Range>,
1224 changed_ranges: &[Range<usize>],
1225 new_ranges: &[tree_sitter::Range],
1226) -> Vec<tree_sitter::Range> {
1227 let mut changed_ranges = changed_ranges.into_iter().peekable();
1228 let mut new_ranges = new_ranges.into_iter().peekable();
1229 let mut ranges_ix = 0;
1230 loop {
1231 let new_range = new_ranges.peek();
1232 let mut changed_range = changed_ranges.peek();
1233
1234 // Remove ranges that have changed before inserting any new ranges
1235 // into those ranges.
1236 if let Some((changed, new)) = changed_range.zip(new_range) {
1237 if new.end_byte < changed.start {
1238 changed_range = None;
1239 }
1240 }
1241
1242 if let Some(changed) = changed_range {
1243 let mut start_ix = ranges_ix
1244 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1245 Ok(ix) | Err(ix) => ix,
1246 };
1247 let mut end_ix = ranges_ix
1248 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1249 Ok(ix) => ix + 1,
1250 Err(ix) => ix,
1251 };
1252
1253 // If there are empty ranges, then there may be multiple ranges with the same
1254 // start or end. Expand the splice to include any adjacent ranges that touch
1255 // the changed range.
1256 while start_ix > 0 {
1257 if ranges[start_ix - 1].end_byte == changed.start {
1258 start_ix -= 1;
1259 } else {
1260 break;
1261 }
1262 }
1263 while let Some(range) = ranges.get(end_ix) {
1264 if range.start_byte == changed.end {
1265 end_ix += 1;
1266 } else {
1267 break;
1268 }
1269 }
1270
1271 if end_ix > start_ix {
1272 ranges.splice(start_ix..end_ix, []);
1273 }
1274 changed_ranges.next();
1275 ranges_ix = start_ix;
1276 } else if let Some(new_range) = new_range {
1277 let ix = ranges_ix
1278 + match ranges[ranges_ix..]
1279 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1280 {
1281 Ok(ix) | Err(ix) => ix,
1282 };
1283 ranges.insert(ix, **new_range);
1284 new_ranges.next();
1285 ranges_ix = ix + 1;
1286 } else {
1287 break;
1288 }
1289 }
1290 ranges
1291}
1292
1293impl<'a> SyntaxLayerInfo<'a> {
1294 pub(crate) fn override_id(&self, offset: usize, text: &text::BufferSnapshot) -> Option<u32> {
1295 let text = TextProvider(text.as_rope());
1296 let config = self.language.grammar.as_ref()?.override_config.as_ref()?;
1297
1298 let mut query_cursor = QueryCursorHandle::new();
1299 query_cursor.set_byte_range(offset..offset);
1300
1301 let mut smallest_match: Option<(u32, Range<usize>)> = None;
1302 for mat in query_cursor.matches(&config.query, self.node, text) {
1303 for capture in mat.captures {
1304 if !config.values.contains_key(&capture.index) {
1305 continue;
1306 }
1307
1308 let range = capture.node.byte_range();
1309 if offset <= range.start || offset >= range.end {
1310 continue;
1311 }
1312
1313 if let Some((_, smallest_range)) = &smallest_match {
1314 if range.len() < smallest_range.len() {
1315 smallest_match = Some((capture.index, range))
1316 }
1317 continue;
1318 }
1319
1320 smallest_match = Some((capture.index, range));
1321 }
1322 }
1323
1324 smallest_match.map(|(index, _)| index)
1325 }
1326}
1327
1328impl std::ops::Deref for SyntaxMap {
1329 type Target = SyntaxSnapshot;
1330
1331 fn deref(&self) -> &Self::Target {
1332 &self.snapshot
1333 }
1334}
1335
1336impl PartialEq for ParseStep {
1337 fn eq(&self, _: &Self) -> bool {
1338 false
1339 }
1340}
1341
1342impl Eq for ParseStep {}
1343
1344impl PartialOrd for ParseStep {
1345 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1346 Some(self.cmp(&other))
1347 }
1348}
1349
1350impl Ord for ParseStep {
1351 fn cmp(&self, other: &Self) -> Ordering {
1352 let range_a = self.range();
1353 let range_b = other.range();
1354 Ord::cmp(&other.depth, &self.depth)
1355 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1356 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1357 .then_with(|| self.language.id().cmp(&other.language.id()))
1358 }
1359}
1360
1361impl ParseStep {
1362 fn range(&self) -> Range<usize> {
1363 if let ParseMode::Combined {
1364 parent_layer_range, ..
1365 } = &self.mode
1366 {
1367 parent_layer_range.clone()
1368 } else {
1369 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1370 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1371 start..end
1372 }
1373 }
1374}
1375
1376impl ChangedRegion {
1377 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1378 let range_a = &self.range;
1379 let range_b = &other.range;
1380 Ord::cmp(&self.depth, &other.depth)
1381 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1382 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1383 }
1384}
1385
1386impl ChangeRegionSet {
1387 fn start_position(&self) -> ChangeStartPosition {
1388 self.0.first().map_or(
1389 ChangeStartPosition {
1390 depth: usize::MAX,
1391 position: Anchor::MAX,
1392 },
1393 |region| ChangeStartPosition {
1394 depth: region.depth,
1395 position: region.range.start,
1396 },
1397 )
1398 }
1399
1400 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1401 for region in &self.0 {
1402 if region.depth < layer.depth {
1403 continue;
1404 }
1405 if region.depth > layer.depth {
1406 break;
1407 }
1408 if region.range.end.cmp(&layer.range.start, text).is_le() {
1409 continue;
1410 }
1411 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1412 break;
1413 }
1414 return true;
1415 }
1416 false
1417 }
1418
1419 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1420 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1421 self.0.insert(ix, region);
1422 }
1423 }
1424
1425 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1426 let prev_len = self.0.len();
1427 self.0.retain(|region| {
1428 region.depth > summary.max_depth
1429 || (region.depth == summary.max_depth
1430 && region
1431 .range
1432 .end
1433 .cmp(&summary.last_layer_range.start, text)
1434 .is_gt())
1435 });
1436 self.0.len() < prev_len
1437 }
1438}
1439
1440impl Default for SyntaxLayerSummary {
1441 fn default() -> Self {
1442 Self {
1443 max_depth: 0,
1444 min_depth: 0,
1445 range: Anchor::MAX..Anchor::MIN,
1446 last_layer_range: Anchor::MIN..Anchor::MAX,
1447 last_layer_language: None,
1448 contains_unknown_injections: false,
1449 }
1450 }
1451}
1452
1453impl sum_tree::Summary for SyntaxLayerSummary {
1454 type Context = BufferSnapshot;
1455
1456 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1457 if other.max_depth > self.max_depth {
1458 self.max_depth = other.max_depth;
1459 self.range = other.range.clone();
1460 } else {
1461 if self.range == (Anchor::MAX..Anchor::MAX) {
1462 self.range.start = other.range.start;
1463 }
1464 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1465 self.range.end = other.range.end;
1466 }
1467 }
1468 self.last_layer_range = other.last_layer_range.clone();
1469 self.last_layer_language = other.last_layer_language;
1470 self.contains_unknown_injections |= other.contains_unknown_injections;
1471 }
1472}
1473
1474impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1475 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1476 Ord::cmp(&self.depth, &cursor_location.max_depth)
1477 .then_with(|| {
1478 self.range
1479 .start
1480 .cmp(&cursor_location.last_layer_range.start, buffer)
1481 })
1482 .then_with(|| {
1483 cursor_location
1484 .last_layer_range
1485 .end
1486 .cmp(&self.range.end, buffer)
1487 })
1488 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1489 }
1490}
1491
1492impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for ChangeStartPosition {
1493 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1494 Ord::cmp(&self.depth, &cursor_location.max_depth)
1495 .then_with(|| self.position.cmp(&cursor_location.range.end, text))
1496 }
1497}
1498
1499impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1500 for SyntaxLayerPositionBeforeChange
1501{
1502 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1503 if self.change.cmp(cursor_location, buffer).is_le() {
1504 return Ordering::Less;
1505 } else {
1506 self.position.cmp(cursor_location, buffer)
1507 }
1508 }
1509}
1510
1511impl sum_tree::Item for SyntaxLayer {
1512 type Summary = SyntaxLayerSummary;
1513
1514 fn summary(&self) -> Self::Summary {
1515 SyntaxLayerSummary {
1516 min_depth: self.depth,
1517 max_depth: self.depth,
1518 range: self.range.clone(),
1519 last_layer_range: self.range.clone(),
1520 last_layer_language: self.content.language_id(),
1521 contains_unknown_injections: matches!(self.content, SyntaxLayerContent::Pending { .. }),
1522 }
1523 }
1524}
1525
1526impl std::fmt::Debug for SyntaxLayer {
1527 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1528 f.debug_struct("SyntaxLayer")
1529 .field("depth", &self.depth)
1530 .field("range", &self.range)
1531 .field("tree", &self.content.tree())
1532 .finish()
1533 }
1534}
1535
1536impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1537 type I = ByteChunks<'a>;
1538
1539 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1540 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1541 }
1542}
1543
1544impl<'a> Iterator for ByteChunks<'a> {
1545 type Item = &'a [u8];
1546
1547 fn next(&mut self) -> Option<Self::Item> {
1548 self.0.next().map(str::as_bytes)
1549 }
1550}
1551
1552impl QueryCursorHandle {
1553 pub(crate) fn new() -> Self {
1554 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1555 cursor.set_match_limit(64);
1556 QueryCursorHandle(Some(cursor))
1557 }
1558}
1559
1560impl Deref for QueryCursorHandle {
1561 type Target = QueryCursor;
1562
1563 fn deref(&self) -> &Self::Target {
1564 self.0.as_ref().unwrap()
1565 }
1566}
1567
1568impl DerefMut for QueryCursorHandle {
1569 fn deref_mut(&mut self) -> &mut Self::Target {
1570 self.0.as_mut().unwrap()
1571 }
1572}
1573
1574impl Drop for QueryCursorHandle {
1575 fn drop(&mut self) {
1576 let mut cursor = self.0.take().unwrap();
1577 cursor.set_byte_range(0..usize::MAX);
1578 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1579 QUERY_CURSORS.lock().push(cursor)
1580 }
1581}
1582
1583pub(crate) trait ToTreeSitterPoint {
1584 fn to_ts_point(self) -> tree_sitter::Point;
1585 fn from_ts_point(point: tree_sitter::Point) -> Self;
1586}
1587
1588impl ToTreeSitterPoint for Point {
1589 fn to_ts_point(self) -> tree_sitter::Point {
1590 tree_sitter::Point::new(self.row as usize, self.column as usize)
1591 }
1592
1593 fn from_ts_point(point: tree_sitter::Point) -> Self {
1594 Point::new(point.row as u32, point.column as u32)
1595 }
1596}
1597
1598#[cfg(test)]
1599mod tests {
1600 use super::*;
1601 use crate::LanguageConfig;
1602 use rand::rngs::StdRng;
1603 use std::env;
1604 use text::Buffer;
1605 use unindent::Unindent as _;
1606 use util::test::marked_text_ranges;
1607
1608 #[test]
1609 fn test_splice_included_ranges() {
1610 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1611
1612 let new_ranges = splice_included_ranges(
1613 ranges.clone(),
1614 &[54..56, 58..68],
1615 &[ts_range(50..54), ts_range(59..67)],
1616 );
1617 assert_eq!(
1618 new_ranges,
1619 &[
1620 ts_range(20..30),
1621 ts_range(50..54),
1622 ts_range(59..67),
1623 ts_range(80..90),
1624 ]
1625 );
1626
1627 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1628 assert_eq!(
1629 new_ranges,
1630 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1631 );
1632
1633 let new_ranges =
1634 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1635 assert_eq!(
1636 new_ranges,
1637 &[
1638 ts_range(0..2),
1639 ts_range(20..30),
1640 ts_range(50..60),
1641 ts_range(70..75),
1642 ts_range(80..90)
1643 ]
1644 );
1645
1646 let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
1647 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
1648
1649 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1650 tree_sitter::Range {
1651 start_byte: range.start,
1652 start_point: tree_sitter::Point {
1653 row: 0,
1654 column: range.start,
1655 },
1656 end_byte: range.end,
1657 end_point: tree_sitter::Point {
1658 row: 0,
1659 column: range.end,
1660 },
1661 }
1662 }
1663 }
1664
1665 #[gpui::test]
1666 fn test_syntax_map_layers_for_range() {
1667 let registry = Arc::new(LanguageRegistry::test());
1668 let language = Arc::new(rust_lang());
1669 registry.add(language.clone());
1670
1671 let mut buffer = Buffer::new(
1672 0,
1673 0,
1674 r#"
1675 fn a() {
1676 assert_eq!(
1677 b(vec![C {}]),
1678 vec![d.e],
1679 );
1680 println!("{}", f(|_| true));
1681 }
1682 "#
1683 .unindent(),
1684 );
1685
1686 let mut syntax_map = SyntaxMap::new();
1687 syntax_map.set_language_registry(registry.clone());
1688 syntax_map.reparse(language.clone(), &buffer);
1689
1690 assert_layers_for_range(
1691 &syntax_map,
1692 &buffer,
1693 Point::new(2, 0)..Point::new(2, 0),
1694 &[
1695 "...(function_item ... (block (expression_statement (macro_invocation...",
1696 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1697 ],
1698 );
1699 assert_layers_for_range(
1700 &syntax_map,
1701 &buffer,
1702 Point::new(2, 14)..Point::new(2, 16),
1703 &[
1704 "...(function_item ...",
1705 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1706 "...(array_expression (struct_expression ...",
1707 ],
1708 );
1709 assert_layers_for_range(
1710 &syntax_map,
1711 &buffer,
1712 Point::new(3, 14)..Point::new(3, 16),
1713 &[
1714 "...(function_item ...",
1715 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1716 "...(array_expression (field_expression ...",
1717 ],
1718 );
1719 assert_layers_for_range(
1720 &syntax_map,
1721 &buffer,
1722 Point::new(5, 12)..Point::new(5, 16),
1723 &[
1724 "...(function_item ...",
1725 "...(call_expression ... (arguments (closure_expression ...",
1726 ],
1727 );
1728
1729 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1730 let macro_name_range = range_for_text(&buffer, "vec!");
1731 buffer.edit([(macro_name_range, "&")]);
1732 syntax_map.interpolate(&buffer);
1733 syntax_map.reparse(language.clone(), &buffer);
1734
1735 assert_layers_for_range(
1736 &syntax_map,
1737 &buffer,
1738 Point::new(2, 14)..Point::new(2, 16),
1739 &[
1740 "...(function_item ...",
1741 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1742 ],
1743 );
1744
1745 // Put the vec! macro back, adding back the syntactic layer.
1746 buffer.undo();
1747 syntax_map.interpolate(&buffer);
1748 syntax_map.reparse(language.clone(), &buffer);
1749
1750 assert_layers_for_range(
1751 &syntax_map,
1752 &buffer,
1753 Point::new(2, 14)..Point::new(2, 16),
1754 &[
1755 "...(function_item ...",
1756 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1757 "...(array_expression (struct_expression ...",
1758 ],
1759 );
1760 }
1761
1762 #[gpui::test]
1763 fn test_dynamic_language_injection() {
1764 let registry = Arc::new(LanguageRegistry::test());
1765 let markdown = Arc::new(markdown_lang());
1766 registry.add(markdown.clone());
1767 registry.add(Arc::new(rust_lang()));
1768 registry.add(Arc::new(ruby_lang()));
1769
1770 let mut buffer = Buffer::new(
1771 0,
1772 0,
1773 r#"
1774 This is a code block:
1775
1776 ```rs
1777 fn foo() {}
1778 ```
1779 "#
1780 .unindent(),
1781 );
1782
1783 let mut syntax_map = SyntaxMap::new();
1784 syntax_map.set_language_registry(registry.clone());
1785 syntax_map.reparse(markdown.clone(), &buffer);
1786 assert_layers_for_range(
1787 &syntax_map,
1788 &buffer,
1789 Point::new(3, 0)..Point::new(3, 0),
1790 &[
1791 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1792 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
1793 ],
1794 );
1795
1796 // Replace Rust with Ruby in code block.
1797 let macro_name_range = range_for_text(&buffer, "rs");
1798 buffer.edit([(macro_name_range, "ruby")]);
1799 syntax_map.interpolate(&buffer);
1800 syntax_map.reparse(markdown.clone(), &buffer);
1801 assert_layers_for_range(
1802 &syntax_map,
1803 &buffer,
1804 Point::new(3, 0)..Point::new(3, 0),
1805 &[
1806 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1807 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
1808 ],
1809 );
1810
1811 // Replace Ruby with a language that hasn't been loaded yet.
1812 let macro_name_range = range_for_text(&buffer, "ruby");
1813 buffer.edit([(macro_name_range, "html")]);
1814 syntax_map.interpolate(&buffer);
1815 syntax_map.reparse(markdown.clone(), &buffer);
1816 assert_layers_for_range(
1817 &syntax_map,
1818 &buffer,
1819 Point::new(3, 0)..Point::new(3, 0),
1820 &[
1821 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
1822 ],
1823 );
1824 assert!(syntax_map.contains_unknown_injections());
1825
1826 registry.add(Arc::new(html_lang()));
1827 syntax_map.reparse(markdown.clone(), &buffer);
1828 assert_layers_for_range(
1829 &syntax_map,
1830 &buffer,
1831 Point::new(3, 0)..Point::new(3, 0),
1832 &[
1833 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1834 "(fragment (text))",
1835 ],
1836 );
1837 assert!(!syntax_map.contains_unknown_injections());
1838 }
1839
1840 #[gpui::test]
1841 fn test_typing_multiple_new_injections() {
1842 let (buffer, syntax_map) = test_edit_sequence(
1843 "Rust",
1844 &[
1845 "fn a() { dbg }",
1846 "fn a() { dbg«!» }",
1847 "fn a() { dbg!«()» }",
1848 "fn a() { dbg!(«b») }",
1849 "fn a() { dbg!(b«.») }",
1850 "fn a() { dbg!(b.«c») }",
1851 "fn a() { dbg!(b.c«()») }",
1852 "fn a() { dbg!(b.c(«vec»)) }",
1853 "fn a() { dbg!(b.c(vec«!»)) }",
1854 "fn a() { dbg!(b.c(vec!«[]»)) }",
1855 "fn a() { dbg!(b.c(vec![«d»])) }",
1856 "fn a() { dbg!(b.c(vec![d«.»])) }",
1857 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1858 ],
1859 );
1860
1861 assert_capture_ranges(
1862 &syntax_map,
1863 &buffer,
1864 &["field"],
1865 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1866 );
1867 }
1868
1869 #[gpui::test]
1870 fn test_pasting_new_injection_line_between_others() {
1871 let (buffer, syntax_map) = test_edit_sequence(
1872 "Rust",
1873 &[
1874 "
1875 fn a() {
1876 b!(B {});
1877 c!(C {});
1878 d!(D {});
1879 e!(E {});
1880 f!(F {});
1881 g!(G {});
1882 }
1883 ",
1884 "
1885 fn a() {
1886 b!(B {});
1887 c!(C {});
1888 d!(D {});
1889 « h!(H {});
1890 » e!(E {});
1891 f!(F {});
1892 g!(G {});
1893 }
1894 ",
1895 ],
1896 );
1897
1898 assert_capture_ranges(
1899 &syntax_map,
1900 &buffer,
1901 &["struct"],
1902 "
1903 fn a() {
1904 b!(«B {}»);
1905 c!(«C {}»);
1906 d!(«D {}»);
1907 h!(«H {}»);
1908 e!(«E {}»);
1909 f!(«F {}»);
1910 g!(«G {}»);
1911 }
1912 ",
1913 );
1914 }
1915
1916 #[gpui::test]
1917 fn test_joining_injections_with_child_injections() {
1918 let (buffer, syntax_map) = test_edit_sequence(
1919 "Rust",
1920 &[
1921 "
1922 fn a() {
1923 b!(
1924 c![one.two.three],
1925 d![four.five.six],
1926 );
1927 e!(
1928 f![seven.eight],
1929 );
1930 }
1931 ",
1932 "
1933 fn a() {
1934 b!(
1935 c![one.two.three],
1936 d![four.five.six],
1937 ˇ f![seven.eight],
1938 );
1939 }
1940 ",
1941 ],
1942 );
1943
1944 assert_capture_ranges(
1945 &syntax_map,
1946 &buffer,
1947 &["field"],
1948 "
1949 fn a() {
1950 b!(
1951 c![one.«two».«three»],
1952 d![four.«five».«six»],
1953 f![seven.«eight»],
1954 );
1955 }
1956 ",
1957 );
1958 }
1959
1960 #[gpui::test]
1961 fn test_editing_edges_of_injection() {
1962 test_edit_sequence(
1963 "Rust",
1964 &[
1965 "
1966 fn a() {
1967 b!(c!())
1968 }
1969 ",
1970 "
1971 fn a() {
1972 «d»!(c!())
1973 }
1974 ",
1975 "
1976 fn a() {
1977 «e»d!(c!())
1978 }
1979 ",
1980 "
1981 fn a() {
1982 ed!«[»c!()«]»
1983 }
1984 ",
1985 ],
1986 );
1987 }
1988
1989 #[gpui::test]
1990 fn test_edits_preceding_and_intersecting_injection() {
1991 test_edit_sequence(
1992 "Rust",
1993 &[
1994 //
1995 "const aaaaaaaaaaaa: B = c!(d(e.f));",
1996 "const aˇa: B = c!(d(eˇ));",
1997 ],
1998 );
1999 }
2000
2001 #[gpui::test]
2002 fn test_non_local_changes_create_injections() {
2003 test_edit_sequence(
2004 "Rust",
2005 &[
2006 "
2007 // a! {
2008 static B: C = d;
2009 // }
2010 ",
2011 "
2012 ˇa! {
2013 static B: C = d;
2014 ˇ}
2015 ",
2016 ],
2017 );
2018 }
2019
2020 #[gpui::test]
2021 fn test_creating_many_injections_in_one_edit() {
2022 test_edit_sequence(
2023 "Rust",
2024 &[
2025 "
2026 fn a() {
2027 one(Two::three(3));
2028 four(Five::six(6));
2029 seven(Eight::nine(9));
2030 }
2031 ",
2032 "
2033 fn a() {
2034 one«!»(Two::three(3));
2035 four«!»(Five::six(6));
2036 seven«!»(Eight::nine(9));
2037 }
2038 ",
2039 "
2040 fn a() {
2041 one!(Two::three«!»(3));
2042 four!(Five::six«!»(6));
2043 seven!(Eight::nine«!»(9));
2044 }
2045 ",
2046 ],
2047 );
2048 }
2049
2050 #[gpui::test]
2051 fn test_editing_across_injection_boundary() {
2052 test_edit_sequence(
2053 "Rust",
2054 &[
2055 "
2056 fn one() {
2057 two();
2058 three!(
2059 three.four,
2060 five.six,
2061 );
2062 }
2063 ",
2064 "
2065 fn one() {
2066 two();
2067 th«irty_five![»
2068 three.four,
2069 five.six,
2070 « seven.eight,
2071 ];»
2072 }
2073 ",
2074 ],
2075 );
2076 }
2077
2078 #[gpui::test]
2079 fn test_removing_injection_by_replacing_across_boundary() {
2080 test_edit_sequence(
2081 "Rust",
2082 &[
2083 "
2084 fn one() {
2085 two!(
2086 three.four,
2087 );
2088 }
2089 ",
2090 "
2091 fn one() {
2092 t«en
2093 .eleven(
2094 twelve,
2095 »
2096 three.four,
2097 );
2098 }
2099 ",
2100 ],
2101 );
2102 }
2103
2104 #[gpui::test]
2105 fn test_combined_injections() {
2106 let (buffer, syntax_map) = test_edit_sequence(
2107 "ERB",
2108 &[
2109 "
2110 <body>
2111 <% if @one %>
2112 <div class=one>
2113 <% else %>
2114 <div class=two>
2115 <% end %>
2116 </div>
2117 </body>
2118 ",
2119 "
2120 <body>
2121 <% if @one %>
2122 <div class=one>
2123 ˇ else ˇ
2124 <div class=two>
2125 <% end %>
2126 </div>
2127 </body>
2128 ",
2129 "
2130 <body>
2131 <% if @one «;» end %>
2132 </div>
2133 </body>
2134 ",
2135 ],
2136 );
2137
2138 assert_capture_ranges(
2139 &syntax_map,
2140 &buffer,
2141 &["tag", "ivar"],
2142 "
2143 <«body»>
2144 <% if «@one» ; end %>
2145 </«div»>
2146 </«body»>
2147 ",
2148 );
2149 }
2150
2151 #[gpui::test]
2152 fn test_combined_injections_empty_ranges() {
2153 test_edit_sequence(
2154 "ERB",
2155 &[
2156 "
2157 <% if @one %>
2158 <% else %>
2159 <% end %>
2160 ",
2161 "
2162 <% if @one %>
2163 ˇ<% end %>
2164 ",
2165 ],
2166 );
2167 }
2168
2169 #[gpui::test]
2170 fn test_combined_injections_edit_edges_of_ranges() {
2171 let (buffer, syntax_map) = test_edit_sequence(
2172 "ERB",
2173 &[
2174 "
2175 <%= one @two %>
2176 <%= three @four %>
2177 ",
2178 "
2179 <%= one @two %ˇ
2180 <%= three @four %>
2181 ",
2182 "
2183 <%= one @two %«>»
2184 <%= three @four %>
2185 ",
2186 ],
2187 );
2188
2189 assert_capture_ranges(
2190 &syntax_map,
2191 &buffer,
2192 &["tag", "ivar"],
2193 "
2194 <%= one «@two» %>
2195 <%= three «@four» %>
2196 ",
2197 );
2198 }
2199
2200 #[gpui::test]
2201 fn test_combined_injections_splitting_some_injections() {
2202 let (_buffer, _syntax_map) = test_edit_sequence(
2203 "ERB",
2204 &[
2205 r#"
2206 <%A if b(:c) %>
2207 d
2208 <% end %>
2209 eee
2210 <% f %>
2211 "#,
2212 r#"
2213 <%« AAAAAAA %>
2214 hhhhhhh
2215 <%=» if b(:c) %>
2216 d
2217 <% end %>
2218 eee
2219 <% f %>
2220 "#,
2221 ],
2222 );
2223 }
2224
2225 #[gpui::test]
2226 fn test_combined_injections_inside_injections() {
2227 let (_buffer, _syntax_map) = test_edit_sequence(
2228 "Markdown",
2229 &[
2230 r#"
2231 here is some ERB code:
2232
2233 ```erb
2234 <ul>
2235 <% people.each do |person| %>
2236 <li><%= person.name %></li>
2237 <% end %>
2238 </ul>
2239 ```
2240 "#,
2241 r#"
2242 here is some ERB code:
2243
2244 ```erb
2245 <ul>
2246 <% people«2».each do |person| %>
2247 <li><%= person.name %></li>
2248 <% end %>
2249 </ul>
2250 ```
2251 "#,
2252 ],
2253 );
2254 }
2255
2256 #[gpui::test(iterations = 50)]
2257 fn test_random_syntax_map_edits(mut rng: StdRng) {
2258 let operations = env::var("OPERATIONS")
2259 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2260 .unwrap_or(10);
2261
2262 let text = r#"
2263 fn test_something() {
2264 let vec = vec![5, 1, 3, 8];
2265 assert_eq!(
2266 vec
2267 .into_iter()
2268 .map(|i| i * 2)
2269 .collect::<Vec<usize>>(),
2270 vec![
2271 5 * 2, 1 * 2, 3 * 2, 8 * 2
2272 ],
2273 );
2274 }
2275 "#
2276 .unindent()
2277 .repeat(2);
2278
2279 let registry = Arc::new(LanguageRegistry::test());
2280 let language = Arc::new(rust_lang());
2281 registry.add(language.clone());
2282 let mut buffer = Buffer::new(0, 0, text);
2283
2284 let mut syntax_map = SyntaxMap::new();
2285 syntax_map.set_language_registry(registry.clone());
2286 syntax_map.reparse(language.clone(), &buffer);
2287
2288 let mut reference_syntax_map = SyntaxMap::new();
2289 reference_syntax_map.set_language_registry(registry.clone());
2290
2291 log::info!("initial text:\n{}", buffer.text());
2292
2293 for _ in 0..operations {
2294 let prev_buffer = buffer.snapshot();
2295 let prev_syntax_map = syntax_map.snapshot();
2296
2297 buffer.randomly_edit(&mut rng, 3);
2298 log::info!("text:\n{}", buffer.text());
2299
2300 syntax_map.interpolate(&buffer);
2301 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2302
2303 syntax_map.reparse(language.clone(), &buffer);
2304
2305 reference_syntax_map.clear();
2306 reference_syntax_map.reparse(language.clone(), &buffer);
2307 }
2308
2309 for i in 0..operations {
2310 let i = operations - i - 1;
2311 buffer.undo();
2312 log::info!("undoing operation {}", i);
2313 log::info!("text:\n{}", buffer.text());
2314
2315 syntax_map.interpolate(&buffer);
2316 syntax_map.reparse(language.clone(), &buffer);
2317
2318 reference_syntax_map.clear();
2319 reference_syntax_map.reparse(language.clone(), &buffer);
2320 assert_eq!(
2321 syntax_map.layers(&buffer).len(),
2322 reference_syntax_map.layers(&buffer).len(),
2323 "wrong number of layers after undoing edit {i}"
2324 );
2325 }
2326
2327 let layers = syntax_map.layers(&buffer);
2328 let reference_layers = reference_syntax_map.layers(&buffer);
2329 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2330 {
2331 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2332 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2333 }
2334 }
2335
2336 #[gpui::test(iterations = 50)]
2337 fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) {
2338 let operations = env::var("OPERATIONS")
2339 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2340 .unwrap_or(10);
2341
2342 let text = r#"
2343 <div id="main">
2344 <% if one?(:two) %>
2345 <p class="three" four>
2346 <%= yield :five %>
2347 </p>
2348 <% elsif Six.seven(8) %>
2349 <p id="three" four>
2350 <%= yield :five %>
2351 </p>
2352 <% else %>
2353 <span>Ok</span>
2354 <% end %>
2355 </div>
2356 "#
2357 .unindent()
2358 .repeat(8);
2359
2360 let registry = Arc::new(LanguageRegistry::test());
2361 let language = Arc::new(erb_lang());
2362 registry.add(language.clone());
2363 registry.add(Arc::new(ruby_lang()));
2364 registry.add(Arc::new(html_lang()));
2365 let mut buffer = Buffer::new(0, 0, text);
2366
2367 let mut syntax_map = SyntaxMap::new();
2368 syntax_map.set_language_registry(registry.clone());
2369 syntax_map.reparse(language.clone(), &buffer);
2370
2371 let mut reference_syntax_map = SyntaxMap::new();
2372 reference_syntax_map.set_language_registry(registry.clone());
2373
2374 log::info!("initial text:\n{}", buffer.text());
2375
2376 for _ in 0..operations {
2377 let prev_buffer = buffer.snapshot();
2378 let prev_syntax_map = syntax_map.snapshot();
2379
2380 buffer.randomly_edit(&mut rng, 3);
2381 log::info!("text:\n{}", buffer.text());
2382
2383 syntax_map.interpolate(&buffer);
2384 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2385
2386 syntax_map.reparse(language.clone(), &buffer);
2387
2388 reference_syntax_map.clear();
2389 reference_syntax_map.reparse(language.clone(), &buffer);
2390 }
2391
2392 for i in 0..operations {
2393 let i = operations - i - 1;
2394 buffer.undo();
2395 log::info!("undoing operation {}", i);
2396 log::info!("text:\n{}", buffer.text());
2397
2398 syntax_map.interpolate(&buffer);
2399 syntax_map.reparse(language.clone(), &buffer);
2400
2401 reference_syntax_map.clear();
2402 reference_syntax_map.reparse(language.clone(), &buffer);
2403 assert_eq!(
2404 syntax_map.layers(&buffer).len(),
2405 reference_syntax_map.layers(&buffer).len(),
2406 "wrong number of layers after undoing edit {i}"
2407 );
2408 }
2409
2410 let layers = syntax_map.layers(&buffer);
2411 let reference_layers = reference_syntax_map.layers(&buffer);
2412 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2413 {
2414 assert_eq!(edited_layer.node.to_sexp(), reference_layer.node.to_sexp());
2415 assert_eq!(edited_layer.node.range(), reference_layer.node.range());
2416 }
2417 }
2418
2419 fn check_interpolation(
2420 old_syntax_map: &SyntaxSnapshot,
2421 new_syntax_map: &SyntaxSnapshot,
2422 old_buffer: &BufferSnapshot,
2423 new_buffer: &BufferSnapshot,
2424 ) {
2425 let edits = new_buffer
2426 .edits_since::<usize>(&old_buffer.version())
2427 .collect::<Vec<_>>();
2428
2429 for (old_layer, new_layer) in old_syntax_map
2430 .layers
2431 .iter()
2432 .zip(new_syntax_map.layers.iter())
2433 {
2434 assert_eq!(old_layer.range, new_layer.range);
2435 let Some(old_tree) = old_layer.content.tree() else { continue };
2436 let Some(new_tree) = new_layer.content.tree() else { continue };
2437 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
2438 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
2439 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
2440 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
2441 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
2442 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
2443 check_node_edits(
2444 old_layer.depth,
2445 &old_layer.range,
2446 old_node,
2447 new_node,
2448 old_buffer,
2449 new_buffer,
2450 &edits,
2451 );
2452 }
2453
2454 fn check_node_edits(
2455 depth: usize,
2456 range: &Range<Anchor>,
2457 old_node: Node,
2458 new_node: Node,
2459 old_buffer: &BufferSnapshot,
2460 new_buffer: &BufferSnapshot,
2461 edits: &[text::Edit<usize>],
2462 ) {
2463 assert_eq!(old_node.kind(), new_node.kind());
2464
2465 let old_range = old_node.byte_range();
2466 let new_range = new_node.byte_range();
2467
2468 let is_edited = edits
2469 .iter()
2470 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
2471 if is_edited {
2472 assert!(
2473 new_node.has_changes(),
2474 concat!(
2475 "failed to mark node as edited.\n",
2476 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2477 "node kind: {}, old node range: {:?}, new node range: {:?}",
2478 ),
2479 depth,
2480 range.to_offset(old_buffer),
2481 range.to_offset(new_buffer),
2482 new_node.kind(),
2483 old_range,
2484 new_range,
2485 );
2486 }
2487
2488 if !new_node.has_changes() {
2489 assert_eq!(
2490 old_buffer
2491 .text_for_range(old_range.clone())
2492 .collect::<String>(),
2493 new_buffer
2494 .text_for_range(new_range.clone())
2495 .collect::<String>(),
2496 concat!(
2497 "mismatched text for node\n",
2498 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2499 "node kind: {}, old node range:{:?}, new node range:{:?}",
2500 ),
2501 depth,
2502 range.to_offset(old_buffer),
2503 range.to_offset(new_buffer),
2504 new_node.kind(),
2505 old_range,
2506 new_range,
2507 );
2508 }
2509
2510 for i in 0..new_node.child_count() {
2511 check_node_edits(
2512 depth,
2513 range,
2514 old_node.child(i).unwrap(),
2515 new_node.child(i).unwrap(),
2516 old_buffer,
2517 new_buffer,
2518 edits,
2519 )
2520 }
2521 }
2522 }
2523
2524 fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
2525 let registry = Arc::new(LanguageRegistry::test());
2526 registry.add(Arc::new(rust_lang()));
2527 registry.add(Arc::new(ruby_lang()));
2528 registry.add(Arc::new(html_lang()));
2529 registry.add(Arc::new(erb_lang()));
2530 registry.add(Arc::new(markdown_lang()));
2531 let language = registry
2532 .language_for_name(language_name)
2533 .now_or_never()
2534 .unwrap()
2535 .unwrap();
2536 let mut buffer = Buffer::new(0, 0, Default::default());
2537
2538 let mut mutated_syntax_map = SyntaxMap::new();
2539 mutated_syntax_map.set_language_registry(registry.clone());
2540 mutated_syntax_map.reparse(language.clone(), &buffer);
2541
2542 for (i, marked_string) in steps.into_iter().enumerate() {
2543 buffer.edit_via_marked_text(&marked_string.unindent());
2544
2545 // Reparse the syntax map
2546 mutated_syntax_map.interpolate(&buffer);
2547 mutated_syntax_map.reparse(language.clone(), &buffer);
2548
2549 // Create a second syntax map from scratch
2550 let mut reference_syntax_map = SyntaxMap::new();
2551 reference_syntax_map.set_language_registry(registry.clone());
2552 reference_syntax_map.reparse(language.clone(), &buffer);
2553
2554 // Compare the mutated syntax map to the new syntax map
2555 let mutated_layers = mutated_syntax_map.layers(&buffer);
2556 let reference_layers = reference_syntax_map.layers(&buffer);
2557 assert_eq!(
2558 mutated_layers.len(),
2559 reference_layers.len(),
2560 "wrong number of layers at step {i}"
2561 );
2562 for (edited_layer, reference_layer) in
2563 mutated_layers.into_iter().zip(reference_layers.into_iter())
2564 {
2565 assert_eq!(
2566 edited_layer.node.to_sexp(),
2567 reference_layer.node.to_sexp(),
2568 "different layer at step {i}"
2569 );
2570 assert_eq!(
2571 edited_layer.node.range(),
2572 reference_layer.node.range(),
2573 "different layer at step {i}"
2574 );
2575 }
2576 }
2577
2578 (buffer, mutated_syntax_map)
2579 }
2580
2581 fn html_lang() -> Language {
2582 Language::new(
2583 LanguageConfig {
2584 name: "HTML".into(),
2585 path_suffixes: vec!["html".to_string()],
2586 ..Default::default()
2587 },
2588 Some(tree_sitter_html::language()),
2589 )
2590 .with_highlights_query(
2591 r#"
2592 (tag_name) @tag
2593 (erroneous_end_tag_name) @tag
2594 (attribute_name) @property
2595 "#,
2596 )
2597 .unwrap()
2598 }
2599
2600 fn ruby_lang() -> Language {
2601 Language::new(
2602 LanguageConfig {
2603 name: "Ruby".into(),
2604 path_suffixes: vec!["rb".to_string()],
2605 ..Default::default()
2606 },
2607 Some(tree_sitter_ruby::language()),
2608 )
2609 .with_highlights_query(
2610 r#"
2611 ["if" "do" "else" "end"] @keyword
2612 (instance_variable) @ivar
2613 "#,
2614 )
2615 .unwrap()
2616 }
2617
2618 fn erb_lang() -> Language {
2619 Language::new(
2620 LanguageConfig {
2621 name: "ERB".into(),
2622 path_suffixes: vec!["erb".to_string()],
2623 ..Default::default()
2624 },
2625 Some(tree_sitter_embedded_template::language()),
2626 )
2627 .with_highlights_query(
2628 r#"
2629 ["<%" "%>"] @keyword
2630 "#,
2631 )
2632 .unwrap()
2633 .with_injection_query(
2634 r#"
2635 ((code) @content
2636 (#set! "language" "ruby")
2637 (#set! "combined"))
2638
2639 ((content) @content
2640 (#set! "language" "html")
2641 (#set! "combined"))
2642 "#,
2643 )
2644 .unwrap()
2645 }
2646
2647 fn rust_lang() -> Language {
2648 Language::new(
2649 LanguageConfig {
2650 name: "Rust".into(),
2651 path_suffixes: vec!["rs".to_string()],
2652 ..Default::default()
2653 },
2654 Some(tree_sitter_rust::language()),
2655 )
2656 .with_highlights_query(
2657 r#"
2658 (field_identifier) @field
2659 (struct_expression) @struct
2660 "#,
2661 )
2662 .unwrap()
2663 .with_injection_query(
2664 r#"
2665 (macro_invocation
2666 (token_tree) @content
2667 (#set! "language" "rust"))
2668 "#,
2669 )
2670 .unwrap()
2671 }
2672
2673 fn markdown_lang() -> Language {
2674 Language::new(
2675 LanguageConfig {
2676 name: "Markdown".into(),
2677 path_suffixes: vec!["md".into()],
2678 ..Default::default()
2679 },
2680 Some(tree_sitter_markdown::language()),
2681 )
2682 .with_injection_query(
2683 r#"
2684 (fenced_code_block
2685 (info_string
2686 (language) @language)
2687 (code_fence_content) @content)
2688 "#,
2689 )
2690 .unwrap()
2691 }
2692
2693 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2694 let start = buffer.as_rope().to_string().find(text).unwrap();
2695 start..start + text.len()
2696 }
2697
2698 fn assert_layers_for_range(
2699 syntax_map: &SyntaxMap,
2700 buffer: &BufferSnapshot,
2701 range: Range<Point>,
2702 expected_layers: &[&str],
2703 ) {
2704 let layers = syntax_map
2705 .layers_for_range(range, &buffer)
2706 .collect::<Vec<_>>();
2707 assert_eq!(
2708 layers.len(),
2709 expected_layers.len(),
2710 "wrong number of layers"
2711 );
2712 for (i, (SyntaxLayerInfo { node, .. }, expected_s_exp)) in
2713 layers.iter().zip(expected_layers.iter()).enumerate()
2714 {
2715 let actual_s_exp = node.to_sexp();
2716 assert!(
2717 string_contains_sequence(
2718 &actual_s_exp,
2719 &expected_s_exp.split("...").collect::<Vec<_>>()
2720 ),
2721 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2722 );
2723 }
2724 }
2725
2726 fn assert_capture_ranges(
2727 syntax_map: &SyntaxMap,
2728 buffer: &BufferSnapshot,
2729 highlight_query_capture_names: &[&str],
2730 marked_string: &str,
2731 ) {
2732 let mut actual_ranges = Vec::<Range<usize>>::new();
2733 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2734 grammar.highlights_query.as_ref()
2735 });
2736 let queries = captures
2737 .grammars()
2738 .iter()
2739 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2740 .collect::<Vec<_>>();
2741 for capture in captures {
2742 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2743 if highlight_query_capture_names.contains(&name.as_str()) {
2744 actual_ranges.push(capture.node.byte_range());
2745 }
2746 }
2747
2748 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2749 assert_eq!(text, buffer.text());
2750 assert_eq!(actual_ranges, expected_ranges);
2751 }
2752
2753 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2754 let mut last_part_end = 0;
2755 for part in parts {
2756 if let Some(start_ix) = text[last_part_end..].find(part) {
2757 last_part_end = start_ix + part.len();
2758 } else {
2759 return false;
2760 }
2761 }
2762 true
2763 }
2764}