1use crate::{Grammar, InjectionConfig, Language, LanguageRegistry};
2use collections::HashMap;
3use futures::FutureExt;
4use lazy_static::lazy_static;
5use parking_lot::Mutex;
6use std::{
7 borrow::Cow,
8 cell::RefCell,
9 cmp::{self, Ordering, Reverse},
10 collections::BinaryHeap,
11 iter,
12 ops::{Deref, DerefMut, Range},
13 sync::Arc,
14};
15use sum_tree::{Bias, SeekTarget, SumTree};
16use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
17use tree_sitter::{
18 Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree,
19};
20
21thread_local! {
22 static PARSER: RefCell<Parser> = RefCell::new(Parser::new());
23}
24
25lazy_static! {
26 static ref QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Default::default();
27}
28
29#[derive(Default)]
30pub struct SyntaxMap {
31 snapshot: SyntaxSnapshot,
32 language_registry: Option<Arc<LanguageRegistry>>,
33}
34
35#[derive(Clone, Default)]
36pub struct SyntaxSnapshot {
37 layers: SumTree<SyntaxLayer>,
38 parsed_version: clock::Global,
39 interpolated_version: clock::Global,
40 language_registry_version: usize,
41}
42
43#[derive(Default)]
44pub struct SyntaxMapCaptures<'a> {
45 layers: Vec<SyntaxMapCapturesLayer<'a>>,
46 active_layer_count: usize,
47 grammars: Vec<&'a Grammar>,
48}
49
50#[derive(Default)]
51pub struct SyntaxMapMatches<'a> {
52 layers: Vec<SyntaxMapMatchesLayer<'a>>,
53 active_layer_count: usize,
54 grammars: Vec<&'a Grammar>,
55}
56
57#[derive(Debug)]
58pub struct SyntaxMapCapture<'a> {
59 pub depth: usize,
60 pub node: Node<'a>,
61 pub index: u32,
62 pub grammar_index: usize,
63}
64
65#[derive(Debug)]
66pub struct SyntaxMapMatch<'a> {
67 pub depth: usize,
68 pub pattern_index: usize,
69 pub captures: &'a [QueryCapture<'a>],
70 pub grammar_index: usize,
71}
72
73struct SyntaxMapCapturesLayer<'a> {
74 depth: usize,
75 captures: QueryCaptures<'a, 'a, TextProvider<'a>>,
76 next_capture: Option<QueryCapture<'a>>,
77 grammar_index: usize,
78 _query_cursor: QueryCursorHandle,
79}
80
81struct SyntaxMapMatchesLayer<'a> {
82 depth: usize,
83 next_pattern_index: usize,
84 next_captures: Vec<QueryCapture<'a>>,
85 has_next: bool,
86 matches: QueryMatches<'a, 'a, TextProvider<'a>>,
87 grammar_index: usize,
88 _query_cursor: QueryCursorHandle,
89}
90
91#[derive(Clone)]
92struct SyntaxLayer {
93 depth: usize,
94 range: Range<Anchor>,
95 content: SyntaxLayerContent,
96}
97
98#[derive(Clone)]
99enum SyntaxLayerContent {
100 Parsed {
101 tree: tree_sitter::Tree,
102 language: Arc<Language>,
103 },
104 Pending {
105 language_name: Arc<str>,
106 },
107}
108
109impl SyntaxLayerContent {
110 fn language_id(&self) -> Option<usize> {
111 match self {
112 SyntaxLayerContent::Parsed { language, .. } => language.id(),
113 SyntaxLayerContent::Pending { .. } => None,
114 }
115 }
116
117 fn tree(&self) -> Option<&Tree> {
118 match self {
119 SyntaxLayerContent::Parsed { tree, .. } => Some(tree),
120 SyntaxLayerContent::Pending { .. } => None,
121 }
122 }
123}
124
125#[derive(Debug)]
126pub struct SyntaxLayerInfo<'a> {
127 pub depth: usize,
128 pub language: &'a Arc<Language>,
129 tree: &'a Tree,
130 offset: (usize, tree_sitter::Point),
131}
132
133#[derive(Clone)]
134pub struct OwnedSyntaxLayerInfo {
135 pub depth: usize,
136 pub language: Arc<Language>,
137 tree: tree_sitter::Tree,
138 offset: (usize, tree_sitter::Point),
139}
140
141#[derive(Debug, Clone)]
142struct SyntaxLayerSummary {
143 min_depth: usize,
144 max_depth: usize,
145 range: Range<Anchor>,
146 last_layer_range: Range<Anchor>,
147 last_layer_language: Option<usize>,
148 contains_unknown_injections: bool,
149}
150
151#[derive(Clone, Debug)]
152struct SyntaxLayerPosition {
153 depth: usize,
154 range: Range<Anchor>,
155 language: Option<usize>,
156}
157
158#[derive(Clone, Debug)]
159struct ChangeStartPosition {
160 depth: usize,
161 position: Anchor,
162}
163
164#[derive(Clone, Debug)]
165struct SyntaxLayerPositionBeforeChange {
166 position: SyntaxLayerPosition,
167 change: ChangeStartPosition,
168}
169
170struct ParseStep {
171 depth: usize,
172 language: ParseStepLanguage,
173 range: Range<Anchor>,
174 included_ranges: Vec<tree_sitter::Range>,
175 mode: ParseMode,
176}
177
178#[derive(Debug)]
179enum ParseStepLanguage {
180 Loaded { language: Arc<Language> },
181 Pending { name: Arc<str> },
182}
183
184impl ParseStepLanguage {
185 fn id(&self) -> Option<usize> {
186 match self {
187 ParseStepLanguage::Loaded { language } => language.id(),
188 ParseStepLanguage::Pending { .. } => None,
189 }
190 }
191}
192
193enum ParseMode {
194 Single,
195 Combined {
196 parent_layer_range: Range<usize>,
197 parent_layer_changed_ranges: Vec<Range<usize>>,
198 },
199}
200
201#[derive(Debug, PartialEq, Eq)]
202struct ChangedRegion {
203 depth: usize,
204 range: Range<Anchor>,
205}
206
207#[derive(Default)]
208struct ChangeRegionSet(Vec<ChangedRegion>);
209
210struct TextProvider<'a>(&'a Rope);
211
212struct ByteChunks<'a>(text::Chunks<'a>);
213
214struct QueryCursorHandle(Option<QueryCursor>);
215
216impl SyntaxMap {
217 pub fn new() -> Self {
218 Self::default()
219 }
220
221 pub fn set_language_registry(&mut self, registry: Arc<LanguageRegistry>) {
222 self.language_registry = Some(registry);
223 }
224
225 pub fn snapshot(&self) -> SyntaxSnapshot {
226 self.snapshot.clone()
227 }
228
229 pub fn language_registry(&self) -> Option<Arc<LanguageRegistry>> {
230 self.language_registry.clone()
231 }
232
233 pub fn interpolate(&mut self, text: &BufferSnapshot) {
234 self.snapshot.interpolate(text);
235 }
236
237 #[cfg(test)]
238 pub fn reparse(&mut self, language: Arc<Language>, text: &BufferSnapshot) {
239 self.snapshot
240 .reparse(text, self.language_registry.clone(), language);
241 }
242
243 pub fn did_parse(&mut self, snapshot: SyntaxSnapshot) {
244 self.snapshot = snapshot;
245 }
246
247 pub fn clear(&mut self) {
248 self.snapshot = SyntaxSnapshot::default();
249 }
250}
251
252impl SyntaxSnapshot {
253 pub fn is_empty(&self) -> bool {
254 self.layers.is_empty()
255 }
256
257 fn interpolate(&mut self, text: &BufferSnapshot) {
258 let edits = text
259 .anchored_edits_since::<(usize, Point)>(&self.interpolated_version)
260 .collect::<Vec<_>>();
261 self.interpolated_version = text.version().clone();
262
263 if edits.is_empty() {
264 return;
265 }
266
267 let mut layers = SumTree::new();
268 let mut first_edit_ix_for_depth = 0;
269 let mut prev_depth = 0;
270 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
271 cursor.next(text);
272
273 'outer: loop {
274 let depth = cursor.end(text).max_depth;
275 if depth > prev_depth {
276 first_edit_ix_for_depth = 0;
277 prev_depth = depth;
278 }
279
280 // Preserve any layers at this depth that precede the first edit.
281 if let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) {
282 let target = ChangeStartPosition {
283 depth,
284 position: edit_range.start,
285 };
286 if target.cmp(&cursor.start(), text).is_gt() {
287 let slice = cursor.slice(&target, Bias::Left, text);
288 layers.push_tree(slice, text);
289 }
290 }
291 // If this layer follows all of the edits, then preserve it and any
292 // subsequent layers at this same depth.
293 else if cursor.item().is_some() {
294 let slice = cursor.slice(
295 &SyntaxLayerPosition {
296 depth: depth + 1,
297 range: Anchor::MIN..Anchor::MAX,
298 language: None,
299 },
300 Bias::Left,
301 text,
302 );
303 layers.push_tree(slice, text);
304 continue;
305 };
306
307 let Some(layer) = cursor.item() else { break };
308 let (start_byte, start_point) = layer.range.start.summary::<(usize, Point)>(text);
309
310 // Ignore edits that end before the start of this layer, and don't consider them
311 // for any subsequent layers at this same depth.
312 loop {
313 let Some((_, edit_range)) = edits.get(first_edit_ix_for_depth) else { continue 'outer };
314 if edit_range.end.cmp(&layer.range.start, text).is_le() {
315 first_edit_ix_for_depth += 1;
316 } else {
317 break;
318 }
319 }
320
321 let mut layer = layer.clone();
322 if let SyntaxLayerContent::Parsed { tree, .. } = &mut layer.content {
323 for (edit, edit_range) in &edits[first_edit_ix_for_depth..] {
324 // Ignore any edits that follow this layer.
325 if edit_range.start.cmp(&layer.range.end, text).is_ge() {
326 break;
327 }
328
329 // Apply any edits that intersect this layer to the layer's syntax tree.
330 let tree_edit = if edit_range.start.cmp(&layer.range.start, text).is_ge() {
331 tree_sitter::InputEdit {
332 start_byte: edit.new.start.0 - start_byte,
333 old_end_byte: edit.new.start.0 - start_byte
334 + (edit.old.end.0 - edit.old.start.0),
335 new_end_byte: edit.new.end.0 - start_byte,
336 start_position: (edit.new.start.1 - start_point).to_ts_point(),
337 old_end_position: (edit.new.start.1 - start_point
338 + (edit.old.end.1 - edit.old.start.1))
339 .to_ts_point(),
340 new_end_position: (edit.new.end.1 - start_point).to_ts_point(),
341 }
342 } else {
343 let node = tree.root_node();
344 tree_sitter::InputEdit {
345 start_byte: 0,
346 old_end_byte: node.end_byte(),
347 new_end_byte: 0,
348 start_position: Default::default(),
349 old_end_position: node.end_position(),
350 new_end_position: Default::default(),
351 }
352 };
353
354 tree.edit(&tree_edit);
355 }
356
357 debug_assert!(
358 tree.root_node().end_byte() <= text.len(),
359 "tree's size {}, is larger than text size {}",
360 tree.root_node().end_byte(),
361 text.len(),
362 );
363 }
364
365 layers.push(layer, text);
366 cursor.next(text);
367 }
368
369 layers.push_tree(cursor.suffix(&text), &text);
370 drop(cursor);
371 self.layers = layers;
372 }
373
374 pub fn reparse(
375 &mut self,
376 text: &BufferSnapshot,
377 registry: Option<Arc<LanguageRegistry>>,
378 root_language: Arc<Language>,
379 ) {
380 let edit_ranges = text
381 .edits_since::<usize>(&self.parsed_version)
382 .map(|edit| edit.new)
383 .collect::<Vec<_>>();
384 self.reparse_with_ranges(text, root_language.clone(), edit_ranges, registry.as_ref());
385
386 if let Some(registry) = registry {
387 if registry.version() != self.language_registry_version {
388 let mut resolved_injection_ranges = Vec::new();
389 let mut cursor = self
390 .layers
391 .filter::<_, ()>(|summary| summary.contains_unknown_injections);
392 cursor.next(text);
393 while let Some(layer) = cursor.item() {
394 let SyntaxLayerContent::Pending { language_name } = &layer.content else { unreachable!() };
395 if registry
396 .language_for_name_or_extension(language_name)
397 .now_or_never()
398 .and_then(|language| language.ok())
399 .is_some()
400 {
401 resolved_injection_ranges.push(layer.range.to_offset(text));
402 }
403
404 cursor.next(text);
405 }
406 drop(cursor);
407
408 if !resolved_injection_ranges.is_empty() {
409 self.reparse_with_ranges(
410 text,
411 root_language,
412 resolved_injection_ranges,
413 Some(®istry),
414 );
415 }
416 self.language_registry_version = registry.version();
417 }
418 }
419 }
420
421 fn reparse_with_ranges(
422 &mut self,
423 text: &BufferSnapshot,
424 root_language: Arc<Language>,
425 invalidated_ranges: Vec<Range<usize>>,
426 registry: Option<&Arc<LanguageRegistry>>,
427 ) {
428 let max_depth = self.layers.summary().max_depth;
429 let mut cursor = self.layers.cursor::<SyntaxLayerSummary>();
430 cursor.next(&text);
431 let mut layers = SumTree::new();
432
433 let mut changed_regions = ChangeRegionSet::default();
434 let mut queue = BinaryHeap::new();
435 let mut combined_injection_ranges = HashMap::default();
436 queue.push(ParseStep {
437 depth: 0,
438 language: ParseStepLanguage::Loaded {
439 language: root_language,
440 },
441 included_ranges: vec![tree_sitter::Range {
442 start_byte: 0,
443 end_byte: text.len(),
444 start_point: Point::zero().to_ts_point(),
445 end_point: text.max_point().to_ts_point(),
446 }],
447 range: Anchor::MIN..Anchor::MAX,
448 mode: ParseMode::Single,
449 });
450
451 loop {
452 let step = queue.pop();
453 let position = if let Some(step) = &step {
454 SyntaxLayerPosition {
455 depth: step.depth,
456 range: step.range.clone(),
457 language: step.language.id(),
458 }
459 } else {
460 SyntaxLayerPosition {
461 depth: max_depth + 1,
462 range: Anchor::MAX..Anchor::MAX,
463 language: None,
464 }
465 };
466
467 let mut done = cursor.item().is_none();
468 while !done && position.cmp(&cursor.end(text), &text).is_gt() {
469 done = true;
470
471 let bounded_position = SyntaxLayerPositionBeforeChange {
472 position: position.clone(),
473 change: changed_regions.start_position(),
474 };
475 if bounded_position.cmp(&cursor.start(), &text).is_gt() {
476 let slice = cursor.slice(&bounded_position, Bias::Left, text);
477 if !slice.is_empty() {
478 layers.push_tree(slice, &text);
479 if changed_regions.prune(cursor.end(text), text) {
480 done = false;
481 }
482 }
483 }
484
485 while position.cmp(&cursor.end(text), text).is_gt() {
486 let Some(layer) = cursor.item() else { break };
487
488 if changed_regions.intersects(&layer, text) {
489 changed_regions.insert(
490 ChangedRegion {
491 depth: layer.depth + 1,
492 range: layer.range.clone(),
493 },
494 text,
495 );
496 } else {
497 layers.push(layer.clone(), text);
498 }
499
500 cursor.next(text);
501 if changed_regions.prune(cursor.end(text), text) {
502 done = false;
503 }
504 }
505 }
506
507 let Some(step) = step else { break };
508 let (step_start_byte, step_start_point) =
509 step.range.start.summary::<(usize, Point)>(text);
510 let step_end_byte = step.range.end.to_offset(text);
511
512 let mut old_layer = cursor.item();
513 if let Some(layer) = old_layer {
514 if layer.range.to_offset(text) == (step_start_byte..step_end_byte)
515 && layer.content.language_id() == step.language.id()
516 {
517 cursor.next(&text);
518 } else {
519 old_layer = None;
520 }
521 }
522
523 let content = match step.language {
524 ParseStepLanguage::Loaded { language } => {
525 let Some(grammar) = language.grammar() else { continue };
526 let tree;
527 let changed_ranges;
528
529 let mut included_ranges = step.included_ranges;
530 for range in &mut included_ranges {
531 range.start_byte -= step_start_byte;
532 range.end_byte -= step_start_byte;
533 range.start_point = (Point::from_ts_point(range.start_point)
534 - step_start_point)
535 .to_ts_point();
536 range.end_point = (Point::from_ts_point(range.end_point)
537 - step_start_point)
538 .to_ts_point();
539 }
540
541 if let Some(SyntaxLayerContent::Parsed { tree: old_tree, .. }) =
542 old_layer.map(|layer| &layer.content)
543 {
544 if let ParseMode::Combined {
545 mut parent_layer_changed_ranges,
546 ..
547 } = step.mode
548 {
549 for range in &mut parent_layer_changed_ranges {
550 range.start -= step_start_byte;
551 range.end -= step_start_byte;
552 }
553
554 included_ranges = splice_included_ranges(
555 old_tree.included_ranges(),
556 &parent_layer_changed_ranges,
557 &included_ranges,
558 );
559 }
560
561 tree = parse_text(
562 grammar,
563 text.as_rope(),
564 step_start_byte,
565 included_ranges,
566 Some(old_tree.clone()),
567 );
568 changed_ranges = join_ranges(
569 invalidated_ranges.iter().cloned().filter(|range| {
570 range.start <= step_end_byte && range.end >= step_start_byte
571 }),
572 old_tree.changed_ranges(&tree).map(|r| {
573 step_start_byte + r.start_byte..step_start_byte + r.end_byte
574 }),
575 );
576 } else {
577 tree = parse_text(
578 grammar,
579 text.as_rope(),
580 step_start_byte,
581 included_ranges,
582 None,
583 );
584 changed_ranges = vec![step_start_byte..step_end_byte];
585 }
586
587 if let (Some((config, registry)), false) = (
588 grammar.injection_config.as_ref().zip(registry.as_ref()),
589 changed_ranges.is_empty(),
590 ) {
591 for range in &changed_ranges {
592 changed_regions.insert(
593 ChangedRegion {
594 depth: step.depth + 1,
595 range: text.anchor_before(range.start)
596 ..text.anchor_after(range.end),
597 },
598 text,
599 );
600 }
601 get_injections(
602 config,
603 text,
604 tree.root_node_with_offset(
605 step_start_byte,
606 step_start_point.to_ts_point(),
607 ),
608 registry,
609 step.depth + 1,
610 &changed_ranges,
611 &mut combined_injection_ranges,
612 &mut queue,
613 );
614 }
615
616 SyntaxLayerContent::Parsed { tree, language }
617 }
618 ParseStepLanguage::Pending { name } => SyntaxLayerContent::Pending {
619 language_name: name,
620 },
621 };
622
623 layers.push(
624 SyntaxLayer {
625 depth: step.depth,
626 range: step.range,
627 content,
628 },
629 &text,
630 );
631 }
632
633 drop(cursor);
634 self.layers = layers;
635 self.interpolated_version = text.version.clone();
636 self.parsed_version = text.version.clone();
637 #[cfg(debug_assertions)]
638 self.check_invariants(text);
639 }
640
641 #[cfg(debug_assertions)]
642 fn check_invariants(&self, text: &BufferSnapshot) {
643 let mut max_depth = 0;
644 let mut prev_range: Option<Range<Anchor>> = None;
645 for layer in self.layers.iter() {
646 if layer.depth == max_depth {
647 if let Some(prev_range) = prev_range {
648 match layer.range.start.cmp(&prev_range.start, text) {
649 Ordering::Less => panic!("layers out of order"),
650 Ordering::Equal => {
651 assert!(layer.range.end.cmp(&prev_range.end, text).is_ge())
652 }
653 Ordering::Greater => {}
654 }
655 }
656 } else if layer.depth < max_depth {
657 panic!("layers out of order")
658 }
659 max_depth = layer.depth;
660 prev_range = Some(layer.range.clone());
661 }
662 }
663
664 pub fn single_tree_captures<'a>(
665 range: Range<usize>,
666 text: &'a Rope,
667 tree: &'a Tree,
668 language: &'a Arc<Language>,
669 query: fn(&Grammar) -> Option<&Query>,
670 ) -> SyntaxMapCaptures<'a> {
671 SyntaxMapCaptures::new(
672 range.clone(),
673 text,
674 [SyntaxLayerInfo {
675 language,
676 tree,
677 depth: 0,
678 offset: (0, tree_sitter::Point::new(0, 0)),
679 }]
680 .into_iter(),
681 query,
682 )
683 }
684
685 pub fn captures<'a>(
686 &'a self,
687 range: Range<usize>,
688 buffer: &'a BufferSnapshot,
689 query: fn(&Grammar) -> Option<&Query>,
690 ) -> SyntaxMapCaptures {
691 SyntaxMapCaptures::new(
692 range.clone(),
693 buffer.as_rope(),
694 self.layers_for_range(range, buffer).into_iter(),
695 query,
696 )
697 }
698
699 pub fn matches<'a>(
700 &'a self,
701 range: Range<usize>,
702 buffer: &'a BufferSnapshot,
703 query: fn(&Grammar) -> Option<&Query>,
704 ) -> SyntaxMapMatches {
705 SyntaxMapMatches::new(
706 range.clone(),
707 buffer.as_rope(),
708 self.layers_for_range(range, buffer).into_iter(),
709 query,
710 )
711 }
712
713 #[cfg(test)]
714 pub fn layers<'a>(&'a self, buffer: &'a BufferSnapshot) -> Vec<SyntaxLayerInfo> {
715 self.layers_for_range(0..buffer.len(), buffer).collect()
716 }
717
718 pub fn layers_for_range<'a, T: ToOffset>(
719 &'a self,
720 range: Range<T>,
721 buffer: &'a BufferSnapshot,
722 ) -> impl 'a + Iterator<Item = SyntaxLayerInfo> {
723 let start = buffer.anchor_before(range.start.to_offset(buffer));
724 let end = buffer.anchor_after(range.end.to_offset(buffer));
725
726 let mut cursor = self.layers.filter::<_, ()>(move |summary| {
727 if summary.max_depth > summary.min_depth {
728 true
729 } else {
730 let is_before_start = summary.range.end.cmp(&start, buffer).is_lt();
731 let is_after_end = summary.range.start.cmp(&end, buffer).is_gt();
732 !is_before_start && !is_after_end
733 }
734 });
735
736 cursor.next(buffer);
737 iter::from_fn(move || {
738 while let Some(layer) = cursor.item() {
739 if let SyntaxLayerContent::Parsed { tree, language } = &layer.content {
740 let info = SyntaxLayerInfo {
741 tree,
742 language,
743 depth: layer.depth,
744 offset: (
745 layer.range.start.to_offset(buffer),
746 layer.range.start.to_point(buffer).to_ts_point(),
747 ),
748 };
749 cursor.next(buffer);
750 return Some(info);
751 } else {
752 cursor.next(buffer);
753 }
754 }
755 None
756 })
757 }
758
759 pub fn contains_unknown_injections(&self) -> bool {
760 self.layers.summary().contains_unknown_injections
761 }
762
763 pub fn language_registry_version(&self) -> usize {
764 self.language_registry_version
765 }
766}
767
768impl<'a> SyntaxMapCaptures<'a> {
769 fn new(
770 range: Range<usize>,
771 text: &'a Rope,
772 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
773 query: fn(&Grammar) -> Option<&Query>,
774 ) -> Self {
775 let mut result = Self {
776 layers: Vec::new(),
777 grammars: Vec::new(),
778 active_layer_count: 0,
779 };
780 for layer in layers {
781 let grammar = match &layer.language.grammar {
782 Some(grammar) => grammar,
783 None => continue,
784 };
785 let query = match query(&grammar) {
786 Some(query) => query,
787 None => continue,
788 };
789
790 let mut query_cursor = QueryCursorHandle::new();
791
792 // TODO - add a Tree-sitter API to remove the need for this.
793 let cursor = unsafe {
794 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
795 };
796
797 cursor.set_byte_range(range.clone());
798 let captures = cursor.captures(query, layer.node(), TextProvider(text));
799 let grammar_index = result
800 .grammars
801 .iter()
802 .position(|g| g.id == grammar.id())
803 .unwrap_or_else(|| {
804 result.grammars.push(grammar);
805 result.grammars.len() - 1
806 });
807 let mut layer = SyntaxMapCapturesLayer {
808 depth: layer.depth,
809 grammar_index,
810 next_capture: None,
811 captures,
812 _query_cursor: query_cursor,
813 };
814
815 layer.advance();
816 if layer.next_capture.is_some() {
817 let key = layer.sort_key();
818 let ix = match result.layers[..result.active_layer_count]
819 .binary_search_by_key(&key, |layer| layer.sort_key())
820 {
821 Ok(ix) | Err(ix) => ix,
822 };
823 result.layers.insert(ix, layer);
824 result.active_layer_count += 1;
825 } else {
826 result.layers.push(layer);
827 }
828 }
829
830 result
831 }
832
833 pub fn grammars(&self) -> &[&'a Grammar] {
834 &self.grammars
835 }
836
837 pub fn peek(&self) -> Option<SyntaxMapCapture<'a>> {
838 let layer = self.layers[..self.active_layer_count].first()?;
839 let capture = layer.next_capture?;
840 Some(SyntaxMapCapture {
841 depth: layer.depth,
842 grammar_index: layer.grammar_index,
843 index: capture.index,
844 node: capture.node,
845 })
846 }
847
848 pub fn advance(&mut self) -> bool {
849 let layer = if let Some(layer) = self.layers[..self.active_layer_count].first_mut() {
850 layer
851 } else {
852 return false;
853 };
854
855 layer.advance();
856 if layer.next_capture.is_some() {
857 let key = layer.sort_key();
858 let i = 1 + self.layers[1..self.active_layer_count]
859 .iter()
860 .position(|later_layer| key < later_layer.sort_key())
861 .unwrap_or(self.active_layer_count - 1);
862 self.layers[0..i].rotate_left(1);
863 } else {
864 self.layers[0..self.active_layer_count].rotate_left(1);
865 self.active_layer_count -= 1;
866 }
867
868 true
869 }
870
871 pub fn set_byte_range(&mut self, range: Range<usize>) {
872 for layer in &mut self.layers {
873 layer.captures.set_byte_range(range.clone());
874 if let Some(capture) = &layer.next_capture {
875 if capture.node.end_byte() > range.start {
876 continue;
877 }
878 }
879 layer.advance();
880 }
881 self.layers.sort_unstable_by_key(|layer| layer.sort_key());
882 self.active_layer_count = self
883 .layers
884 .iter()
885 .position(|layer| layer.next_capture.is_none())
886 .unwrap_or(self.layers.len());
887 }
888}
889
890impl<'a> SyntaxMapMatches<'a> {
891 fn new(
892 range: Range<usize>,
893 text: &'a Rope,
894 layers: impl Iterator<Item = SyntaxLayerInfo<'a>>,
895 query: fn(&Grammar) -> Option<&Query>,
896 ) -> Self {
897 let mut result = Self::default();
898 for layer in layers {
899 let grammar = match &layer.language.grammar {
900 Some(grammar) => grammar,
901 None => continue,
902 };
903 let query = match query(&grammar) {
904 Some(query) => query,
905 None => continue,
906 };
907
908 let mut query_cursor = QueryCursorHandle::new();
909
910 // TODO - add a Tree-sitter API to remove the need for this.
911 let cursor = unsafe {
912 std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
913 };
914
915 cursor.set_byte_range(range.clone());
916 let matches = cursor.matches(query, layer.node(), TextProvider(text));
917 let grammar_index = result
918 .grammars
919 .iter()
920 .position(|g| g.id == grammar.id())
921 .unwrap_or_else(|| {
922 result.grammars.push(grammar);
923 result.grammars.len() - 1
924 });
925 let mut layer = SyntaxMapMatchesLayer {
926 depth: layer.depth,
927 grammar_index,
928 matches,
929 next_pattern_index: 0,
930 next_captures: Vec::new(),
931 has_next: false,
932 _query_cursor: query_cursor,
933 };
934
935 layer.advance();
936 if layer.has_next {
937 let key = layer.sort_key();
938 let ix = match result.layers[..result.active_layer_count]
939 .binary_search_by_key(&key, |layer| layer.sort_key())
940 {
941 Ok(ix) | Err(ix) => ix,
942 };
943 result.layers.insert(ix, layer);
944 result.active_layer_count += 1;
945 } else {
946 result.layers.push(layer);
947 }
948 }
949 result
950 }
951
952 pub fn grammars(&self) -> &[&'a Grammar] {
953 &self.grammars
954 }
955
956 pub fn peek(&self) -> Option<SyntaxMapMatch> {
957 let layer = self.layers.first()?;
958 if !layer.has_next {
959 return None;
960 }
961 Some(SyntaxMapMatch {
962 depth: layer.depth,
963 grammar_index: layer.grammar_index,
964 pattern_index: layer.next_pattern_index,
965 captures: &layer.next_captures,
966 })
967 }
968
969 pub fn advance(&mut self) -> bool {
970 let layer = if let Some(layer) = self.layers.first_mut() {
971 layer
972 } else {
973 return false;
974 };
975
976 layer.advance();
977 if layer.has_next {
978 let key = layer.sort_key();
979 let i = 1 + self.layers[1..self.active_layer_count]
980 .iter()
981 .position(|later_layer| key < later_layer.sort_key())
982 .unwrap_or(self.active_layer_count - 1);
983 self.layers[0..i].rotate_left(1);
984 } else {
985 self.layers[0..self.active_layer_count].rotate_left(1);
986 self.active_layer_count -= 1;
987 }
988
989 true
990 }
991}
992
993impl<'a> SyntaxMapCapturesLayer<'a> {
994 fn advance(&mut self) {
995 self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
996 }
997
998 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
999 if let Some(capture) = &self.next_capture {
1000 let range = capture.node.byte_range();
1001 (range.start, Reverse(range.end), self.depth)
1002 } else {
1003 (usize::MAX, Reverse(0), usize::MAX)
1004 }
1005 }
1006}
1007
1008impl<'a> SyntaxMapMatchesLayer<'a> {
1009 fn advance(&mut self) {
1010 if let Some(mat) = self.matches.next() {
1011 self.next_captures.clear();
1012 self.next_captures.extend_from_slice(&mat.captures);
1013 self.next_pattern_index = mat.pattern_index;
1014 self.has_next = true;
1015 } else {
1016 self.has_next = false;
1017 }
1018 }
1019
1020 fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
1021 if self.has_next {
1022 let captures = &self.next_captures;
1023 if let Some((first, last)) = captures.first().zip(captures.last()) {
1024 return (
1025 first.node.start_byte(),
1026 Reverse(last.node.end_byte()),
1027 self.depth,
1028 );
1029 }
1030 }
1031 (usize::MAX, Reverse(0), usize::MAX)
1032 }
1033}
1034
1035impl<'a> Iterator for SyntaxMapCaptures<'a> {
1036 type Item = SyntaxMapCapture<'a>;
1037
1038 fn next(&mut self) -> Option<Self::Item> {
1039 let result = self.peek();
1040 self.advance();
1041 result
1042 }
1043}
1044
1045fn join_ranges(
1046 a: impl Iterator<Item = Range<usize>>,
1047 b: impl Iterator<Item = Range<usize>>,
1048) -> Vec<Range<usize>> {
1049 let mut result = Vec::<Range<usize>>::new();
1050 let mut a = a.peekable();
1051 let mut b = b.peekable();
1052 loop {
1053 let range = match (a.peek(), b.peek()) {
1054 (Some(range_a), Some(range_b)) => {
1055 if range_a.start < range_b.start {
1056 a.next().unwrap()
1057 } else {
1058 b.next().unwrap()
1059 }
1060 }
1061 (None, Some(_)) => b.next().unwrap(),
1062 (Some(_), None) => a.next().unwrap(),
1063 (None, None) => break,
1064 };
1065
1066 if let Some(last) = result.last_mut() {
1067 if range.start <= last.end {
1068 last.end = last.end.max(range.end);
1069 continue;
1070 }
1071 }
1072 result.push(range);
1073 }
1074 result
1075}
1076
1077fn parse_text(
1078 grammar: &Grammar,
1079 text: &Rope,
1080 start_byte: usize,
1081 ranges: Vec<tree_sitter::Range>,
1082 old_tree: Option<Tree>,
1083) -> Tree {
1084 PARSER.with(|parser| {
1085 let mut parser = parser.borrow_mut();
1086 let mut chunks = text.chunks_in_range(start_byte..text.len());
1087 parser
1088 .set_included_ranges(&ranges)
1089 .expect("overlapping ranges");
1090 parser
1091 .set_language(grammar.ts_language)
1092 .expect("incompatible grammar");
1093 parser
1094 .parse_with(
1095 &mut move |offset, _| {
1096 chunks.seek(start_byte + offset);
1097 chunks.next().unwrap_or("").as_bytes()
1098 },
1099 old_tree.as_ref(),
1100 )
1101 .expect("invalid language")
1102 })
1103}
1104
1105fn get_injections(
1106 config: &InjectionConfig,
1107 text: &BufferSnapshot,
1108 node: Node,
1109 language_registry: &Arc<LanguageRegistry>,
1110 depth: usize,
1111 changed_ranges: &[Range<usize>],
1112 combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
1113 queue: &mut BinaryHeap<ParseStep>,
1114) {
1115 let mut query_cursor = QueryCursorHandle::new();
1116 let mut prev_match = None;
1117
1118 // Ensure that a `ParseStep` is created for every combined injection language, even
1119 // if there currently no matches for that injection.
1120 combined_injection_ranges.clear();
1121 for pattern in &config.patterns {
1122 if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
1123 if let Some(language) = language_registry
1124 .language_for_name_or_extension(language_name)
1125 .now_or_never()
1126 .and_then(|language| language.ok())
1127 {
1128 combined_injection_ranges.insert(language, Vec::new());
1129 }
1130 }
1131 }
1132
1133 for query_range in changed_ranges {
1134 query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
1135 for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
1136 let content_ranges = mat
1137 .nodes_for_capture_index(config.content_capture_ix)
1138 .map(|node| node.range())
1139 .collect::<Vec<_>>();
1140 if content_ranges.is_empty() {
1141 continue;
1142 }
1143
1144 // Avoid duplicate matches if two changed ranges intersect the same injection.
1145 let content_range =
1146 content_ranges.first().unwrap().start_byte..content_ranges.last().unwrap().end_byte;
1147 if let Some((last_pattern_ix, last_range)) = &prev_match {
1148 if mat.pattern_index == *last_pattern_ix && content_range == *last_range {
1149 continue;
1150 }
1151 }
1152 prev_match = Some((mat.pattern_index, content_range.clone()));
1153
1154 let combined = config.patterns[mat.pattern_index].combined;
1155
1156 let mut language_name = None;
1157 let mut step_range = content_range.clone();
1158 if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {
1159 language_name = Some(Cow::Borrowed(name.as_ref()))
1160 } else if let Some(language_node) = config
1161 .language_capture_ix
1162 .and_then(|ix| mat.nodes_for_capture_index(ix).next())
1163 {
1164 step_range.start = cmp::min(content_range.start, language_node.start_byte());
1165 step_range.end = cmp::max(content_range.end, language_node.end_byte());
1166 language_name = Some(Cow::Owned(
1167 text.text_for_range(language_node.byte_range()).collect(),
1168 ))
1169 };
1170
1171 if let Some(language_name) = language_name {
1172 let language = language_registry
1173 .language_for_name_or_extension(&language_name)
1174 .now_or_never()
1175 .and_then(|language| language.ok());
1176 let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
1177 if let Some(language) = language {
1178 if combined {
1179 combined_injection_ranges
1180 .entry(language.clone())
1181 .or_default()
1182 .extend(content_ranges);
1183 } else {
1184 queue.push(ParseStep {
1185 depth,
1186 language: ParseStepLanguage::Loaded { language },
1187 included_ranges: content_ranges,
1188 range,
1189 mode: ParseMode::Single,
1190 });
1191 }
1192 } else {
1193 queue.push(ParseStep {
1194 depth,
1195 language: ParseStepLanguage::Pending {
1196 name: language_name.into(),
1197 },
1198 included_ranges: content_ranges,
1199 range,
1200 mode: ParseMode::Single,
1201 });
1202 }
1203 }
1204 }
1205 }
1206
1207 for (language, mut included_ranges) in combined_injection_ranges.drain() {
1208 included_ranges.sort_unstable();
1209 let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
1210 queue.push(ParseStep {
1211 depth,
1212 language: ParseStepLanguage::Loaded { language },
1213 range,
1214 included_ranges,
1215 mode: ParseMode::Combined {
1216 parent_layer_range: node.start_byte()..node.end_byte(),
1217 parent_layer_changed_ranges: changed_ranges.to_vec(),
1218 },
1219 })
1220 }
1221}
1222
1223fn splice_included_ranges(
1224 mut ranges: Vec<tree_sitter::Range>,
1225 changed_ranges: &[Range<usize>],
1226 new_ranges: &[tree_sitter::Range],
1227) -> Vec<tree_sitter::Range> {
1228 let mut changed_ranges = changed_ranges.into_iter().peekable();
1229 let mut new_ranges = new_ranges.into_iter().peekable();
1230 let mut ranges_ix = 0;
1231 loop {
1232 let new_range = new_ranges.peek();
1233 let mut changed_range = changed_ranges.peek();
1234
1235 // Remove ranges that have changed before inserting any new ranges
1236 // into those ranges.
1237 if let Some((changed, new)) = changed_range.zip(new_range) {
1238 if new.end_byte < changed.start {
1239 changed_range = None;
1240 }
1241 }
1242
1243 if let Some(changed) = changed_range {
1244 let mut start_ix = ranges_ix
1245 + match ranges[ranges_ix..].binary_search_by_key(&changed.start, |r| r.end_byte) {
1246 Ok(ix) | Err(ix) => ix,
1247 };
1248 let mut end_ix = ranges_ix
1249 + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
1250 Ok(ix) => ix + 1,
1251 Err(ix) => ix,
1252 };
1253
1254 // If there are empty ranges, then there may be multiple ranges with the same
1255 // start or end. Expand the splice to include any adjacent ranges that touch
1256 // the changed range.
1257 while start_ix > 0 {
1258 if ranges[start_ix - 1].end_byte == changed.start {
1259 start_ix -= 1;
1260 } else {
1261 break;
1262 }
1263 }
1264 while let Some(range) = ranges.get(end_ix) {
1265 if range.start_byte == changed.end {
1266 end_ix += 1;
1267 } else {
1268 break;
1269 }
1270 }
1271
1272 if end_ix > start_ix {
1273 ranges.splice(start_ix..end_ix, []);
1274 }
1275 changed_ranges.next();
1276 ranges_ix = start_ix;
1277 } else if let Some(new_range) = new_range {
1278 let ix = ranges_ix
1279 + match ranges[ranges_ix..]
1280 .binary_search_by_key(&new_range.start_byte, |r| r.start_byte)
1281 {
1282 Ok(ix) | Err(ix) => ix,
1283 };
1284 ranges.insert(ix, **new_range);
1285 new_ranges.next();
1286 ranges_ix = ix + 1;
1287 } else {
1288 break;
1289 }
1290 }
1291 ranges
1292}
1293
1294impl OwnedSyntaxLayerInfo {
1295 pub fn node(&self) -> Node {
1296 self.tree
1297 .root_node_with_offset(self.offset.0, self.offset.1)
1298 }
1299}
1300
1301impl<'a> SyntaxLayerInfo<'a> {
1302 pub fn to_owned(&self) -> OwnedSyntaxLayerInfo {
1303 OwnedSyntaxLayerInfo {
1304 tree: self.tree.clone(),
1305 offset: self.offset,
1306 depth: self.depth,
1307 language: self.language.clone(),
1308 }
1309 }
1310
1311 pub fn node(&self) -> Node<'a> {
1312 self.tree
1313 .root_node_with_offset(self.offset.0, self.offset.1)
1314 }
1315
1316 pub(crate) fn override_id(&self, offset: usize, text: &text::BufferSnapshot) -> Option<u32> {
1317 let text = TextProvider(text.as_rope());
1318 let config = self.language.grammar.as_ref()?.override_config.as_ref()?;
1319
1320 let mut query_cursor = QueryCursorHandle::new();
1321 query_cursor.set_byte_range(offset..offset);
1322
1323 let mut smallest_match: Option<(u32, Range<usize>)> = None;
1324 for mat in query_cursor.matches(&config.query, self.node(), text) {
1325 for capture in mat.captures {
1326 if !config.values.contains_key(&capture.index) {
1327 continue;
1328 }
1329
1330 let range = capture.node.byte_range();
1331 if offset <= range.start || offset >= range.end {
1332 continue;
1333 }
1334
1335 if let Some((_, smallest_range)) = &smallest_match {
1336 if range.len() < smallest_range.len() {
1337 smallest_match = Some((capture.index, range))
1338 }
1339 continue;
1340 }
1341
1342 smallest_match = Some((capture.index, range));
1343 }
1344 }
1345
1346 smallest_match.map(|(index, _)| index)
1347 }
1348}
1349
1350impl std::ops::Deref for SyntaxMap {
1351 type Target = SyntaxSnapshot;
1352
1353 fn deref(&self) -> &Self::Target {
1354 &self.snapshot
1355 }
1356}
1357
1358impl PartialEq for ParseStep {
1359 fn eq(&self, _: &Self) -> bool {
1360 false
1361 }
1362}
1363
1364impl Eq for ParseStep {}
1365
1366impl PartialOrd for ParseStep {
1367 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1368 Some(self.cmp(&other))
1369 }
1370}
1371
1372impl Ord for ParseStep {
1373 fn cmp(&self, other: &Self) -> Ordering {
1374 let range_a = self.range();
1375 let range_b = other.range();
1376 Ord::cmp(&other.depth, &self.depth)
1377 .then_with(|| Ord::cmp(&range_b.start, &range_a.start))
1378 .then_with(|| Ord::cmp(&range_a.end, &range_b.end))
1379 .then_with(|| self.language.id().cmp(&other.language.id()))
1380 }
1381}
1382
1383impl ParseStep {
1384 fn range(&self) -> Range<usize> {
1385 if let ParseMode::Combined {
1386 parent_layer_range, ..
1387 } = &self.mode
1388 {
1389 parent_layer_range.clone()
1390 } else {
1391 let start = self.included_ranges.first().map_or(0, |r| r.start_byte);
1392 let end = self.included_ranges.last().map_or(0, |r| r.end_byte);
1393 start..end
1394 }
1395 }
1396}
1397
1398impl ChangedRegion {
1399 fn cmp(&self, other: &Self, buffer: &BufferSnapshot) -> Ordering {
1400 let range_a = &self.range;
1401 let range_b = &other.range;
1402 Ord::cmp(&self.depth, &other.depth)
1403 .then_with(|| range_a.start.cmp(&range_b.start, buffer))
1404 .then_with(|| range_b.end.cmp(&range_a.end, buffer))
1405 }
1406}
1407
1408impl ChangeRegionSet {
1409 fn start_position(&self) -> ChangeStartPosition {
1410 self.0.first().map_or(
1411 ChangeStartPosition {
1412 depth: usize::MAX,
1413 position: Anchor::MAX,
1414 },
1415 |region| ChangeStartPosition {
1416 depth: region.depth,
1417 position: region.range.start,
1418 },
1419 )
1420 }
1421
1422 fn intersects(&self, layer: &SyntaxLayer, text: &BufferSnapshot) -> bool {
1423 for region in &self.0 {
1424 if region.depth < layer.depth {
1425 continue;
1426 }
1427 if region.depth > layer.depth {
1428 break;
1429 }
1430 if region.range.end.cmp(&layer.range.start, text).is_le() {
1431 continue;
1432 }
1433 if region.range.start.cmp(&layer.range.end, text).is_ge() {
1434 break;
1435 }
1436 return true;
1437 }
1438 false
1439 }
1440
1441 fn insert(&mut self, region: ChangedRegion, text: &BufferSnapshot) {
1442 if let Err(ix) = self.0.binary_search_by(|probe| probe.cmp(®ion, text)) {
1443 self.0.insert(ix, region);
1444 }
1445 }
1446
1447 fn prune(&mut self, summary: SyntaxLayerSummary, text: &BufferSnapshot) -> bool {
1448 let prev_len = self.0.len();
1449 self.0.retain(|region| {
1450 region.depth > summary.max_depth
1451 || (region.depth == summary.max_depth
1452 && region
1453 .range
1454 .end
1455 .cmp(&summary.last_layer_range.start, text)
1456 .is_gt())
1457 });
1458 self.0.len() < prev_len
1459 }
1460}
1461
1462impl Default for SyntaxLayerSummary {
1463 fn default() -> Self {
1464 Self {
1465 max_depth: 0,
1466 min_depth: 0,
1467 range: Anchor::MAX..Anchor::MIN,
1468 last_layer_range: Anchor::MIN..Anchor::MAX,
1469 last_layer_language: None,
1470 contains_unknown_injections: false,
1471 }
1472 }
1473}
1474
1475impl sum_tree::Summary for SyntaxLayerSummary {
1476 type Context = BufferSnapshot;
1477
1478 fn add_summary(&mut self, other: &Self, buffer: &Self::Context) {
1479 if other.max_depth > self.max_depth {
1480 self.max_depth = other.max_depth;
1481 self.range = other.range.clone();
1482 } else {
1483 if self.range == (Anchor::MAX..Anchor::MAX) {
1484 self.range.start = other.range.start;
1485 }
1486 if other.range.end.cmp(&self.range.end, buffer).is_gt() {
1487 self.range.end = other.range.end;
1488 }
1489 }
1490 self.last_layer_range = other.last_layer_range.clone();
1491 self.last_layer_language = other.last_layer_language;
1492 self.contains_unknown_injections |= other.contains_unknown_injections;
1493 }
1494}
1495
1496impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for SyntaxLayerPosition {
1497 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1498 Ord::cmp(&self.depth, &cursor_location.max_depth)
1499 .then_with(|| {
1500 self.range
1501 .start
1502 .cmp(&cursor_location.last_layer_range.start, buffer)
1503 })
1504 .then_with(|| {
1505 cursor_location
1506 .last_layer_range
1507 .end
1508 .cmp(&self.range.end, buffer)
1509 })
1510 .then_with(|| self.language.cmp(&cursor_location.last_layer_language))
1511 }
1512}
1513
1514impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary> for ChangeStartPosition {
1515 fn cmp(&self, cursor_location: &SyntaxLayerSummary, text: &BufferSnapshot) -> Ordering {
1516 Ord::cmp(&self.depth, &cursor_location.max_depth)
1517 .then_with(|| self.position.cmp(&cursor_location.range.end, text))
1518 }
1519}
1520
1521impl<'a> SeekTarget<'a, SyntaxLayerSummary, SyntaxLayerSummary>
1522 for SyntaxLayerPositionBeforeChange
1523{
1524 fn cmp(&self, cursor_location: &SyntaxLayerSummary, buffer: &BufferSnapshot) -> Ordering {
1525 if self.change.cmp(cursor_location, buffer).is_le() {
1526 return Ordering::Less;
1527 } else {
1528 self.position.cmp(cursor_location, buffer)
1529 }
1530 }
1531}
1532
1533impl sum_tree::Item for SyntaxLayer {
1534 type Summary = SyntaxLayerSummary;
1535
1536 fn summary(&self) -> Self::Summary {
1537 SyntaxLayerSummary {
1538 min_depth: self.depth,
1539 max_depth: self.depth,
1540 range: self.range.clone(),
1541 last_layer_range: self.range.clone(),
1542 last_layer_language: self.content.language_id(),
1543 contains_unknown_injections: matches!(self.content, SyntaxLayerContent::Pending { .. }),
1544 }
1545 }
1546}
1547
1548impl std::fmt::Debug for SyntaxLayer {
1549 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1550 f.debug_struct("SyntaxLayer")
1551 .field("depth", &self.depth)
1552 .field("range", &self.range)
1553 .field("tree", &self.content.tree())
1554 .finish()
1555 }
1556}
1557
1558impl<'a> tree_sitter::TextProvider<'a> for TextProvider<'a> {
1559 type I = ByteChunks<'a>;
1560
1561 fn text(&mut self, node: tree_sitter::Node) -> Self::I {
1562 ByteChunks(self.0.chunks_in_range(node.byte_range()))
1563 }
1564}
1565
1566impl<'a> Iterator for ByteChunks<'a> {
1567 type Item = &'a [u8];
1568
1569 fn next(&mut self) -> Option<Self::Item> {
1570 self.0.next().map(str::as_bytes)
1571 }
1572}
1573
1574impl QueryCursorHandle {
1575 pub(crate) fn new() -> Self {
1576 let mut cursor = QUERY_CURSORS.lock().pop().unwrap_or_else(QueryCursor::new);
1577 cursor.set_match_limit(64);
1578 QueryCursorHandle(Some(cursor))
1579 }
1580}
1581
1582impl Deref for QueryCursorHandle {
1583 type Target = QueryCursor;
1584
1585 fn deref(&self) -> &Self::Target {
1586 self.0.as_ref().unwrap()
1587 }
1588}
1589
1590impl DerefMut for QueryCursorHandle {
1591 fn deref_mut(&mut self) -> &mut Self::Target {
1592 self.0.as_mut().unwrap()
1593 }
1594}
1595
1596impl Drop for QueryCursorHandle {
1597 fn drop(&mut self) {
1598 let mut cursor = self.0.take().unwrap();
1599 cursor.set_byte_range(0..usize::MAX);
1600 cursor.set_point_range(Point::zero().to_ts_point()..Point::MAX.to_ts_point());
1601 QUERY_CURSORS.lock().push(cursor)
1602 }
1603}
1604
1605pub(crate) trait ToTreeSitterPoint {
1606 fn to_ts_point(self) -> tree_sitter::Point;
1607 fn from_ts_point(point: tree_sitter::Point) -> Self;
1608}
1609
1610impl ToTreeSitterPoint for Point {
1611 fn to_ts_point(self) -> tree_sitter::Point {
1612 tree_sitter::Point::new(self.row as usize, self.column as usize)
1613 }
1614
1615 fn from_ts_point(point: tree_sitter::Point) -> Self {
1616 Point::new(point.row as u32, point.column as u32)
1617 }
1618}
1619
1620#[cfg(test)]
1621mod tests {
1622 use super::*;
1623 use crate::LanguageConfig;
1624 use rand::rngs::StdRng;
1625 use std::env;
1626 use text::Buffer;
1627 use unindent::Unindent as _;
1628 use util::test::marked_text_ranges;
1629
1630 #[test]
1631 fn test_splice_included_ranges() {
1632 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
1633
1634 let new_ranges = splice_included_ranges(
1635 ranges.clone(),
1636 &[54..56, 58..68],
1637 &[ts_range(50..54), ts_range(59..67)],
1638 );
1639 assert_eq!(
1640 new_ranges,
1641 &[
1642 ts_range(20..30),
1643 ts_range(50..54),
1644 ts_range(59..67),
1645 ts_range(80..90),
1646 ]
1647 );
1648
1649 let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
1650 assert_eq!(
1651 new_ranges,
1652 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
1653 );
1654
1655 let new_ranges =
1656 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
1657 assert_eq!(
1658 new_ranges,
1659 &[
1660 ts_range(0..2),
1661 ts_range(20..30),
1662 ts_range(50..60),
1663 ts_range(70..75),
1664 ts_range(80..90)
1665 ]
1666 );
1667
1668 let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
1669 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
1670
1671 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
1672 tree_sitter::Range {
1673 start_byte: range.start,
1674 start_point: tree_sitter::Point {
1675 row: 0,
1676 column: range.start,
1677 },
1678 end_byte: range.end,
1679 end_point: tree_sitter::Point {
1680 row: 0,
1681 column: range.end,
1682 },
1683 }
1684 }
1685 }
1686
1687 #[gpui::test]
1688 fn test_syntax_map_layers_for_range() {
1689 let registry = Arc::new(LanguageRegistry::test());
1690 let language = Arc::new(rust_lang());
1691 registry.add(language.clone());
1692
1693 let mut buffer = Buffer::new(
1694 0,
1695 0,
1696 r#"
1697 fn a() {
1698 assert_eq!(
1699 b(vec![C {}]),
1700 vec![d.e],
1701 );
1702 println!("{}", f(|_| true));
1703 }
1704 "#
1705 .unindent(),
1706 );
1707
1708 let mut syntax_map = SyntaxMap::new();
1709 syntax_map.set_language_registry(registry.clone());
1710 syntax_map.reparse(language.clone(), &buffer);
1711
1712 assert_layers_for_range(
1713 &syntax_map,
1714 &buffer,
1715 Point::new(2, 0)..Point::new(2, 0),
1716 &[
1717 "...(function_item ... (block (expression_statement (macro_invocation...",
1718 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1719 ],
1720 );
1721 assert_layers_for_range(
1722 &syntax_map,
1723 &buffer,
1724 Point::new(2, 14)..Point::new(2, 16),
1725 &[
1726 "...(function_item ...",
1727 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1728 "...(array_expression (struct_expression ...",
1729 ],
1730 );
1731 assert_layers_for_range(
1732 &syntax_map,
1733 &buffer,
1734 Point::new(3, 14)..Point::new(3, 16),
1735 &[
1736 "...(function_item ...",
1737 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1738 "...(array_expression (field_expression ...",
1739 ],
1740 );
1741 assert_layers_for_range(
1742 &syntax_map,
1743 &buffer,
1744 Point::new(5, 12)..Point::new(5, 16),
1745 &[
1746 "...(function_item ...",
1747 "...(call_expression ... (arguments (closure_expression ...",
1748 ],
1749 );
1750
1751 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
1752 let macro_name_range = range_for_text(&buffer, "vec!");
1753 buffer.edit([(macro_name_range, "&")]);
1754 syntax_map.interpolate(&buffer);
1755 syntax_map.reparse(language.clone(), &buffer);
1756
1757 assert_layers_for_range(
1758 &syntax_map,
1759 &buffer,
1760 Point::new(2, 14)..Point::new(2, 16),
1761 &[
1762 "...(function_item ...",
1763 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
1764 ],
1765 );
1766
1767 // Put the vec! macro back, adding back the syntactic layer.
1768 buffer.undo();
1769 syntax_map.interpolate(&buffer);
1770 syntax_map.reparse(language.clone(), &buffer);
1771
1772 assert_layers_for_range(
1773 &syntax_map,
1774 &buffer,
1775 Point::new(2, 14)..Point::new(2, 16),
1776 &[
1777 "...(function_item ...",
1778 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
1779 "...(array_expression (struct_expression ...",
1780 ],
1781 );
1782 }
1783
1784 #[gpui::test]
1785 fn test_dynamic_language_injection() {
1786 let registry = Arc::new(LanguageRegistry::test());
1787 let markdown = Arc::new(markdown_lang());
1788 registry.add(markdown.clone());
1789 registry.add(Arc::new(rust_lang()));
1790 registry.add(Arc::new(ruby_lang()));
1791
1792 let mut buffer = Buffer::new(
1793 0,
1794 0,
1795 r#"
1796 This is a code block:
1797
1798 ```rs
1799 fn foo() {}
1800 ```
1801 "#
1802 .unindent(),
1803 );
1804
1805 let mut syntax_map = SyntaxMap::new();
1806 syntax_map.set_language_registry(registry.clone());
1807 syntax_map.reparse(markdown.clone(), &buffer);
1808 assert_layers_for_range(
1809 &syntax_map,
1810 &buffer,
1811 Point::new(3, 0)..Point::new(3, 0),
1812 &[
1813 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1814 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
1815 ],
1816 );
1817
1818 // Replace Rust with Ruby in code block.
1819 let macro_name_range = range_for_text(&buffer, "rs");
1820 buffer.edit([(macro_name_range, "ruby")]);
1821 syntax_map.interpolate(&buffer);
1822 syntax_map.reparse(markdown.clone(), &buffer);
1823 assert_layers_for_range(
1824 &syntax_map,
1825 &buffer,
1826 Point::new(3, 0)..Point::new(3, 0),
1827 &[
1828 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1829 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
1830 ],
1831 );
1832
1833 // Replace Ruby with a language that hasn't been loaded yet.
1834 let macro_name_range = range_for_text(&buffer, "ruby");
1835 buffer.edit([(macro_name_range, "html")]);
1836 syntax_map.interpolate(&buffer);
1837 syntax_map.reparse(markdown.clone(), &buffer);
1838 assert_layers_for_range(
1839 &syntax_map,
1840 &buffer,
1841 Point::new(3, 0)..Point::new(3, 0),
1842 &[
1843 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
1844 ],
1845 );
1846 assert!(syntax_map.contains_unknown_injections());
1847
1848 registry.add(Arc::new(html_lang()));
1849 syntax_map.reparse(markdown.clone(), &buffer);
1850 assert_layers_for_range(
1851 &syntax_map,
1852 &buffer,
1853 Point::new(3, 0)..Point::new(3, 0),
1854 &[
1855 "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
1856 "(fragment (text))",
1857 ],
1858 );
1859 assert!(!syntax_map.contains_unknown_injections());
1860 }
1861
1862 #[gpui::test]
1863 fn test_typing_multiple_new_injections() {
1864 let (buffer, syntax_map) = test_edit_sequence(
1865 "Rust",
1866 &[
1867 "fn a() { dbg }",
1868 "fn a() { dbg«!» }",
1869 "fn a() { dbg!«()» }",
1870 "fn a() { dbg!(«b») }",
1871 "fn a() { dbg!(b«.») }",
1872 "fn a() { dbg!(b.«c») }",
1873 "fn a() { dbg!(b.c«()») }",
1874 "fn a() { dbg!(b.c(«vec»)) }",
1875 "fn a() { dbg!(b.c(vec«!»)) }",
1876 "fn a() { dbg!(b.c(vec!«[]»)) }",
1877 "fn a() { dbg!(b.c(vec![«d»])) }",
1878 "fn a() { dbg!(b.c(vec![d«.»])) }",
1879 "fn a() { dbg!(b.c(vec![d.«e»])) }",
1880 ],
1881 );
1882
1883 assert_capture_ranges(
1884 &syntax_map,
1885 &buffer,
1886 &["field"],
1887 "fn a() { dbg!(b.«c»(vec![d.«e»])) }",
1888 );
1889 }
1890
1891 #[gpui::test]
1892 fn test_pasting_new_injection_line_between_others() {
1893 let (buffer, syntax_map) = test_edit_sequence(
1894 "Rust",
1895 &[
1896 "
1897 fn a() {
1898 b!(B {});
1899 c!(C {});
1900 d!(D {});
1901 e!(E {});
1902 f!(F {});
1903 g!(G {});
1904 }
1905 ",
1906 "
1907 fn a() {
1908 b!(B {});
1909 c!(C {});
1910 d!(D {});
1911 « h!(H {});
1912 » e!(E {});
1913 f!(F {});
1914 g!(G {});
1915 }
1916 ",
1917 ],
1918 );
1919
1920 assert_capture_ranges(
1921 &syntax_map,
1922 &buffer,
1923 &["struct"],
1924 "
1925 fn a() {
1926 b!(«B {}»);
1927 c!(«C {}»);
1928 d!(«D {}»);
1929 h!(«H {}»);
1930 e!(«E {}»);
1931 f!(«F {}»);
1932 g!(«G {}»);
1933 }
1934 ",
1935 );
1936 }
1937
1938 #[gpui::test]
1939 fn test_joining_injections_with_child_injections() {
1940 let (buffer, syntax_map) = test_edit_sequence(
1941 "Rust",
1942 &[
1943 "
1944 fn a() {
1945 b!(
1946 c![one.two.three],
1947 d![four.five.six],
1948 );
1949 e!(
1950 f![seven.eight],
1951 );
1952 }
1953 ",
1954 "
1955 fn a() {
1956 b!(
1957 c![one.two.three],
1958 d![four.five.six],
1959 ˇ f![seven.eight],
1960 );
1961 }
1962 ",
1963 ],
1964 );
1965
1966 assert_capture_ranges(
1967 &syntax_map,
1968 &buffer,
1969 &["field"],
1970 "
1971 fn a() {
1972 b!(
1973 c![one.«two».«three»],
1974 d![four.«five».«six»],
1975 f![seven.«eight»],
1976 );
1977 }
1978 ",
1979 );
1980 }
1981
1982 #[gpui::test]
1983 fn test_editing_edges_of_injection() {
1984 test_edit_sequence(
1985 "Rust",
1986 &[
1987 "
1988 fn a() {
1989 b!(c!())
1990 }
1991 ",
1992 "
1993 fn a() {
1994 «d»!(c!())
1995 }
1996 ",
1997 "
1998 fn a() {
1999 «e»d!(c!())
2000 }
2001 ",
2002 "
2003 fn a() {
2004 ed!«[»c!()«]»
2005 }
2006 ",
2007 ],
2008 );
2009 }
2010
2011 #[gpui::test]
2012 fn test_edits_preceding_and_intersecting_injection() {
2013 test_edit_sequence(
2014 "Rust",
2015 &[
2016 //
2017 "const aaaaaaaaaaaa: B = c!(d(e.f));",
2018 "const aˇa: B = c!(d(eˇ));",
2019 ],
2020 );
2021 }
2022
2023 #[gpui::test]
2024 fn test_non_local_changes_create_injections() {
2025 test_edit_sequence(
2026 "Rust",
2027 &[
2028 "
2029 // a! {
2030 static B: C = d;
2031 // }
2032 ",
2033 "
2034 ˇa! {
2035 static B: C = d;
2036 ˇ}
2037 ",
2038 ],
2039 );
2040 }
2041
2042 #[gpui::test]
2043 fn test_creating_many_injections_in_one_edit() {
2044 test_edit_sequence(
2045 "Rust",
2046 &[
2047 "
2048 fn a() {
2049 one(Two::three(3));
2050 four(Five::six(6));
2051 seven(Eight::nine(9));
2052 }
2053 ",
2054 "
2055 fn a() {
2056 one«!»(Two::three(3));
2057 four«!»(Five::six(6));
2058 seven«!»(Eight::nine(9));
2059 }
2060 ",
2061 "
2062 fn a() {
2063 one!(Two::three«!»(3));
2064 four!(Five::six«!»(6));
2065 seven!(Eight::nine«!»(9));
2066 }
2067 ",
2068 ],
2069 );
2070 }
2071
2072 #[gpui::test]
2073 fn test_editing_across_injection_boundary() {
2074 test_edit_sequence(
2075 "Rust",
2076 &[
2077 "
2078 fn one() {
2079 two();
2080 three!(
2081 three.four,
2082 five.six,
2083 );
2084 }
2085 ",
2086 "
2087 fn one() {
2088 two();
2089 th«irty_five![»
2090 three.four,
2091 five.six,
2092 « seven.eight,
2093 ];»
2094 }
2095 ",
2096 ],
2097 );
2098 }
2099
2100 #[gpui::test]
2101 fn test_removing_injection_by_replacing_across_boundary() {
2102 test_edit_sequence(
2103 "Rust",
2104 &[
2105 "
2106 fn one() {
2107 two!(
2108 three.four,
2109 );
2110 }
2111 ",
2112 "
2113 fn one() {
2114 t«en
2115 .eleven(
2116 twelve,
2117 »
2118 three.four,
2119 );
2120 }
2121 ",
2122 ],
2123 );
2124 }
2125
2126 #[gpui::test]
2127 fn test_combined_injections() {
2128 let (buffer, syntax_map) = test_edit_sequence(
2129 "ERB",
2130 &[
2131 "
2132 <body>
2133 <% if @one %>
2134 <div class=one>
2135 <% else %>
2136 <div class=two>
2137 <% end %>
2138 </div>
2139 </body>
2140 ",
2141 "
2142 <body>
2143 <% if @one %>
2144 <div class=one>
2145 ˇ else ˇ
2146 <div class=two>
2147 <% end %>
2148 </div>
2149 </body>
2150 ",
2151 "
2152 <body>
2153 <% if @one «;» end %>
2154 </div>
2155 </body>
2156 ",
2157 ],
2158 );
2159
2160 assert_capture_ranges(
2161 &syntax_map,
2162 &buffer,
2163 &["tag", "ivar"],
2164 "
2165 <«body»>
2166 <% if «@one» ; end %>
2167 </«div»>
2168 </«body»>
2169 ",
2170 );
2171 }
2172
2173 #[gpui::test]
2174 fn test_combined_injections_empty_ranges() {
2175 test_edit_sequence(
2176 "ERB",
2177 &[
2178 "
2179 <% if @one %>
2180 <% else %>
2181 <% end %>
2182 ",
2183 "
2184 <% if @one %>
2185 ˇ<% end %>
2186 ",
2187 ],
2188 );
2189 }
2190
2191 #[gpui::test]
2192 fn test_combined_injections_edit_edges_of_ranges() {
2193 let (buffer, syntax_map) = test_edit_sequence(
2194 "ERB",
2195 &[
2196 "
2197 <%= one @two %>
2198 <%= three @four %>
2199 ",
2200 "
2201 <%= one @two %ˇ
2202 <%= three @four %>
2203 ",
2204 "
2205 <%= one @two %«>»
2206 <%= three @four %>
2207 ",
2208 ],
2209 );
2210
2211 assert_capture_ranges(
2212 &syntax_map,
2213 &buffer,
2214 &["tag", "ivar"],
2215 "
2216 <%= one «@two» %>
2217 <%= three «@four» %>
2218 ",
2219 );
2220 }
2221
2222 #[gpui::test]
2223 fn test_combined_injections_splitting_some_injections() {
2224 let (_buffer, _syntax_map) = test_edit_sequence(
2225 "ERB",
2226 &[
2227 r#"
2228 <%A if b(:c) %>
2229 d
2230 <% end %>
2231 eee
2232 <% f %>
2233 "#,
2234 r#"
2235 <%« AAAAAAA %>
2236 hhhhhhh
2237 <%=» if b(:c) %>
2238 d
2239 <% end %>
2240 eee
2241 <% f %>
2242 "#,
2243 ],
2244 );
2245 }
2246
2247 #[gpui::test]
2248 fn test_combined_injections_inside_injections() {
2249 let (_buffer, _syntax_map) = test_edit_sequence(
2250 "Markdown",
2251 &[
2252 r#"
2253 here is some ERB code:
2254
2255 ```erb
2256 <ul>
2257 <% people.each do |person| %>
2258 <li><%= person.name %></li>
2259 <% end %>
2260 </ul>
2261 ```
2262 "#,
2263 r#"
2264 here is some ERB code:
2265
2266 ```erb
2267 <ul>
2268 <% people«2».each do |person| %>
2269 <li><%= person.name %></li>
2270 <% end %>
2271 </ul>
2272 ```
2273 "#,
2274 ],
2275 );
2276 }
2277
2278 #[gpui::test(iterations = 50)]
2279 fn test_random_syntax_map_edits(mut rng: StdRng) {
2280 let operations = env::var("OPERATIONS")
2281 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2282 .unwrap_or(10);
2283
2284 let text = r#"
2285 fn test_something() {
2286 let vec = vec![5, 1, 3, 8];
2287 assert_eq!(
2288 vec
2289 .into_iter()
2290 .map(|i| i * 2)
2291 .collect::<Vec<usize>>(),
2292 vec![
2293 5 * 2, 1 * 2, 3 * 2, 8 * 2
2294 ],
2295 );
2296 }
2297 "#
2298 .unindent()
2299 .repeat(2);
2300
2301 let registry = Arc::new(LanguageRegistry::test());
2302 let language = Arc::new(rust_lang());
2303 registry.add(language.clone());
2304 let mut buffer = Buffer::new(0, 0, text);
2305
2306 let mut syntax_map = SyntaxMap::new();
2307 syntax_map.set_language_registry(registry.clone());
2308 syntax_map.reparse(language.clone(), &buffer);
2309
2310 let mut reference_syntax_map = SyntaxMap::new();
2311 reference_syntax_map.set_language_registry(registry.clone());
2312
2313 log::info!("initial text:\n{}", buffer.text());
2314
2315 for _ in 0..operations {
2316 let prev_buffer = buffer.snapshot();
2317 let prev_syntax_map = syntax_map.snapshot();
2318
2319 buffer.randomly_edit(&mut rng, 3);
2320 log::info!("text:\n{}", buffer.text());
2321
2322 syntax_map.interpolate(&buffer);
2323 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2324
2325 syntax_map.reparse(language.clone(), &buffer);
2326
2327 reference_syntax_map.clear();
2328 reference_syntax_map.reparse(language.clone(), &buffer);
2329 }
2330
2331 for i in 0..operations {
2332 let i = operations - i - 1;
2333 buffer.undo();
2334 log::info!("undoing operation {}", i);
2335 log::info!("text:\n{}", buffer.text());
2336
2337 syntax_map.interpolate(&buffer);
2338 syntax_map.reparse(language.clone(), &buffer);
2339
2340 reference_syntax_map.clear();
2341 reference_syntax_map.reparse(language.clone(), &buffer);
2342 assert_eq!(
2343 syntax_map.layers(&buffer).len(),
2344 reference_syntax_map.layers(&buffer).len(),
2345 "wrong number of layers after undoing edit {i}"
2346 );
2347 }
2348
2349 let layers = syntax_map.layers(&buffer);
2350 let reference_layers = reference_syntax_map.layers(&buffer);
2351 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2352 {
2353 assert_eq!(
2354 edited_layer.node().to_sexp(),
2355 reference_layer.node().to_sexp()
2356 );
2357 assert_eq!(edited_layer.node().range(), reference_layer.node().range());
2358 }
2359 }
2360
2361 #[gpui::test(iterations = 50)]
2362 fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) {
2363 let operations = env::var("OPERATIONS")
2364 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
2365 .unwrap_or(10);
2366
2367 let text = r#"
2368 <div id="main">
2369 <% if one?(:two) %>
2370 <p class="three" four>
2371 <%= yield :five %>
2372 </p>
2373 <% elsif Six.seven(8) %>
2374 <p id="three" four>
2375 <%= yield :five %>
2376 </p>
2377 <% else %>
2378 <span>Ok</span>
2379 <% end %>
2380 </div>
2381 "#
2382 .unindent()
2383 .repeat(8);
2384
2385 let registry = Arc::new(LanguageRegistry::test());
2386 let language = Arc::new(erb_lang());
2387 registry.add(language.clone());
2388 registry.add(Arc::new(ruby_lang()));
2389 registry.add(Arc::new(html_lang()));
2390 let mut buffer = Buffer::new(0, 0, text);
2391
2392 let mut syntax_map = SyntaxMap::new();
2393 syntax_map.set_language_registry(registry.clone());
2394 syntax_map.reparse(language.clone(), &buffer);
2395
2396 let mut reference_syntax_map = SyntaxMap::new();
2397 reference_syntax_map.set_language_registry(registry.clone());
2398
2399 log::info!("initial text:\n{}", buffer.text());
2400
2401 for _ in 0..operations {
2402 let prev_buffer = buffer.snapshot();
2403 let prev_syntax_map = syntax_map.snapshot();
2404
2405 buffer.randomly_edit(&mut rng, 3);
2406 log::info!("text:\n{}", buffer.text());
2407
2408 syntax_map.interpolate(&buffer);
2409 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
2410
2411 syntax_map.reparse(language.clone(), &buffer);
2412
2413 reference_syntax_map.clear();
2414 reference_syntax_map.reparse(language.clone(), &buffer);
2415 }
2416
2417 for i in 0..operations {
2418 let i = operations - i - 1;
2419 buffer.undo();
2420 log::info!("undoing operation {}", i);
2421 log::info!("text:\n{}", buffer.text());
2422
2423 syntax_map.interpolate(&buffer);
2424 syntax_map.reparse(language.clone(), &buffer);
2425
2426 reference_syntax_map.clear();
2427 reference_syntax_map.reparse(language.clone(), &buffer);
2428 assert_eq!(
2429 syntax_map.layers(&buffer).len(),
2430 reference_syntax_map.layers(&buffer).len(),
2431 "wrong number of layers after undoing edit {i}"
2432 );
2433 }
2434
2435 let layers = syntax_map.layers(&buffer);
2436 let reference_layers = reference_syntax_map.layers(&buffer);
2437 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter())
2438 {
2439 assert_eq!(
2440 edited_layer.node().to_sexp(),
2441 reference_layer.node().to_sexp()
2442 );
2443 assert_eq!(edited_layer.node().range(), reference_layer.node().range());
2444 }
2445 }
2446
2447 fn check_interpolation(
2448 old_syntax_map: &SyntaxSnapshot,
2449 new_syntax_map: &SyntaxSnapshot,
2450 old_buffer: &BufferSnapshot,
2451 new_buffer: &BufferSnapshot,
2452 ) {
2453 let edits = new_buffer
2454 .edits_since::<usize>(&old_buffer.version())
2455 .collect::<Vec<_>>();
2456
2457 for (old_layer, new_layer) in old_syntax_map
2458 .layers
2459 .iter()
2460 .zip(new_syntax_map.layers.iter())
2461 {
2462 assert_eq!(old_layer.range, new_layer.range);
2463 let Some(old_tree) = old_layer.content.tree() else { continue };
2464 let Some(new_tree) = new_layer.content.tree() else { continue };
2465 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
2466 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
2467 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
2468 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
2469 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
2470 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
2471 check_node_edits(
2472 old_layer.depth,
2473 &old_layer.range,
2474 old_node,
2475 new_node,
2476 old_buffer,
2477 new_buffer,
2478 &edits,
2479 );
2480 }
2481
2482 fn check_node_edits(
2483 depth: usize,
2484 range: &Range<Anchor>,
2485 old_node: Node,
2486 new_node: Node,
2487 old_buffer: &BufferSnapshot,
2488 new_buffer: &BufferSnapshot,
2489 edits: &[text::Edit<usize>],
2490 ) {
2491 assert_eq!(old_node.kind(), new_node.kind());
2492
2493 let old_range = old_node.byte_range();
2494 let new_range = new_node.byte_range();
2495
2496 let is_edited = edits
2497 .iter()
2498 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
2499 if is_edited {
2500 assert!(
2501 new_node.has_changes(),
2502 concat!(
2503 "failed to mark node as edited.\n",
2504 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2505 "node kind: {}, old node range: {:?}, new node range: {:?}",
2506 ),
2507 depth,
2508 range.to_offset(old_buffer),
2509 range.to_offset(new_buffer),
2510 new_node.kind(),
2511 old_range,
2512 new_range,
2513 );
2514 }
2515
2516 if !new_node.has_changes() {
2517 assert_eq!(
2518 old_buffer
2519 .text_for_range(old_range.clone())
2520 .collect::<String>(),
2521 new_buffer
2522 .text_for_range(new_range.clone())
2523 .collect::<String>(),
2524 concat!(
2525 "mismatched text for node\n",
2526 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
2527 "node kind: {}, old node range:{:?}, new node range:{:?}",
2528 ),
2529 depth,
2530 range.to_offset(old_buffer),
2531 range.to_offset(new_buffer),
2532 new_node.kind(),
2533 old_range,
2534 new_range,
2535 );
2536 }
2537
2538 for i in 0..new_node.child_count() {
2539 check_node_edits(
2540 depth,
2541 range,
2542 old_node.child(i).unwrap(),
2543 new_node.child(i).unwrap(),
2544 old_buffer,
2545 new_buffer,
2546 edits,
2547 )
2548 }
2549 }
2550 }
2551
2552 fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
2553 let registry = Arc::new(LanguageRegistry::test());
2554 registry.add(Arc::new(rust_lang()));
2555 registry.add(Arc::new(ruby_lang()));
2556 registry.add(Arc::new(html_lang()));
2557 registry.add(Arc::new(erb_lang()));
2558 registry.add(Arc::new(markdown_lang()));
2559 let language = registry
2560 .language_for_name(language_name)
2561 .now_or_never()
2562 .unwrap()
2563 .unwrap();
2564 let mut buffer = Buffer::new(0, 0, Default::default());
2565
2566 let mut mutated_syntax_map = SyntaxMap::new();
2567 mutated_syntax_map.set_language_registry(registry.clone());
2568 mutated_syntax_map.reparse(language.clone(), &buffer);
2569
2570 for (i, marked_string) in steps.into_iter().enumerate() {
2571 buffer.edit_via_marked_text(&marked_string.unindent());
2572
2573 // Reparse the syntax map
2574 mutated_syntax_map.interpolate(&buffer);
2575 mutated_syntax_map.reparse(language.clone(), &buffer);
2576
2577 // Create a second syntax map from scratch
2578 let mut reference_syntax_map = SyntaxMap::new();
2579 reference_syntax_map.set_language_registry(registry.clone());
2580 reference_syntax_map.reparse(language.clone(), &buffer);
2581
2582 // Compare the mutated syntax map to the new syntax map
2583 let mutated_layers = mutated_syntax_map.layers(&buffer);
2584 let reference_layers = reference_syntax_map.layers(&buffer);
2585 assert_eq!(
2586 mutated_layers.len(),
2587 reference_layers.len(),
2588 "wrong number of layers at step {i}"
2589 );
2590 for (edited_layer, reference_layer) in
2591 mutated_layers.into_iter().zip(reference_layers.into_iter())
2592 {
2593 assert_eq!(
2594 edited_layer.node().to_sexp(),
2595 reference_layer.node().to_sexp(),
2596 "different layer at step {i}"
2597 );
2598 assert_eq!(
2599 edited_layer.node().range(),
2600 reference_layer.node().range(),
2601 "different layer at step {i}"
2602 );
2603 }
2604 }
2605
2606 (buffer, mutated_syntax_map)
2607 }
2608
2609 fn html_lang() -> Language {
2610 Language::new(
2611 LanguageConfig {
2612 name: "HTML".into(),
2613 path_suffixes: vec!["html".to_string()],
2614 ..Default::default()
2615 },
2616 Some(tree_sitter_html::language()),
2617 )
2618 .with_highlights_query(
2619 r#"
2620 (tag_name) @tag
2621 (erroneous_end_tag_name) @tag
2622 (attribute_name) @property
2623 "#,
2624 )
2625 .unwrap()
2626 }
2627
2628 fn ruby_lang() -> Language {
2629 Language::new(
2630 LanguageConfig {
2631 name: "Ruby".into(),
2632 path_suffixes: vec!["rb".to_string()],
2633 ..Default::default()
2634 },
2635 Some(tree_sitter_ruby::language()),
2636 )
2637 .with_highlights_query(
2638 r#"
2639 ["if" "do" "else" "end"] @keyword
2640 (instance_variable) @ivar
2641 "#,
2642 )
2643 .unwrap()
2644 }
2645
2646 fn erb_lang() -> Language {
2647 Language::new(
2648 LanguageConfig {
2649 name: "ERB".into(),
2650 path_suffixes: vec!["erb".to_string()],
2651 ..Default::default()
2652 },
2653 Some(tree_sitter_embedded_template::language()),
2654 )
2655 .with_highlights_query(
2656 r#"
2657 ["<%" "%>"] @keyword
2658 "#,
2659 )
2660 .unwrap()
2661 .with_injection_query(
2662 r#"
2663 ((code) @content
2664 (#set! "language" "ruby")
2665 (#set! "combined"))
2666
2667 ((content) @content
2668 (#set! "language" "html")
2669 (#set! "combined"))
2670 "#,
2671 )
2672 .unwrap()
2673 }
2674
2675 fn rust_lang() -> Language {
2676 Language::new(
2677 LanguageConfig {
2678 name: "Rust".into(),
2679 path_suffixes: vec!["rs".to_string()],
2680 ..Default::default()
2681 },
2682 Some(tree_sitter_rust::language()),
2683 )
2684 .with_highlights_query(
2685 r#"
2686 (field_identifier) @field
2687 (struct_expression) @struct
2688 "#,
2689 )
2690 .unwrap()
2691 .with_injection_query(
2692 r#"
2693 (macro_invocation
2694 (token_tree) @content
2695 (#set! "language" "rust"))
2696 "#,
2697 )
2698 .unwrap()
2699 }
2700
2701 fn markdown_lang() -> Language {
2702 Language::new(
2703 LanguageConfig {
2704 name: "Markdown".into(),
2705 path_suffixes: vec!["md".into()],
2706 ..Default::default()
2707 },
2708 Some(tree_sitter_markdown::language()),
2709 )
2710 .with_injection_query(
2711 r#"
2712 (fenced_code_block
2713 (info_string
2714 (language) @language)
2715 (code_fence_content) @content)
2716 "#,
2717 )
2718 .unwrap()
2719 }
2720
2721 fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
2722 let start = buffer.as_rope().to_string().find(text).unwrap();
2723 start..start + text.len()
2724 }
2725
2726 fn assert_layers_for_range(
2727 syntax_map: &SyntaxMap,
2728 buffer: &BufferSnapshot,
2729 range: Range<Point>,
2730 expected_layers: &[&str],
2731 ) {
2732 let layers = syntax_map
2733 .layers_for_range(range, &buffer)
2734 .collect::<Vec<_>>();
2735 assert_eq!(
2736 layers.len(),
2737 expected_layers.len(),
2738 "wrong number of layers"
2739 );
2740 for (i, (layer, expected_s_exp)) in layers.iter().zip(expected_layers.iter()).enumerate() {
2741 let actual_s_exp = layer.node().to_sexp();
2742 assert!(
2743 string_contains_sequence(
2744 &actual_s_exp,
2745 &expected_s_exp.split("...").collect::<Vec<_>>()
2746 ),
2747 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
2748 );
2749 }
2750 }
2751
2752 fn assert_capture_ranges(
2753 syntax_map: &SyntaxMap,
2754 buffer: &BufferSnapshot,
2755 highlight_query_capture_names: &[&str],
2756 marked_string: &str,
2757 ) {
2758 let mut actual_ranges = Vec::<Range<usize>>::new();
2759 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
2760 grammar.highlights_query.as_ref()
2761 });
2762 let queries = captures
2763 .grammars()
2764 .iter()
2765 .map(|grammar| grammar.highlights_query.as_ref().unwrap())
2766 .collect::<Vec<_>>();
2767 for capture in captures {
2768 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
2769 if highlight_query_capture_names.contains(&name.as_str()) {
2770 actual_ranges.push(capture.node.byte_range());
2771 }
2772 }
2773
2774 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
2775 assert_eq!(text, buffer.text());
2776 assert_eq!(actual_ranges, expected_ranges);
2777 }
2778
2779 pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
2780 let mut last_part_end = 0;
2781 for part in parts {
2782 if let Some(start_ix) = text[last_part_end..].find(part) {
2783 last_part_end = start_ix + part.len();
2784 } else {
2785 return false;
2786 }
2787 }
2788 true
2789 }
2790}