1use super::*;
2use crate::{
3 LanguageConfig, LanguageMatcher, LanguageQueries, buffer_tests::markdown_inline_lang,
4 markdown_lang, rust_lang,
5};
6use gpui::App;
7use pretty_assertions::assert_eq;
8use rand::rngs::StdRng;
9use std::borrow::Cow;
10use std::{env, ops::Range, sync::Arc};
11use text::{Buffer, BufferId, ReplicaId};
12use tree_sitter::Node;
13use unindent::Unindent as _;
14use util::test::marked_text_ranges;
15
16#[test]
17fn test_splice_included_ranges() {
18 let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
19
20 let (new_ranges, change) = splice_included_ranges(
21 ranges.clone(),
22 &[54..56, 58..68],
23 &[ts_range(50..54), ts_range(59..67)],
24 );
25 assert_eq!(
26 new_ranges,
27 &[
28 ts_range(20..30),
29 ts_range(50..54),
30 ts_range(59..67),
31 ts_range(80..90),
32 ]
33 );
34 assert_eq!(change, 1..3);
35
36 let (new_ranges, change) = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]);
37 assert_eq!(
38 new_ranges,
39 &[ts_range(20..30), ts_range(50..60), ts_range(80..90)]
40 );
41 assert_eq!(change, 2..3);
42
43 let (new_ranges, change) =
44 splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]);
45 assert_eq!(
46 new_ranges,
47 &[
48 ts_range(0..2),
49 ts_range(20..30),
50 ts_range(50..60),
51 ts_range(70..75),
52 ts_range(80..90)
53 ]
54 );
55 assert_eq!(change, 0..4);
56
57 let (new_ranges, change) =
58 splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
59 assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
60 assert_eq!(change, 0..1);
61
62 // does not create overlapping ranges
63 let (new_ranges, change) = splice_included_ranges(ranges, &[0..18], &[ts_range(20..32)]);
64 assert_eq!(
65 new_ranges,
66 &[ts_range(20..32), ts_range(50..60), ts_range(80..90)]
67 );
68 assert_eq!(change, 0..1);
69
70 fn ts_range(range: Range<usize>) -> tree_sitter::Range {
71 tree_sitter::Range {
72 start_byte: range.start,
73 start_point: tree_sitter::Point {
74 row: 0,
75 column: range.start,
76 },
77 end_byte: range.end,
78 end_point: tree_sitter::Point {
79 row: 0,
80 column: range.end,
81 },
82 }
83 }
84}
85
86#[gpui::test]
87fn test_syntax_map_layers_for_range(cx: &mut App) {
88 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
89 let language = rust_lang();
90 registry.add(language.clone());
91
92 let mut buffer = Buffer::new(
93 ReplicaId::LOCAL,
94 BufferId::new(1).unwrap(),
95 r#"
96 fn a() {
97 assert_eq!(
98 b(vec![C {}]),
99 vec![d.e],
100 );
101 println!("{}", f(|_| true));
102 }
103 "#
104 .unindent(),
105 );
106
107 let mut syntax_map = SyntaxMap::new(&buffer);
108 syntax_map.set_language_registry(registry);
109 syntax_map.reparse(language.clone(), &buffer);
110
111 assert_layers_for_range(
112 &syntax_map,
113 &buffer,
114 Point::new(2, 0)..Point::new(2, 0),
115 &[
116 "...(function_item ... (block (expression_statement (macro_invocation...",
117 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
118 ],
119 );
120 assert_layers_for_range(
121 &syntax_map,
122 &buffer,
123 Point::new(2, 14)..Point::new(2, 16),
124 &[
125 "...(function_item ...",
126 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
127 "...(array_expression (struct_expression ...",
128 ],
129 );
130 assert_layers_for_range(
131 &syntax_map,
132 &buffer,
133 Point::new(3, 14)..Point::new(3, 16),
134 &[
135 "...(function_item ...",
136 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
137 "...(array_expression (field_expression ...",
138 ],
139 );
140 assert_layers_for_range(
141 &syntax_map,
142 &buffer,
143 Point::new(5, 12)..Point::new(5, 16),
144 &[
145 "...(function_item ...",
146 "...(call_expression ... (arguments (closure_expression ...",
147 ],
148 );
149
150 // Replace a vec! macro invocation with a plain slice, removing a syntactic layer.
151 let macro_name_range = range_for_text(&buffer, "vec!");
152 buffer.edit([(macro_name_range, "&")]);
153 syntax_map.interpolate(&buffer);
154 syntax_map.reparse(language.clone(), &buffer);
155
156 assert_layers_for_range(
157 &syntax_map,
158 &buffer,
159 Point::new(2, 14)..Point::new(2, 16),
160 &[
161 "...(function_item ...",
162 "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...",
163 ],
164 );
165
166 // Put the vec! macro back, adding back the syntactic layer.
167 buffer.undo();
168 syntax_map.interpolate(&buffer);
169 syntax_map.reparse(language, &buffer);
170
171 assert_layers_for_range(
172 &syntax_map,
173 &buffer,
174 Point::new(2, 14)..Point::new(2, 16),
175 &[
176 "...(function_item ...",
177 "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...",
178 "...(array_expression (struct_expression ...",
179 ],
180 );
181}
182
183#[gpui::test]
184fn test_syntax_map_languages_match_layers_for_range(cx: &mut App) {
185 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
186 let markdown = markdown_lang();
187 let markdown_inline = Arc::new(markdown_inline_lang());
188 registry.add(markdown.clone());
189 registry.add(markdown_inline);
190 registry.add(rust_lang());
191
192 let buffer = Buffer::new(
193 ReplicaId::LOCAL,
194 BufferId::new(1).unwrap(),
195 r#"
196 This is `inline`.
197
198 ```rs
199 fn a() {}
200 ```
201 "#
202 .unindent(),
203 );
204
205 let mut syntax_map = SyntaxMap::new(&buffer);
206 syntax_map.set_language_registry(registry);
207 syntax_map.reparse(markdown, &buffer);
208
209 let all_language_names = syntax_map
210 .languages(&buffer, true)
211 .map(|language| language.name().to_string())
212 .collect::<Vec<_>>();
213 let all_layer_language_names = syntax_map
214 .layers_for_range(0..buffer.len(), &buffer, true)
215 .map(|layer| layer.language.name().to_string())
216 .collect::<Vec<_>>();
217
218 assert_eq!(all_language_names, all_layer_language_names);
219 assert!(
220 all_language_names
221 .iter()
222 .any(|language_name| language_name == "Markdown-Inline"),
223 "expected hidden languages to be included when include_hidden is true"
224 );
225 assert!(
226 all_language_names
227 .iter()
228 .any(|language_name| language_name == "Markdown")
229 );
230 assert!(
231 all_language_names
232 .iter()
233 .any(|language_name| language_name == "Rust")
234 );
235
236 let visible_language_names = syntax_map
237 .languages(&buffer, false)
238 .map(|language| language.name().to_string())
239 .collect::<Vec<_>>();
240 let visible_layer_language_names = syntax_map
241 .layers_for_range(0..buffer.len(), &buffer, false)
242 .map(|layer| layer.language.name().to_string())
243 .collect::<Vec<_>>();
244
245 assert_eq!(visible_language_names, visible_layer_language_names);
246 assert!(
247 !visible_language_names
248 .iter()
249 .any(|language_name| language_name == "Markdown-Inline"),
250 "expected hidden languages to be excluded when include_hidden is false"
251 );
252 assert!(
253 visible_language_names
254 .iter()
255 .any(|language_name| language_name == "Markdown")
256 );
257 assert!(
258 visible_language_names
259 .iter()
260 .any(|language_name| language_name == "Rust")
261 );
262}
263
264#[gpui::test]
265fn test_dynamic_language_injection(cx: &mut App) {
266 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
267 let markdown = markdown_lang();
268 let markdown_inline = Arc::new(markdown_inline_lang());
269 registry.add(markdown.clone());
270 registry.add(markdown_inline.clone());
271 registry.add(rust_lang());
272 registry.add(Arc::new(ruby_lang()));
273
274 let mut buffer = Buffer::new(
275 ReplicaId::LOCAL,
276 BufferId::new(1).unwrap(),
277 r#"
278 This is a code block:
279
280 ```rs
281 fn foo() {}
282 ```
283 "#
284 .unindent(),
285 );
286
287 let mut syntax_map = SyntaxMap::new(&buffer);
288 syntax_map.set_language_registry(registry.clone());
289 syntax_map.reparse(markdown.clone(), &buffer);
290 syntax_map.reparse(markdown_inline.clone(), &buffer);
291 assert_layers_for_range(
292 &syntax_map,
293 &buffer,
294 Point::new(3, 0)..Point::new(3, 0),
295 &[
296 "(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
297 "(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
298 "...(function_item name: (identifier) parameters: (parameters) body: (block)...",
299 ],
300 );
301
302 // Replace `rs` with a path to ending in `.rb` in code block.
303 let macro_name_range = range_for_text(&buffer, "rs");
304 buffer.edit([(macro_name_range, "foo/bar/baz.rb")]);
305 syntax_map.interpolate(&buffer);
306 syntax_map.reparse(markdown.clone(), &buffer);
307 syntax_map.reparse(markdown_inline.clone(), &buffer);
308 assert_layers_for_range(
309 &syntax_map,
310 &buffer,
311 Point::new(3, 0)..Point::new(3, 0),
312 &[
313 "(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
314 "(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
315 "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
316 ],
317 );
318
319 // Replace Ruby with a language that hasn't been loaded yet.
320 let macro_name_range = range_for_text(&buffer, "foo/bar/baz.rb");
321 buffer.edit([(macro_name_range, "html")]);
322 syntax_map.interpolate(&buffer);
323 syntax_map.reparse(markdown.clone(), &buffer);
324 syntax_map.reparse(markdown_inline.clone(), &buffer);
325 assert_layers_for_range(
326 &syntax_map,
327 &buffer,
328 Point::new(3, 0)..Point::new(3, 0),
329 &[
330 "(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
331 "(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
332 ],
333 );
334 assert!(syntax_map.contains_unknown_injections());
335
336 registry.add(Arc::new(html_lang()));
337 syntax_map.reparse(markdown, &buffer);
338 syntax_map.reparse(markdown_inline, &buffer);
339 assert_layers_for_range(
340 &syntax_map,
341 &buffer,
342 Point::new(3, 0)..Point::new(3, 0),
343 &[
344 "(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
345 "(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
346 "(document (text))",
347 ],
348 );
349 assert!(!syntax_map.contains_unknown_injections());
350}
351
352#[gpui::test]
353fn test_typing_multiple_new_injections(cx: &mut App) {
354 let (buffer, syntax_map) = test_edit_sequence(
355 "Rust",
356 &[
357 "fn a() { test_macro }",
358 "fn a() { test_macro«!» }",
359 "fn a() { test_macro!«()» }",
360 "fn a() { test_macro!(«b») }",
361 "fn a() { test_macro!(b«.») }",
362 "fn a() { test_macro!(b.«c») }",
363 "fn a() { test_macro!(b.c«()») }",
364 "fn a() { test_macro!(b.c(«vec»)) }",
365 "fn a() { test_macro!(b.c(vec«!»)) }",
366 "fn a() { test_macro!(b.c(vec!«[]»)) }",
367 "fn a() { test_macro!(b.c(vec![«d»])) }",
368 "fn a() { test_macro!(b.c(vec![d«.»])) }",
369 "fn a() { test_macro!(b.c(vec![d.«e»])) }",
370 ],
371 cx,
372 );
373
374 assert_capture_ranges(
375 &syntax_map,
376 &buffer,
377 &["property"],
378 "fn a() { test_macro!(b.«c»(vec![d.«e»])) }",
379 );
380}
381
382#[gpui::test]
383fn test_pasting_new_injection_line_between_others(cx: &mut App) {
384 let (buffer, syntax_map) = test_edit_sequence(
385 "Rust",
386 &[
387 "
388 fn a() {
389 b!(B {});
390 c!(C {});
391 d!(D {});
392 e!(E {});
393 f!(F {});
394 g!(G {});
395 }
396 ",
397 "
398 fn a() {
399 b!(B {});
400 c!(C {});
401 d!(D {});
402 « h!(H {});
403 » e!(E {});
404 f!(F {});
405 g!(G {});
406 }
407 ",
408 ],
409 cx,
410 );
411
412 assert_capture_ranges(
413 &syntax_map,
414 &buffer,
415 &["type"],
416 "
417 fn a() {
418 b!(«B» {});
419 c!(«C» {});
420 d!(«D» {});
421 h!(«H» {});
422 e!(«E» {});
423 f!(«F» {});
424 g!(«G» {});
425 }
426 ",
427 );
428}
429
430#[gpui::test]
431fn test_joining_injections_with_child_injections(cx: &mut App) {
432 let (buffer, syntax_map) = test_edit_sequence(
433 "Rust",
434 &[
435 "
436 fn a() {
437 b!(
438 c![one.two.three],
439 d![four.five.six],
440 );
441 e!(
442 f![seven.eight],
443 );
444 }
445 ",
446 "
447 fn a() {
448 b!(
449 c![one.two.three],
450 d![four.five.six],
451 ˇ f![seven.eight],
452 );
453 }
454 ",
455 ],
456 cx,
457 );
458
459 assert_capture_ranges(
460 &syntax_map,
461 &buffer,
462 &["property"],
463 "
464 fn a() {
465 b!(
466 c![one.«two».«three»],
467 d![four.«five».«six»],
468 f![seven.«eight»],
469 );
470 }
471 ",
472 );
473}
474
475#[gpui::test]
476fn test_editing_edges_of_injection(cx: &mut App) {
477 test_edit_sequence(
478 "Rust",
479 &[
480 "
481 fn a() {
482 b!(c!())
483 }
484 ",
485 "
486 fn a() {
487 «d»!(c!())
488 }
489 ",
490 "
491 fn a() {
492 «e»d!(c!())
493 }
494 ",
495 "
496 fn a() {
497 ed!«[»c!()«]»
498 }
499 ",
500 ],
501 cx,
502 );
503}
504
505#[gpui::test]
506fn test_edits_preceding_and_intersecting_injection(cx: &mut App) {
507 test_edit_sequence(
508 "Rust",
509 &[
510 //
511 "const aaaaaaaaaaaa: B = c!(d(e.f));",
512 "const aˇa: B = c!(d(eˇ));",
513 ],
514 cx,
515 );
516}
517
518#[gpui::test]
519fn test_non_local_changes_create_injections(cx: &mut App) {
520 test_edit_sequence(
521 "Rust",
522 &[
523 "
524 // a! {
525 static B: C = d;
526 // }
527 ",
528 "
529 ˇa! {
530 static B: C = d;
531 ˇ}
532 ",
533 ],
534 cx,
535 );
536}
537
538#[gpui::test]
539fn test_creating_many_injections_in_one_edit(cx: &mut App) {
540 test_edit_sequence(
541 "Rust",
542 &[
543 "
544 fn a() {
545 one(Two::three(3));
546 four(Five::six(6));
547 seven(Eight::nine(9));
548 }
549 ",
550 "
551 fn a() {
552 one«!»(Two::three(3));
553 four«!»(Five::six(6));
554 seven«!»(Eight::nine(9));
555 }
556 ",
557 "
558 fn a() {
559 one!(Two::three«!»(3));
560 four!(Five::six«!»(6));
561 seven!(Eight::nine«!»(9));
562 }
563 ",
564 ],
565 cx,
566 );
567}
568
569#[gpui::test]
570fn test_editing_across_injection_boundary(cx: &mut App) {
571 test_edit_sequence(
572 "Rust",
573 &[
574 "
575 fn one() {
576 two();
577 three!(
578 three.four,
579 five.six,
580 );
581 }
582 ",
583 "
584 fn one() {
585 two();
586 th«irty_five![»
587 three.four,
588 five.six,
589 « seven.eight,
590 ];»
591 }
592 ",
593 ],
594 cx,
595 );
596}
597
598#[gpui::test]
599fn test_removing_injection_by_replacing_across_boundary(cx: &mut App) {
600 test_edit_sequence(
601 "Rust",
602 &[
603 "
604 fn one() {
605 two!(
606 three.four,
607 );
608 }
609 ",
610 "
611 fn one() {
612 t«en
613 .eleven(
614 twelve,
615 »
616 three.four,
617 );
618 }
619 ",
620 ],
621 cx,
622 );
623}
624
625#[gpui::test]
626fn test_combined_injections_simple(cx: &mut App) {
627 let (buffer, syntax_map) = test_edit_sequence(
628 "ERB",
629 &[
630 "
631 <body>
632 <% if @one %>
633 <div class=one>
634 <% else %>
635 <div class=two>
636 <% end %>
637 </div>
638 </body>
639 ",
640 "
641 <body>
642 <% if @one %>
643 <div class=one>
644 ˇ else ˇ
645 <div class=two>
646 <% end %>
647 </div>
648 </body>
649 ",
650 "
651 <body>
652 <% if @one «;» end %>
653 </div>
654 </body>
655 ",
656 ],
657 cx,
658 );
659
660 assert_capture_ranges(
661 &syntax_map,
662 &buffer,
663 &["tag", "ivar"],
664 "
665 <«body»>
666 <% if «@one» ; end %>
667 </«div»>
668 </«body»>
669 ",
670 );
671}
672
673#[gpui::test]
674fn test_combined_injections_empty_ranges(cx: &mut App) {
675 test_edit_sequence(
676 "ERB",
677 &[
678 "
679 <% if @one %>
680 <% else %>
681 <% end %>
682 ",
683 "
684 <% if @one %>
685 ˇ<% end %>
686 ",
687 ],
688 cx,
689 );
690}
691
692#[gpui::test]
693fn test_combined_injections_edit_edges_of_ranges(cx: &mut App) {
694 let (buffer, syntax_map) = test_edit_sequence(
695 "ERB",
696 &[
697 "
698 <%= one @two %>
699 <%= three @four %>
700 ",
701 "
702 <%= one @two %ˇ
703 <%= three @four %>
704 ",
705 "
706 <%= one @two %«>»
707 <%= three @four %>
708 ",
709 ],
710 cx,
711 );
712
713 assert_capture_ranges(
714 &syntax_map,
715 &buffer,
716 &["tag", "ivar"],
717 "
718 <%= one «@two» %>
719 <%= three «@four» %>
720 ",
721 );
722}
723
724#[gpui::test]
725fn test_combined_injections_splitting_some_injections(cx: &mut App) {
726 let (_buffer, _syntax_map) = test_edit_sequence(
727 "ERB",
728 &[
729 r#"
730 <%A if b(:c) %>
731 d
732 <% end %>
733 eee
734 <% f %>
735 "#,
736 r#"
737 <%« AAAAAAA %>
738 hhhhhhh
739 <%=» if b(:c) %>
740 d
741 <% end %>
742 eee
743 <% f %>
744 "#,
745 ],
746 cx,
747 );
748}
749
750#[gpui::test]
751fn test_combined_injections_editing_after_last_injection(cx: &mut App) {
752 test_edit_sequence(
753 "ERB",
754 &[
755 r#"
756 <% foo %>
757 <div></div>
758 <% bar %>
759 "#,
760 r#"
761 <% foo %>
762 <div></div>
763 <% bar %>«
764 more text»
765 "#,
766 ],
767 cx,
768 );
769}
770
771#[gpui::test]
772fn test_combined_injections_inside_injections(cx: &mut App) {
773 let (buffer, syntax_map) = test_edit_sequence(
774 "Markdown",
775 &[
776 r#"
777 here is
778 some
779 ERB code:
780
781 ```erb
782 <ul>
783 <% people.each do |person| %>
784 <li><%= person.name %></li>
785 <li><%= person.age %></li>
786 <% end %>
787 </ul>
788 ```
789 "#,
790 r#"
791 here is
792 some
793 ERB code:
794
795 ```erb
796 <ul>
797 <% people«2».each do |person| %>
798 <li><%= person.name %></li>
799 <li><%= person.age %></li>
800 <% end %>
801 </ul>
802 ```
803 "#,
804 // Inserting a comment character inside one code directive
805 // does not cause the other code directive to become a comment,
806 // because newlines are included in between each injection range.
807 r#"
808 here is
809 some
810 ERB code:
811
812 ```erb
813 <ul>
814 <% people2.each do |person| %>
815 <li><%= «# »person.name %></li>
816 <li><%= person.age %></li>
817 <% end %>
818 </ul>
819 ```
820 "#,
821 ],
822 cx,
823 );
824
825 // Check that the code directive below the ruby comment is
826 // not parsed as a comment.
827 assert_capture_ranges(
828 &syntax_map,
829 &buffer,
830 &["method"],
831 "
832 here is
833 some
834 ERB code:
835
836 ```erb
837 <ul>
838 <% people2.«each» do |person| %>
839 <li><%= # person.name %></li>
840 <li><%= person.«age» %></li>
841 <% end %>
842 </ul>
843 ```
844 ",
845 );
846}
847
848#[gpui::test]
849fn test_empty_combined_injections_inside_injections(cx: &mut App) {
850 let (buffer, syntax_map) = test_edit_sequence(
851 "Markdown",
852 &[r#"
853 ```erb
854 hello
855 ```
856
857 goodbye
858 "#],
859 cx,
860 );
861
862 assert_layers_for_range(
863 &syntax_map,
864 &buffer,
865 Point::new(0, 0)..Point::new(5, 0),
866 &[
867 // Markdown document
868 "(document (section (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter)) (paragraph (inline))))",
869 // ERB template in the code block
870 "(template...",
871 // Markdown inline content
872 "(inline)",
873 // The ruby syntax tree should be empty, since there are
874 // no interpolations in the ERB template.
875 "(program)",
876 // HTML within the ERB
877 "(document (text))",
878 ],
879 );
880}
881
882#[gpui::test]
883fn test_comment_triggered_injection_toggle(cx: &mut App) {
884 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
885
886 let python = Arc::new(python_lang());
887 let comment = Arc::new(comment_lang());
888 registry.add(python.clone());
889 registry.add(comment);
890 // Note: SQL is an extension language (not built-in as of v0.222.0), so we can use
891 // contains_unknown_injections() to detect when the injection is triggered.
892 // We register a mock "comment" language because Python injects all comments as
893 // language "comment", and we only want SQL to trigger unknown injections.
894
895 // Start with Python code with incomplete #sq comment (not enough to trigger injection)
896 let mut buffer = Buffer::new(
897 ReplicaId::LOCAL,
898 BufferId::new(1).unwrap(),
899 "#sq\ncmd = \"SELECT col1, col2 FROM tbl\"".to_string(),
900 );
901
902 let mut syntax_map = SyntaxMap::new(&buffer);
903 syntax_map.set_language_registry(registry);
904 syntax_map.reparse(python.clone(), &buffer);
905
906 // Should have no unknown injections (#sq doesn't match the injection pattern)
907 assert!(
908 !syntax_map.contains_unknown_injections(),
909 "Expected no unknown injections with incomplete #sq comment"
910 );
911
912 // Complete the comment by adding 'l' to make #sql
913 let sq_end = buffer.as_rope().to_string().find('\n').unwrap();
914 buffer.edit([(sq_end..sq_end, "l")]);
915 syntax_map.interpolate(&buffer);
916 syntax_map.reparse(python.clone(), &buffer);
917
918 // Should now have unknown injections (SQL injection triggered but SQL not registered)
919 assert!(
920 syntax_map.contains_unknown_injections(),
921 "Expected unknown injections after completing #sql comment"
922 );
923
924 // Remove the 'l' to go back to #sq
925 let l_position = buffer.as_rope().to_string().find("l\n").unwrap();
926 buffer.edit([(l_position..l_position + 1, "")]);
927 syntax_map.interpolate(&buffer);
928 syntax_map.reparse(python, &buffer);
929
930 // Should have no unknown injections again - SQL injection should be invalidated
931 assert!(
932 !syntax_map.contains_unknown_injections(),
933 "Expected no unknown injections after removing 'l' from #sql comment"
934 );
935}
936
937#[gpui::test]
938fn test_syntax_map_languages_loading_with_erb(cx: &mut App) {
939 let text = r#"
940 <body>
941 <% if @one %>
942 <div class=one>
943 <% else %>
944 <div class=two>
945 <% end %>
946 </div>
947 </body>
948 "#
949 .unindent();
950
951 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
952 let mut buffer = Buffer::new(ReplicaId::LOCAL, BufferId::new(1).unwrap(), text);
953
954 let mut syntax_map = SyntaxMap::new(&buffer);
955 syntax_map.set_language_registry(registry.clone());
956
957 let language = Arc::new(erb_lang());
958
959 log::info!("parsing");
960 registry.add(language.clone());
961 syntax_map.reparse(language.clone(), &buffer);
962
963 log::info!("loading html");
964 registry.add(Arc::new(html_lang()));
965 syntax_map.reparse(language.clone(), &buffer);
966
967 log::info!("loading ruby");
968 registry.add(Arc::new(ruby_lang()));
969 syntax_map.reparse(language.clone(), &buffer);
970
971 assert_capture_ranges(
972 &syntax_map,
973 &buffer,
974 &["tag", "ivar"],
975 "
976 <«body»>
977 <% if «@one» %>
978 <«div» class=one>
979 <% else %>
980 <«div» class=two>
981 <% end %>
982 </«div»>
983 </«body»>
984 ",
985 );
986
987 let text = r#"
988 <body>
989 <% if @one«_hundred» %>
990 <div class=one>
991 <% else %>
992 <div class=two>
993 <% end %>
994 </div>
995 </body>
996 "#
997 .unindent();
998
999 log::info!("editing");
1000 buffer.edit_via_marked_text(&text);
1001 syntax_map.interpolate(&buffer);
1002 syntax_map.reparse(language, &buffer);
1003
1004 assert_capture_ranges(
1005 &syntax_map,
1006 &buffer,
1007 &["tag", "ivar"],
1008 "
1009 <«body»>
1010 <% if «@one_hundred» %>
1011 <«div» class=one>
1012 <% else %>
1013 <«div» class=two>
1014 <% end %>
1015 </«div»>
1016 </«body»>
1017 ",
1018 );
1019}
1020
1021#[gpui::test(iterations = 50)]
1022fn test_random_syntax_map_edits_rust_macros(rng: StdRng, cx: &mut App) {
1023 let text = r#"
1024 fn test_something() {
1025 let vec = vec![5, 1, 3, 8];
1026 assert_eq!(
1027 vec
1028 .into_iter()
1029 .map(|i| i * 2)
1030 .collect::<Vec<usize>>(),
1031 vec![
1032 5 * 2, 1 * 2, 3 * 2, 8 * 2
1033 ],
1034 );
1035 }
1036 "#
1037 .unindent()
1038 .repeat(2);
1039
1040 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
1041 let language = rust_lang();
1042 registry.add(language.clone());
1043
1044 test_random_edits(text, registry, language, rng);
1045}
1046
1047#[gpui::test(iterations = 50)]
1048fn test_random_syntax_map_edits_with_erb(rng: StdRng, cx: &mut App) {
1049 let text = r#"
1050 <div id="main">
1051 <% if one?(:two) %>
1052 <p class="three" four>
1053 <%= yield :five %>
1054 </p>
1055 <% elsif Six.seven(8) %>
1056 <p id="three" four>
1057 <%= yield :five %>
1058 </p>
1059 <% else %>
1060 <span>Ok</span>
1061 <% end %>
1062 </div>
1063 "#
1064 .unindent()
1065 .repeat(5);
1066
1067 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
1068 let language = Arc::new(erb_lang());
1069 registry.add(language.clone());
1070 registry.add(Arc::new(ruby_lang()));
1071 registry.add(Arc::new(html_lang()));
1072
1073 test_random_edits(text, registry, language, rng);
1074}
1075
1076#[gpui::test(iterations = 50)]
1077fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) {
1078 let text = r#"
1079 defmodule TheModule do
1080 def the_method(assigns) do
1081 ~H"""
1082 <%= if @empty do %>
1083 <div class="h-4"></div>
1084 <% else %>
1085 <div class="max-w-2xl w-full animate-pulse">
1086 <div class="flex-1 space-y-4">
1087 <div class={[@bg_class, "h-4 rounded-lg w-3/4"]}></div>
1088 <div class={[@bg_class, "h-4 rounded-lg"]}></div>
1089 <div class={[@bg_class, "h-4 rounded-lg w-5/6"]}></div>
1090 </div>
1091 </div>
1092 <% end %>
1093 """
1094 end
1095 end
1096 "#
1097 .unindent()
1098 .repeat(3);
1099
1100 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
1101 let language = Arc::new(elixir_lang());
1102 registry.add(language.clone());
1103 registry.add(Arc::new(heex_lang()));
1104 registry.add(Arc::new(html_lang()));
1105
1106 test_random_edits(text, registry, language, rng);
1107}
1108
1109fn test_random_edits(
1110 text: String,
1111 registry: Arc<LanguageRegistry>,
1112 language: Arc<Language>,
1113 mut rng: StdRng,
1114) {
1115 let operations = env::var("OPERATIONS")
1116 .map(|i| i.parse().expect("invalid `OPERATIONS` variable"))
1117 .unwrap_or(10);
1118
1119 let mut buffer = Buffer::new(ReplicaId::LOCAL, BufferId::new(1).unwrap(), text);
1120
1121 let mut syntax_map = SyntaxMap::new(&buffer);
1122 syntax_map.set_language_registry(registry.clone());
1123 syntax_map.reparse(language.clone(), &buffer);
1124
1125 let mut reference_syntax_map = SyntaxMap::new(&buffer);
1126 reference_syntax_map.set_language_registry(registry);
1127
1128 log::info!("initial text:\n{}", buffer.text());
1129
1130 for _ in 0..operations {
1131 let prev_buffer = buffer.snapshot().clone();
1132 let prev_syntax_map = syntax_map.snapshot();
1133
1134 buffer.randomly_edit(&mut rng, 3);
1135 log::info!("text:\n{}", buffer.text());
1136
1137 syntax_map.interpolate(&buffer);
1138 check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer);
1139
1140 syntax_map.reparse(language.clone(), &buffer);
1141
1142 reference_syntax_map.clear(&buffer);
1143 reference_syntax_map.reparse(language.clone(), &buffer);
1144 }
1145
1146 for i in 0..operations {
1147 let i = operations - i - 1;
1148 buffer.undo();
1149 log::info!("undoing operation {}", i);
1150 log::info!("text:\n{}", buffer.text());
1151
1152 syntax_map.interpolate(&buffer);
1153 syntax_map.reparse(language.clone(), &buffer);
1154
1155 reference_syntax_map.clear(&buffer);
1156 reference_syntax_map.reparse(language.clone(), &buffer);
1157 assert_eq!(
1158 syntax_map.layers(&buffer).len(),
1159 reference_syntax_map.layers(&buffer).len(),
1160 "wrong number of layers after undoing edit {i}"
1161 );
1162 }
1163
1164 let layers = syntax_map.layers(&buffer);
1165 let reference_layers = reference_syntax_map.layers(&buffer);
1166 for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) {
1167 assert_eq!(
1168 edited_layer.node().to_sexp(),
1169 reference_layer.node().to_sexp()
1170 );
1171 assert_eq!(edited_layer.node().range(), reference_layer.node().range());
1172 }
1173}
1174
1175fn check_interpolation(
1176 old_syntax_map: &SyntaxSnapshot,
1177 new_syntax_map: &SyntaxSnapshot,
1178 old_buffer: &BufferSnapshot,
1179 new_buffer: &BufferSnapshot,
1180) {
1181 let edits = new_buffer
1182 .edits_since::<usize>(old_buffer.version())
1183 .collect::<Vec<_>>();
1184
1185 for (old_layer, new_layer) in old_syntax_map
1186 .layers
1187 .iter()
1188 .zip(new_syntax_map.layers.iter())
1189 {
1190 assert_eq!(old_layer.range, new_layer.range);
1191 let Some(old_tree) = old_layer.content.tree() else {
1192 continue;
1193 };
1194 let Some(new_tree) = new_layer.content.tree() else {
1195 continue;
1196 };
1197 let old_start_byte = old_layer.range.start.to_offset(old_buffer);
1198 let new_start_byte = new_layer.range.start.to_offset(new_buffer);
1199 let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point();
1200 let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point();
1201 let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point);
1202 let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point);
1203 check_node_edits(
1204 old_layer.depth,
1205 &old_layer.range,
1206 old_node,
1207 new_node,
1208 old_buffer,
1209 new_buffer,
1210 &edits,
1211 );
1212 }
1213
1214 fn check_node_edits(
1215 depth: usize,
1216 range: &Range<Anchor>,
1217 old_node: Node,
1218 new_node: Node,
1219 old_buffer: &BufferSnapshot,
1220 new_buffer: &BufferSnapshot,
1221 edits: &[text::Edit<usize>],
1222 ) {
1223 assert_eq!(old_node.kind(), new_node.kind());
1224
1225 let old_range = old_node.byte_range();
1226 let new_range = new_node.byte_range();
1227
1228 let is_edited = edits
1229 .iter()
1230 .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start);
1231 if is_edited {
1232 assert!(
1233 new_node.has_changes(),
1234 concat!(
1235 "failed to mark node as edited.\n",
1236 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
1237 "node kind: {}, old node range: {:?}, new node range: {:?}",
1238 ),
1239 depth,
1240 range.to_offset(old_buffer),
1241 range.to_offset(new_buffer),
1242 new_node.kind(),
1243 old_range,
1244 new_range,
1245 );
1246 }
1247
1248 if !new_node.has_changes() {
1249 assert_eq!(
1250 old_buffer
1251 .text_for_range(old_range.clone())
1252 .collect::<String>(),
1253 new_buffer
1254 .text_for_range(new_range.clone())
1255 .collect::<String>(),
1256 concat!(
1257 "mismatched text for node\n",
1258 "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n",
1259 "node kind: {}, old node range:{:?}, new node range:{:?}",
1260 ),
1261 depth,
1262 range.to_offset(old_buffer),
1263 range.to_offset(new_buffer),
1264 new_node.kind(),
1265 old_range,
1266 new_range,
1267 );
1268 }
1269
1270 for i in 0..new_node.child_count() {
1271 check_node_edits(
1272 depth,
1273 range,
1274 old_node.child(i as u32).unwrap(),
1275 new_node.child(i as u32).unwrap(),
1276 old_buffer,
1277 new_buffer,
1278 edits,
1279 )
1280 }
1281 }
1282}
1283
1284fn test_edit_sequence(language_name: &str, steps: &[&str], cx: &mut App) -> (Buffer, SyntaxMap) {
1285 let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
1286 registry.add(Arc::new(elixir_lang()));
1287 registry.add(Arc::new(heex_lang()));
1288 registry.add(rust_lang());
1289 registry.add(Arc::new(ruby_lang()));
1290 registry.add(Arc::new(html_lang()));
1291 registry.add(Arc::new(erb_lang()));
1292 registry.add(markdown_lang());
1293 registry.add(Arc::new(markdown_inline_lang()));
1294
1295 let language = registry
1296 .language_for_name(language_name)
1297 .now_or_never()
1298 .unwrap()
1299 .unwrap();
1300 let mut buffer = Buffer::new(ReplicaId::LOCAL, BufferId::new(1).unwrap(), "");
1301
1302 let mut mutated_syntax_map = SyntaxMap::new(&buffer);
1303 mutated_syntax_map.set_language_registry(registry.clone());
1304 mutated_syntax_map.reparse(language.clone(), &buffer);
1305
1306 for (i, marked_string) in steps.iter().enumerate() {
1307 let marked_string = marked_string.unindent();
1308 log::info!("incremental parse {i}: {marked_string:?}");
1309 buffer.edit_via_marked_text(&marked_string);
1310
1311 // Reparse the syntax map
1312 mutated_syntax_map.interpolate(&buffer);
1313 mutated_syntax_map.reparse(language.clone(), &buffer);
1314
1315 // Create a second syntax map from scratch
1316 log::info!("fresh parse {i}: {marked_string:?}");
1317 let mut reference_syntax_map = SyntaxMap::new(&buffer);
1318 reference_syntax_map.set_language_registry(registry.clone());
1319 reference_syntax_map.reparse(language.clone(), &buffer);
1320
1321 // Compare the mutated syntax map to the new syntax map
1322 let mutated_layers = mutated_syntax_map.layers(&buffer);
1323 let reference_layers = reference_syntax_map.layers(&buffer);
1324 assert_eq!(
1325 mutated_layers.len(),
1326 reference_layers.len(),
1327 "wrong number of layers at step {i}"
1328 );
1329 for (edited_layer, reference_layer) in
1330 mutated_layers.into_iter().zip(reference_layers.into_iter())
1331 {
1332 assert_eq!(
1333 edited_layer.node().to_sexp(),
1334 reference_layer.node().to_sexp(),
1335 "different layer at step {i}"
1336 );
1337 assert_eq!(
1338 edited_layer.node().range(),
1339 reference_layer.node().range(),
1340 "different layer at step {i}"
1341 );
1342 }
1343 }
1344
1345 (buffer, mutated_syntax_map)
1346}
1347
1348fn html_lang() -> Language {
1349 Language::new(
1350 LanguageConfig {
1351 name: "HTML".into(),
1352 matcher: LanguageMatcher {
1353 path_suffixes: vec!["html".to_string()],
1354 ..Default::default()
1355 },
1356 ..Default::default()
1357 },
1358 Some(tree_sitter_html::LANGUAGE.into()),
1359 )
1360 .with_highlights_query(
1361 r#"
1362 (tag_name) @tag
1363 (erroneous_end_tag_name) @tag
1364 (attribute_name) @property
1365 "#,
1366 )
1367 .unwrap()
1368}
1369
1370fn ruby_lang() -> Language {
1371 Language::new(
1372 LanguageConfig {
1373 name: "Ruby".into(),
1374 matcher: LanguageMatcher {
1375 path_suffixes: vec!["rb".to_string()],
1376 ..Default::default()
1377 },
1378 ..Default::default()
1379 },
1380 Some(tree_sitter_ruby::LANGUAGE.into()),
1381 )
1382 .with_highlights_query(
1383 r#"
1384 ["if" "do" "else" "end"] @keyword
1385 (instance_variable) @ivar
1386 (call method: (identifier) @method)
1387 "#,
1388 )
1389 .unwrap()
1390}
1391
1392fn erb_lang() -> Language {
1393 Language::new(
1394 LanguageConfig {
1395 name: "ERB".into(),
1396 matcher: LanguageMatcher {
1397 path_suffixes: vec!["erb".to_string()],
1398 ..Default::default()
1399 },
1400 ..Default::default()
1401 },
1402 Some(tree_sitter_embedded_template::LANGUAGE.into()),
1403 )
1404 .with_highlights_query(
1405 r#"
1406 ["<%" "%>"] @keyword
1407 "#,
1408 )
1409 .unwrap()
1410 .with_injection_query(
1411 r#"
1412 (
1413 (code) @injection.content
1414 (#set! injection.language "ruby")
1415 (#set! injection.combined)
1416 )
1417
1418 (
1419 (content) @injection.content
1420 (#set! injection.language "html")
1421 (#set! injection.combined)
1422 )
1423 "#,
1424 )
1425 .unwrap()
1426}
1427
1428fn elixir_lang() -> Language {
1429 Language::new(
1430 LanguageConfig {
1431 name: "Elixir".into(),
1432 matcher: LanguageMatcher {
1433 path_suffixes: vec!["ex".into()],
1434 ..Default::default()
1435 },
1436 ..Default::default()
1437 },
1438 Some(tree_sitter_elixir::LANGUAGE.into()),
1439 )
1440 .with_highlights_query(
1441 r#"
1442
1443 "#,
1444 )
1445 .unwrap()
1446}
1447
1448fn heex_lang() -> Language {
1449 Language::new(
1450 LanguageConfig {
1451 name: "HEEx".into(),
1452 matcher: LanguageMatcher {
1453 path_suffixes: vec!["heex".into()],
1454 ..Default::default()
1455 },
1456 ..Default::default()
1457 },
1458 Some(tree_sitter_heex::LANGUAGE.into()),
1459 )
1460 .with_injection_query(
1461 r#"
1462 (
1463 (directive
1464 [
1465 (partial_expression_value)
1466 (expression_value)
1467 (ending_expression_value)
1468 ] @injection.content)
1469 (#set! injection.language "elixir")
1470 (#set! injection.combined)
1471 )
1472
1473 ((expression (expression_value) @injection.content)
1474 (#set! injection.language "elixir"))
1475 "#,
1476 )
1477 .unwrap()
1478}
1479
1480fn python_lang() -> Language {
1481 Language::new(
1482 LanguageConfig {
1483 name: "Python".into(),
1484 matcher: LanguageMatcher {
1485 path_suffixes: vec!["py".to_string()],
1486 ..Default::default()
1487 },
1488 line_comments: vec!["# ".into()],
1489 ..Default::default()
1490 },
1491 Some(tree_sitter_python::LANGUAGE.into()),
1492 )
1493 .with_queries(LanguageQueries {
1494 injections: Some(Cow::from(include_str!(
1495 "../../../languages/src/python/injections.scm"
1496 ))),
1497 ..Default::default()
1498 })
1499 .expect("Could not parse Python queries")
1500}
1501
1502fn comment_lang() -> Language {
1503 // Mock "comment" language to satisfy Python's comment injection.
1504 // Uses JSON grammar as a stand-in since we just need it to be registered.
1505 Language::new(
1506 LanguageConfig {
1507 name: "comment".into(),
1508 ..Default::default()
1509 },
1510 Some(tree_sitter_json::LANGUAGE.into()),
1511 )
1512}
1513
1514fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
1515 let start = buffer.as_rope().to_string().find(text).unwrap();
1516 start..start + text.len()
1517}
1518
1519#[track_caller]
1520fn assert_layers_for_range(
1521 syntax_map: &SyntaxMap,
1522 buffer: &BufferSnapshot,
1523 range: Range<Point>,
1524 expected_layers: &[&str],
1525) {
1526 let layers = syntax_map
1527 .layers_for_range(range, buffer, true)
1528 .collect::<Vec<_>>();
1529 assert_eq!(
1530 layers.len(),
1531 expected_layers.len(),
1532 "wrong number of layers"
1533 );
1534 for (i, (layer, expected_s_exp)) in layers.iter().zip(expected_layers.iter()).enumerate() {
1535 let actual_s_exp = layer.node().to_sexp();
1536 assert!(
1537 string_contains_sequence(
1538 &actual_s_exp,
1539 &expected_s_exp.split("...").collect::<Vec<_>>()
1540 ),
1541 "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}",
1542 );
1543 }
1544}
1545
1546#[track_caller]
1547fn assert_capture_ranges(
1548 syntax_map: &SyntaxMap,
1549 buffer: &BufferSnapshot,
1550 highlight_query_capture_names: &[&str],
1551 marked_string: &str,
1552) {
1553 let mut actual_ranges = Vec::<Range<usize>>::new();
1554 let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| {
1555 grammar
1556 .highlights_config
1557 .as_ref()
1558 .map(|config| &config.query)
1559 });
1560 let queries = captures
1561 .grammars()
1562 .iter()
1563 .map(|grammar| &grammar.highlights_config.as_ref().unwrap().query)
1564 .collect::<Vec<_>>();
1565 for capture in captures {
1566 let name = &queries[capture.grammar_index].capture_names()[capture.index as usize];
1567 if highlight_query_capture_names.contains(name) {
1568 actual_ranges.push(capture.node.byte_range());
1569 }
1570 }
1571 actual_ranges.dedup();
1572
1573 let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false);
1574 assert_eq!(text, buffer.text());
1575 assert_eq!(actual_ranges, expected_ranges);
1576}
1577
1578pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool {
1579 let mut last_part_end = 0;
1580 for part in parts {
1581 if let Some(start_ix) = text[last_part_end..].find(part) {
1582 last_part_end = start_ix + part.len();
1583 } else {
1584 return false;
1585 }
1586 }
1587 true
1588}