1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
24 let snapshot = TabSnapshot {
25 fold_snapshot,
26 tab_size: tab_size.min(MAX_TABS),
27 max_expansion_column: MAX_EXPANSION_COLUMN,
28 version: 0,
29 };
30 (Self(snapshot.clone()), snapshot)
31 }
32
33 #[cfg(test)]
34 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
35 self.0.max_expansion_column = column;
36 self.0.clone()
37 }
38
39 pub fn sync(
40 &mut self,
41 fold_snapshot: FoldSnapshot,
42 mut fold_edits: Vec<FoldEdit>,
43 tab_size: NonZeroU32,
44 ) -> (TabSnapshot, Vec<TabEdit>) {
45 let old_snapshot = &mut self.0;
46 let mut new_snapshot = TabSnapshot {
47 fold_snapshot,
48 tab_size: tab_size.min(MAX_TABS),
49 max_expansion_column: old_snapshot.max_expansion_column,
50 version: old_snapshot.version,
51 };
52
53 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
54 new_snapshot.version += 1;
55 }
56
57 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
58 // Expand each edit to include the next tab on the same line as the edit,
59 // and any subsequent tabs on that line that moved across the tab expansion
60 // boundary.
61 for fold_edit in &mut fold_edits {
62 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
63 let old_end_row_successor_offset = cmp::min(
64 FoldPoint::new(old_end.row() + 1, 0),
65 old_snapshot.fold_snapshot.max_point(),
66 )
67 .to_offset(&old_snapshot.fold_snapshot);
68 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
69
70 let mut offset_from_edit = 0;
71 let mut first_tab_offset = None;
72 let mut last_tab_with_changed_expansion_offset = None;
73 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
74 fold_edit.old.end..old_end_row_successor_offset,
75 false,
76 Highlights::default(),
77 ) {
78 // todo(performance use tabs bitmask)
79 for (ix, _) in chunk.text.match_indices('\t') {
80 let offset_from_edit = offset_from_edit + (ix as u32);
81 if first_tab_offset.is_none() {
82 first_tab_offset = Some(offset_from_edit);
83 }
84
85 let old_column = old_end.column() + offset_from_edit;
86 let new_column = new_end.column() + offset_from_edit;
87 let was_expanded = old_column < old_snapshot.max_expansion_column;
88 let is_expanded = new_column < new_snapshot.max_expansion_column;
89 if was_expanded != is_expanded {
90 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
91 } else if !was_expanded && !is_expanded {
92 break 'outer;
93 }
94 }
95
96 offset_from_edit += chunk.text.len() as u32;
97 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
98 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
99 {
100 break;
101 }
102 }
103
104 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
105 fold_edit.old.end.0 += offset as usize + 1;
106 fold_edit.new.end.0 += offset as usize + 1;
107 }
108 }
109
110 let _old_alloc_ptr = fold_edits.as_ptr();
111 // Combine any edits that overlap due to the expansion.
112 let mut fold_edits = fold_edits.into_iter();
113 if let Some(mut first_edit) = fold_edits.next() {
114 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
115 #[allow(clippy::filter_map_identity)]
116 let mut v: Vec<_> = fold_edits
117 .scan(&mut first_edit, |state, edit| {
118 if state.old.end >= edit.old.start {
119 state.old.end = edit.old.end;
120 state.new.end = edit.new.end;
121 Some(None) // Skip this edit, it's merged
122 } else {
123 let new_state = edit;
124 let result = Some(Some(state.clone())); // Yield the previous edit
125 **state = new_state;
126 result
127 }
128 })
129 .filter_map(|x| x)
130 .collect();
131 v.push(first_edit);
132 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
133 v.into_iter()
134 .map(|fold_edit| {
135 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
136 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
137 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
138 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
139 TabEdit {
140 old: old_snapshot.to_tab_point(old_start)
141 ..old_snapshot.to_tab_point(old_end),
142 new: new_snapshot.to_tab_point(new_start)
143 ..new_snapshot.to_tab_point(new_end),
144 }
145 })
146 .collect()
147 } else {
148 vec![]
149 }
150 } else {
151 new_snapshot.version += 1;
152 vec![TabEdit {
153 old: TabPoint::zero()..old_snapshot.max_point(),
154 new: TabPoint::zero()..new_snapshot.max_point(),
155 }]
156 };
157 *old_snapshot = new_snapshot;
158 (old_snapshot.clone(), tab_edits)
159 }
160}
161
162#[derive(Clone)]
163pub struct TabSnapshot {
164 pub fold_snapshot: FoldSnapshot,
165 pub tab_size: NonZeroU32,
166 pub max_expansion_column: u32,
167 pub version: usize,
168}
169
170impl TabSnapshot {
171 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
172 &self
173 .fold_snapshot
174 .inlay_snapshot
175 .filter_snapshot
176 .buffer_snapshot
177 }
178
179 pub fn line_len(&self, row: u32) -> u32 {
180 let max_point = self.max_point();
181 if row < max_point.row() {
182 self.to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
183 .0
184 .column
185 } else {
186 max_point.column()
187 }
188 }
189
190 pub fn text_summary(&self) -> TextSummary {
191 self.text_summary_for_range(TabPoint::zero()..self.max_point())
192 }
193
194 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
195 let input_start = self.to_fold_point(range.start, Bias::Left).0;
196 let input_end = self.to_fold_point(range.end, Bias::Right).0;
197 let input_summary = self
198 .fold_snapshot
199 .text_summary_for_range(input_start..input_end);
200
201 let line_end = if range.start.row() == range.end.row() {
202 range.end
203 } else {
204 self.max_point()
205 };
206 let first_line_chars = self
207 .chunks(range.start..line_end, false, Highlights::default())
208 .flat_map(|chunk| chunk.text.chars())
209 .take_while(|&c| c != '\n')
210 .count() as u32;
211
212 let last_line_chars = if range.start.row() == range.end.row() {
213 first_line_chars
214 } else {
215 self.chunks(
216 TabPoint::new(range.end.row(), 0)..range.end,
217 false,
218 Highlights::default(),
219 )
220 .flat_map(|chunk| chunk.text.chars())
221 .count() as u32
222 };
223
224 TextSummary {
225 lines: range.end.0 - range.start.0,
226 first_line_chars,
227 last_line_chars,
228 longest_row: input_summary.longest_row,
229 longest_row_chars: input_summary.longest_row_chars,
230 }
231 }
232
233 pub(crate) fn chunks<'a>(
234 &'a self,
235 range: Range<TabPoint>,
236 language_aware: bool,
237 highlights: Highlights<'a>,
238 ) -> TabChunks<'a> {
239 let (input_start, expanded_char_column, to_next_stop) =
240 self.to_fold_point(range.start, Bias::Left);
241 let input_column = input_start.column();
242 let input_start = input_start.to_offset(&self.fold_snapshot);
243 let input_end = self
244 .to_fold_point(range.end, Bias::Right)
245 .0
246 .to_offset(&self.fold_snapshot);
247 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
248 range.end.column() - range.start.column()
249 } else {
250 to_next_stop
251 };
252
253 TabChunks {
254 snapshot: self,
255 fold_chunks: self.fold_snapshot.chunks(
256 input_start..input_end,
257 language_aware,
258 highlights,
259 ),
260 input_column,
261 column: expanded_char_column,
262 max_expansion_column: self.max_expansion_column,
263 output_position: range.start.0,
264 max_output_position: range.end.0,
265 tab_size: self.tab_size,
266 chunk: Chunk {
267 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
268 is_tab: true,
269 ..Default::default()
270 },
271 inside_leading_tab: to_next_stop > 0,
272 }
273 }
274
275 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
276 self.fold_snapshot.row_infos(row)
277 }
278
279 #[cfg(test)]
280 pub fn text(&self) -> String {
281 self.chunks(
282 TabPoint::zero()..self.max_point(),
283 false,
284 Highlights::default(),
285 )
286 .map(|chunk| chunk.text)
287 .collect()
288 }
289
290 pub fn max_point(&self) -> TabPoint {
291 self.to_tab_point(self.fold_snapshot.max_point())
292 }
293
294 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
295 self.to_tab_point(
296 self.fold_snapshot
297 .clip_point(self.to_fold_point(point, bias).0, bias),
298 )
299 }
300
301 pub fn to_tab_point(&self, input: FoldPoint) -> TabPoint {
302 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
303 let tab_cursor = TabStopCursor::new(chunks);
304 let expanded = self.expand_tabs(tab_cursor, input.column());
305 TabPoint::new(input.row(), expanded)
306 }
307
308 pub fn to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
309 let chunks = self
310 .fold_snapshot
311 .chunks_at(FoldPoint::new(output.row(), 0));
312
313 let tab_cursor = TabStopCursor::new(chunks);
314 let expanded = output.column();
315 let (collapsed, expanded_char_column, to_next_stop) =
316 self.collapse_tabs(tab_cursor, expanded, bias);
317
318 (
319 FoldPoint::new(output.row(), collapsed),
320 expanded_char_column,
321 to_next_stop,
322 )
323 }
324
325 pub fn make_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
326 let filter_point = self
327 .fold_snapshot
328 .inlay_snapshot
329 .filter_snapshot
330 .to_filter_point(point);
331 let inlay_point = self
332 .fold_snapshot
333 .inlay_snapshot
334 .to_inlay_point(filter_point);
335 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
336 self.to_tab_point(fold_point)
337 }
338
339 pub fn to_point(&self, point: TabPoint, bias: Bias) -> Point {
340 let fold_point = self.to_fold_point(point, bias).0;
341 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
342 let filter_point = self
343 .fold_snapshot
344 .inlay_snapshot
345 .to_filter_point(inlay_point);
346 self.fold_snapshot
347 .inlay_snapshot
348 .filter_snapshot
349 .to_buffer_point(filter_point, bias)
350 }
351
352 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
353 where
354 I: Iterator<Item = Chunk<'a>>,
355 {
356 let tab_size = self.tab_size.get();
357
358 let end_column = column.min(self.max_expansion_column);
359 let mut seek_target = end_column;
360 let mut tab_count = 0;
361 let mut expanded_tab_len = 0;
362
363 while let Some(tab_stop) = cursor.seek(seek_target) {
364 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
365 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
366 tab_count += 1;
367 expanded_tab_len += tab_len;
368
369 seek_target = end_column - cursor.byte_offset;
370 }
371
372 let left_over_char_bytes = if !cursor.is_char_boundary() {
373 cursor.bytes_until_next_char().unwrap_or(0) as u32
374 } else {
375 0
376 };
377
378 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
379 let expanded_bytes =
380 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
381
382 expanded_bytes + column.saturating_sub(collapsed_bytes)
383 }
384
385 fn collapse_tabs<'a, I>(
386 &self,
387 mut cursor: TabStopCursor<'a, I>,
388 column: u32,
389 bias: Bias,
390 ) -> (u32, u32, u32)
391 where
392 I: Iterator<Item = Chunk<'a>>,
393 {
394 let tab_size = self.tab_size.get();
395 let mut collapsed_column = column;
396 let mut seek_target = column.min(self.max_expansion_column);
397 let mut tab_count = 0;
398 let mut expanded_tab_len = 0;
399
400 while let Some(tab_stop) = cursor.seek(seek_target) {
401 // Calculate how much we want to expand this tab stop (into spaces)
402 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
403 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
404 // Increment tab count
405 tab_count += 1;
406 // The count of how many spaces we've added to this line in place of tab bytes
407 expanded_tab_len += tab_len;
408
409 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
410 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
411
412 // Did we expand past the search target?
413 if expanded_bytes > column {
414 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
415 // We expanded past the search target, so need to account for the offshoot
416 expanded_chars -= expanded_bytes - column;
417 return match bias {
418 Bias::Left => (
419 cursor.byte_offset() - 1,
420 expanded_chars,
421 expanded_bytes - column,
422 ),
423 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
424 };
425 } else {
426 // otherwise we only want to move the cursor collapse column forward
427 collapsed_column = collapsed_column - tab_len + 1;
428 seek_target = (collapsed_column - cursor.byte_offset)
429 .min(self.max_expansion_column - cursor.byte_offset);
430 }
431 }
432
433 let collapsed_bytes = cursor.byte_offset();
434 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
435 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
436 (
437 collapsed_bytes + column.saturating_sub(expanded_bytes),
438 expanded_chars,
439 0,
440 )
441 }
442}
443
444#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
445pub struct TabPoint(pub Point);
446
447impl TabPoint {
448 pub fn new(row: u32, column: u32) -> Self {
449 Self(Point::new(row, column))
450 }
451
452 pub fn zero() -> Self {
453 Self::new(0, 0)
454 }
455
456 pub fn row(self) -> u32 {
457 self.0.row
458 }
459
460 pub fn column(self) -> u32 {
461 self.0.column
462 }
463}
464
465impl From<Point> for TabPoint {
466 fn from(point: Point) -> Self {
467 Self(point)
468 }
469}
470
471pub type TabEdit = text::Edit<TabPoint>;
472
473#[derive(Clone, Debug, Default, Eq, PartialEq)]
474pub struct TextSummary {
475 pub lines: Point,
476 pub first_line_chars: u32,
477 pub last_line_chars: u32,
478 pub longest_row: u32,
479 pub longest_row_chars: u32,
480}
481
482impl<'a> From<&'a str> for TextSummary {
483 fn from(text: &'a str) -> Self {
484 let sum = text::TextSummary::from(text);
485
486 TextSummary {
487 lines: sum.lines,
488 first_line_chars: sum.first_line_chars,
489 last_line_chars: sum.last_line_chars,
490 longest_row: sum.longest_row,
491 longest_row_chars: sum.longest_row_chars,
492 }
493 }
494}
495
496impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
497 fn add_assign(&mut self, other: &'a Self) {
498 let joined_chars = self.last_line_chars + other.first_line_chars;
499 if joined_chars > self.longest_row_chars {
500 self.longest_row = self.lines.row;
501 self.longest_row_chars = joined_chars;
502 }
503 if other.longest_row_chars > self.longest_row_chars {
504 self.longest_row = self.lines.row + other.longest_row;
505 self.longest_row_chars = other.longest_row_chars;
506 }
507
508 if self.lines.row == 0 {
509 self.first_line_chars += other.first_line_chars;
510 }
511
512 if other.lines.row == 0 {
513 self.last_line_chars += other.first_line_chars;
514 } else {
515 self.last_line_chars = other.last_line_chars;
516 }
517
518 self.lines += &other.lines;
519 }
520}
521
522pub struct TabChunks<'a> {
523 snapshot: &'a TabSnapshot,
524 max_expansion_column: u32,
525 max_output_position: Point,
526 tab_size: NonZeroU32,
527 // region: iteration state
528 fold_chunks: FoldChunks<'a>,
529 chunk: Chunk<'a>,
530 column: u32,
531 output_position: Point,
532 input_column: u32,
533 inside_leading_tab: bool,
534 // endregion: iteration state
535}
536
537impl TabChunks<'_> {
538 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
539 let (input_start, expanded_char_column, to_next_stop) =
540 self.snapshot.to_fold_point(range.start, Bias::Left);
541 let input_column = input_start.column();
542 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
543 let input_end = self
544 .snapshot
545 .to_fold_point(range.end, Bias::Right)
546 .0
547 .to_offset(&self.snapshot.fold_snapshot);
548 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
549 range.end.column() - range.start.column()
550 } else {
551 to_next_stop
552 };
553
554 self.fold_chunks.seek(input_start..input_end);
555 self.input_column = input_column;
556 self.column = expanded_char_column;
557 self.output_position = range.start.0;
558 self.max_output_position = range.end.0;
559 self.chunk = Chunk {
560 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
561 is_tab: true,
562 chars: 1u128.unbounded_shl(to_next_stop) - 1,
563 ..Default::default()
564 };
565 self.inside_leading_tab = to_next_stop > 0;
566 }
567}
568
569impl<'a> Iterator for TabChunks<'a> {
570 type Item = Chunk<'a>;
571
572 fn next(&mut self) -> Option<Self::Item> {
573 if self.chunk.text.is_empty() {
574 if let Some(chunk) = self.fold_chunks.next() {
575 self.chunk = chunk;
576 if self.inside_leading_tab {
577 self.chunk.text = &self.chunk.text[1..];
578 self.inside_leading_tab = false;
579 self.input_column += 1;
580 }
581 } else {
582 return None;
583 }
584 }
585
586 //todo(improve performance by using tab cursor)
587 for (ix, c) in self.chunk.text.char_indices() {
588 match c {
589 '\t' if ix > 0 => {
590 let (prefix, suffix) = self.chunk.text.split_at(ix);
591
592 let mask = 1u128.unbounded_shl(ix as u32).wrapping_sub(1);
593 let chars = self.chunk.chars & mask;
594 let tabs = self.chunk.tabs & mask;
595 self.chunk.tabs = self.chunk.tabs.unbounded_shr(ix as u32);
596 self.chunk.chars = self.chunk.chars.unbounded_shr(ix as u32);
597 self.chunk.text = suffix;
598 return Some(Chunk {
599 text: prefix,
600 chars,
601 tabs,
602 ..self.chunk.clone()
603 });
604 }
605 '\t' => {
606 self.chunk.text = &self.chunk.text[1..];
607 self.chunk.tabs >>= 1;
608 self.chunk.chars >>= 1;
609 let tab_size = if self.input_column < self.max_expansion_column {
610 self.tab_size.get()
611 } else {
612 1
613 };
614 let mut len = tab_size - self.column % tab_size;
615 let next_output_position = cmp::min(
616 self.output_position + Point::new(0, len),
617 self.max_output_position,
618 );
619 len = next_output_position.column - self.output_position.column;
620 self.column += len;
621 self.input_column += 1;
622 self.output_position = next_output_position;
623 return Some(Chunk {
624 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
625 is_tab: true,
626 chars: 1u128.unbounded_shl(len) - 1,
627 tabs: 0,
628 ..self.chunk.clone()
629 });
630 }
631 '\n' => {
632 self.column = 0;
633 self.input_column = 0;
634 self.output_position += Point::new(1, 0);
635 }
636 _ => {
637 self.column += 1;
638 if !self.inside_leading_tab {
639 self.input_column += c.len_utf8() as u32;
640 }
641 self.output_position.column += c.len_utf8() as u32;
642 }
643 }
644 }
645
646 Some(mem::take(&mut self.chunk))
647 }
648}
649
650#[cfg(test)]
651mod tests {
652 use super::*;
653 use crate::{
654 MultiBuffer,
655 display_map::{
656 filter_map::FilterMap,
657 fold_map::{FoldMap, FoldOffset},
658 inlay_map::InlayMap,
659 },
660 };
661 use rand::{Rng, prelude::StdRng};
662 use util;
663
664 impl TabSnapshot {
665 fn expected_collapse_tabs(
666 &self,
667 chars: impl Iterator<Item = char>,
668 column: u32,
669 bias: Bias,
670 ) -> (u32, u32, u32) {
671 let tab_size = self.tab_size.get();
672
673 let mut expanded_bytes = 0;
674 let mut expanded_chars = 0;
675 let mut collapsed_bytes = 0;
676 for c in chars {
677 if expanded_bytes >= column {
678 break;
679 }
680 if collapsed_bytes >= self.max_expansion_column {
681 break;
682 }
683
684 if c == '\t' {
685 let tab_len = tab_size - (expanded_chars % tab_size);
686 expanded_chars += tab_len;
687 expanded_bytes += tab_len;
688 if expanded_bytes > column {
689 expanded_chars -= expanded_bytes - column;
690 return match bias {
691 Bias::Left => {
692 (collapsed_bytes, expanded_chars, expanded_bytes - column)
693 }
694 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
695 };
696 }
697 } else {
698 expanded_chars += 1;
699 expanded_bytes += c.len_utf8() as u32;
700 }
701
702 if expanded_bytes > column && matches!(bias, Bias::Left) {
703 expanded_chars -= 1;
704 break;
705 }
706
707 collapsed_bytes += c.len_utf8() as u32;
708 }
709
710 (
711 collapsed_bytes + column.saturating_sub(expanded_bytes),
712 expanded_chars,
713 0,
714 )
715 }
716
717 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
718 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
719 let expanded = self.expected_expand_tabs(chars, input.column());
720 TabPoint::new(input.row(), expanded)
721 }
722
723 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
724 let tab_size = self.tab_size.get();
725
726 let mut expanded_chars = 0;
727 let mut expanded_bytes = 0;
728 let mut collapsed_bytes = 0;
729 let end_column = column.min(self.max_expansion_column);
730 for c in chars {
731 if collapsed_bytes >= end_column {
732 break;
733 }
734 if c == '\t' {
735 let tab_len = tab_size - expanded_chars % tab_size;
736 expanded_bytes += tab_len;
737 expanded_chars += tab_len;
738 } else {
739 expanded_bytes += c.len_utf8() as u32;
740 expanded_chars += 1;
741 }
742 collapsed_bytes += c.len_utf8() as u32;
743 }
744
745 expanded_bytes + column.saturating_sub(collapsed_bytes)
746 }
747
748 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
749 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
750 let expanded = output.column();
751 let (collapsed, expanded_char_column, to_next_stop) =
752 self.expected_collapse_tabs(chars, expanded, bias);
753 (
754 FoldPoint::new(output.row(), collapsed),
755 expanded_char_column,
756 to_next_stop,
757 )
758 }
759 }
760
761 #[gpui::test]
762 fn test_expand_tabs(cx: &mut gpui::App) {
763 let test_values = [
764 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
765 (" \twςe", 4),
766 ("fε", 1),
767 ("i❎\t", 3),
768 ];
769 let buffer = MultiBuffer::build_simple("", cx);
770 let buffer_snapshot = buffer.read(cx).snapshot(cx);
771 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
772 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
773 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
774 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
775
776 for (text, column) in test_values {
777 let mut tabs = 0u128;
778 let mut chars = 0u128;
779 for (idx, c) in text.char_indices() {
780 if c == '\t' {
781 tabs |= 1 << idx;
782 }
783 chars |= 1 << idx;
784 }
785
786 let chunks = [Chunk {
787 text,
788 tabs,
789 chars,
790 ..Default::default()
791 }];
792
793 let cursor = TabStopCursor::new(chunks);
794
795 assert_eq!(
796 tab_snapshot.expected_expand_tabs(text.chars(), column),
797 tab_snapshot.expand_tabs(cursor, column)
798 );
799 }
800 }
801
802 #[gpui::test]
803 fn test_collapse_tabs(cx: &mut gpui::App) {
804 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
805
806 let buffer = MultiBuffer::build_simple(input, cx);
807 let buffer_snapshot = buffer.read(cx).snapshot(cx);
808 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
809 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
810 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
811 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
812
813 for (ix, _) in input.char_indices() {
814 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
815
816 assert_eq!(
817 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
818 tab_snapshot.to_fold_point(range.start, Bias::Left),
819 "Failed with tab_point at column {ix}"
820 );
821 assert_eq!(
822 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
823 tab_snapshot.to_fold_point(range.start, Bias::Right),
824 "Failed with tab_point at column {ix}"
825 );
826
827 assert_eq!(
828 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
829 tab_snapshot.to_fold_point(range.end, Bias::Left),
830 "Failed with tab_point at column {ix}"
831 );
832 assert_eq!(
833 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
834 tab_snapshot.to_fold_point(range.end, Bias::Right),
835 "Failed with tab_point at column {ix}"
836 );
837 }
838 }
839
840 #[gpui::test]
841 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
842 // This test reproduces a specific panic where to_fold_point returns incorrect results
843 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
844
845 let text = "γ\tw⭐\n🍐🍗 \t";
846 let buffer = MultiBuffer::build_simple(text, cx);
847 let buffer_snapshot = buffer.read(cx).snapshot(cx);
848 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
849 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
850 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
851 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
852
853 // This should panic with the expected vs actual mismatch
854 let tab_point = TabPoint::new(0, 9);
855 let result = tab_snapshot.to_fold_point(tab_point, Bias::Left);
856 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
857
858 assert_eq!(result, expected);
859 }
860
861 #[gpui::test(iterations = 100)]
862 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
863 // Generate random input string with up to 200 characters including tabs
864 // to stay within the MAX_EXPANSION_COLUMN limit of 256
865 let len = rng.random_range(0..=2048);
866 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
867 let mut input = String::with_capacity(len);
868
869 for _ in 0..len {
870 if rng.random_bool(0.1) {
871 // 10% chance of inserting a tab
872 input.push('\t');
873 } else {
874 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
875 let ch = loop {
876 let ascii_code = rng.random_range(32..=126); // printable ASCII range
877 let ch = ascii_code as u8 as char;
878 if ch != '\t' {
879 break ch;
880 }
881 };
882 input.push(ch);
883 }
884 }
885
886 let buffer = MultiBuffer::build_simple(&input, cx);
887 let buffer_snapshot = buffer.read(cx).snapshot(cx);
888 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
889 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
890 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
891 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
892 tab_snapshot.max_expansion_column = rng.random_range(0..323);
893 tab_snapshot.tab_size = tab_size;
894
895 for (ix, _) in input.char_indices() {
896 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
897
898 assert_eq!(
899 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
900 tab_snapshot.to_fold_point(range.start, Bias::Left),
901 "Failed with input: {}, with idx: {ix}",
902 input
903 );
904 assert_eq!(
905 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
906 tab_snapshot.to_fold_point(range.start, Bias::Right),
907 "Failed with input: {}, with idx: {ix}",
908 input
909 );
910
911 assert_eq!(
912 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
913 tab_snapshot.to_fold_point(range.end, Bias::Left),
914 "Failed with input: {}, with idx: {ix}",
915 input
916 );
917 assert_eq!(
918 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
919 tab_snapshot.to_fold_point(range.end, Bias::Right),
920 "Failed with input: {}, with idx: {ix}",
921 input
922 );
923 }
924 }
925
926 #[gpui::test]
927 fn test_long_lines(cx: &mut gpui::App) {
928 let max_expansion_column = 12;
929 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
930 let output = "A BC DEF G HI J K L M";
931
932 let buffer = MultiBuffer::build_simple(input, cx);
933 let buffer_snapshot = buffer.read(cx).snapshot(cx);
934 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
935 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
936 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
937 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
938
939 tab_snapshot.max_expansion_column = max_expansion_column;
940 assert_eq!(tab_snapshot.text(), output);
941
942 for (ix, c) in input.char_indices() {
943 assert_eq!(
944 tab_snapshot
945 .chunks(
946 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
947 false,
948 Highlights::default(),
949 )
950 .map(|c| c.text)
951 .collect::<String>(),
952 &output[ix..],
953 "text from index {ix}"
954 );
955
956 if c != '\t' {
957 let input_point = Point::new(0, ix as u32);
958 let output_point = Point::new(0, output.find(c).unwrap() as u32);
959 assert_eq!(
960 tab_snapshot.to_tab_point(FoldPoint(input_point)),
961 TabPoint(output_point),
962 "to_tab_point({input_point:?})"
963 );
964 assert_eq!(
965 tab_snapshot
966 .to_fold_point(TabPoint(output_point), Bias::Left)
967 .0,
968 FoldPoint(input_point),
969 "to_fold_point({output_point:?})"
970 );
971 }
972 }
973 }
974
975 #[gpui::test]
976 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
977 let max_expansion_column = 8;
978 let input = "abcdefg⋯hij";
979
980 let buffer = MultiBuffer::build_simple(input, cx);
981 let buffer_snapshot = buffer.read(cx).snapshot(cx);
982 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
983 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
984 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
985 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
986
987 tab_snapshot.max_expansion_column = max_expansion_column;
988 assert_eq!(tab_snapshot.text(), input);
989 }
990
991 #[gpui::test]
992 fn test_marking_tabs(cx: &mut gpui::App) {
993 let input = "\t \thello";
994
995 let buffer = MultiBuffer::build_simple(input, cx);
996 let buffer_snapshot = buffer.read(cx).snapshot(cx);
997 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
998 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
999 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1000 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1001
1002 assert_eq!(
1003 chunks(&tab_snapshot, TabPoint::zero()),
1004 vec![
1005 (" ".to_string(), true),
1006 (" ".to_string(), false),
1007 (" ".to_string(), true),
1008 ("hello".to_string(), false),
1009 ]
1010 );
1011 assert_eq!(
1012 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1013 vec![
1014 (" ".to_string(), true),
1015 (" ".to_string(), false),
1016 (" ".to_string(), true),
1017 ("hello".to_string(), false),
1018 ]
1019 );
1020
1021 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1022 let mut chunks = Vec::new();
1023 let mut was_tab = false;
1024 let mut text = String::new();
1025 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1026 {
1027 if chunk.is_tab != was_tab {
1028 if !text.is_empty() {
1029 chunks.push((mem::take(&mut text), was_tab));
1030 }
1031 was_tab = chunk.is_tab;
1032 }
1033 text.push_str(chunk.text);
1034 }
1035
1036 if !text.is_empty() {
1037 chunks.push((text, was_tab));
1038 }
1039 chunks
1040 }
1041 }
1042
1043 #[gpui::test(iterations = 100)]
1044 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1045 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1046 let len = rng.random_range(0..30);
1047 let buffer = if rng.random() {
1048 let text = util::RandomCharIter::new(&mut rng)
1049 .take(len)
1050 .collect::<String>();
1051 MultiBuffer::build_simple(&text, cx)
1052 } else {
1053 MultiBuffer::build_random(&mut rng, cx)
1054 };
1055 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1056 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1057
1058 let (filter_map, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
1059 let (mut inlay_map, inlay_snapshot) = InlayMap::new(filter_snapshot);
1060 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1061 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1062 fold_map.randomly_mutate(&mut rng);
1063 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1064 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1065 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1066 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1067
1068 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1069 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1070
1071 let text = text::Rope::from(tabs_snapshot.text().as_str());
1072 log::info!(
1073 "TabMap text (tab size: {}): {:?}",
1074 tab_size,
1075 tabs_snapshot.text(),
1076 );
1077
1078 for _ in 0..5 {
1079 let end_row = rng.random_range(0..=text.max_point().row);
1080 let end_column = rng.random_range(0..=text.line_len(end_row));
1081 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1082 let start_row = rng.random_range(0..=text.max_point().row);
1083 let start_column = rng.random_range(0..=text.line_len(start_row));
1084 let mut start =
1085 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1086 if start > end {
1087 mem::swap(&mut start, &mut end);
1088 }
1089
1090 let expected_text = text
1091 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1092 .collect::<String>();
1093 let expected_summary = TextSummary::from(expected_text.as_str());
1094 assert_eq!(
1095 tabs_snapshot
1096 .chunks(start..end, false, Highlights::default())
1097 .map(|c| c.text)
1098 .collect::<String>(),
1099 expected_text,
1100 "chunks({:?}..{:?})",
1101 start,
1102 end
1103 );
1104
1105 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1106 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1107 actual_summary.longest_row = expected_summary.longest_row;
1108 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1109 }
1110 assert_eq!(actual_summary, expected_summary);
1111 }
1112
1113 for row in 0..=text.max_point().row {
1114 assert_eq!(
1115 tabs_snapshot.line_len(row),
1116 text.line_len(row),
1117 "line_len({row})"
1118 );
1119 }
1120 }
1121
1122 #[gpui::test(iterations = 100)]
1123 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1124 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1125 let len = rng.random_range(0..=2000);
1126
1127 // Generate random text using RandomCharIter
1128 let text = util::RandomCharIter::new(&mut rng)
1129 .take(len)
1130 .collect::<String>();
1131
1132 // Create buffer and tab map
1133 let buffer = MultiBuffer::build_simple(&text, cx);
1134 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1135 let (filter_map, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
1136 let (mut inlay_map, inlay_snapshot) = InlayMap::new(filter_snapshot);
1137 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1138 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1139
1140 let mut next_inlay_id = 0;
1141 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1142 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1143 let max_fold_point = fold_snapshot.max_point();
1144 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1145
1146 // Test random fold points
1147 for _ in 0..50 {
1148 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1149 // Generate random fold point
1150 let row = rng.random_range(0..=max_fold_point.row());
1151 let max_column = if row < max_fold_point.row() {
1152 fold_snapshot.line_len(row)
1153 } else {
1154 max_fold_point.column()
1155 };
1156 let column = rng.random_range(0..=max_column + 10);
1157 let fold_point = FoldPoint::new(row, column);
1158
1159 let actual = tab_snapshot.to_tab_point(fold_point);
1160 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1161
1162 assert_eq!(
1163 actual, expected,
1164 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1165 fold_point, text
1166 );
1167 }
1168 }
1169
1170 #[gpui::test]
1171 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1172 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1173 let buffer = MultiBuffer::build_simple(text, cx);
1174 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1175 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
1176 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
1177 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1178 let chunks = fold_snapshot.chunks(
1179 FoldOffset(0)..fold_snapshot.len(),
1180 false,
1181 Default::default(),
1182 );
1183 let mut cursor = TabStopCursor::new(chunks);
1184 assert!(cursor.seek(0).is_none());
1185 let mut tab_stops = Vec::new();
1186
1187 let mut all_tab_stops = Vec::new();
1188 let mut byte_offset = 0;
1189 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1190 byte_offset += ch.len_utf8() as u32;
1191
1192 if ch == '\t' {
1193 all_tab_stops.push(TabStop {
1194 byte_offset,
1195 char_offset: offset as u32 + 1,
1196 });
1197 }
1198 }
1199
1200 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1201 tab_stops.push(tab_stop);
1202 }
1203 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1204
1205 assert_eq!(cursor.byte_offset(), byte_offset);
1206 }
1207
1208 #[gpui::test]
1209 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1210 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1211
1212 let buffer = MultiBuffer::build_simple(input, cx);
1213 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1214 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
1215 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
1216 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1217
1218 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1219 let mut cursor = TabStopCursor::new(chunks);
1220
1221 let mut actual_tab_stops = Vec::new();
1222
1223 let mut expected_tab_stops = Vec::new();
1224 let mut byte_offset = 0;
1225 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1226 byte_offset += ch.len_utf8() as u32;
1227
1228 if ch == '\t' {
1229 expected_tab_stops.push(TabStop {
1230 byte_offset,
1231 char_offset: offset as u32 + 1,
1232 });
1233 }
1234 }
1235
1236 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1237 actual_tab_stops.push(tab_stop);
1238 }
1239 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1240
1241 assert_eq!(cursor.byte_offset(), byte_offset);
1242 }
1243
1244 #[gpui::test(iterations = 100)]
1245 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1246 // Generate random input string with up to 512 characters including tabs
1247 let len = rng.random_range(0..=2048);
1248 let mut input = String::with_capacity(len);
1249
1250 let mut skip_tabs = rng.random_bool(0.10);
1251 for idx in 0..len {
1252 if idx % 128 == 0 {
1253 skip_tabs = rng.random_bool(0.10);
1254 }
1255
1256 if rng.random_bool(0.15) && !skip_tabs {
1257 input.push('\t');
1258 } else {
1259 let ch = loop {
1260 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1261 let ch = ascii_code as u8 as char;
1262 if ch != '\t' {
1263 break ch;
1264 }
1265 };
1266 input.push(ch);
1267 }
1268 }
1269
1270 // Build the buffer and create cursor
1271 let buffer = MultiBuffer::build_simple(&input, cx);
1272 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1273 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot.clone());
1274 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
1275 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1276
1277 // First, collect all expected tab positions
1278 let mut all_tab_stops = Vec::new();
1279 let mut byte_offset = 1;
1280 let mut char_offset = 1;
1281 for ch in buffer_snapshot.text().chars() {
1282 if ch == '\t' {
1283 all_tab_stops.push(TabStop {
1284 byte_offset,
1285 char_offset,
1286 });
1287 }
1288 byte_offset += ch.len_utf8() as u32;
1289 char_offset += 1;
1290 }
1291
1292 // Test with various distances
1293 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1294 // let distances = vec![150];
1295
1296 for distance in distances {
1297 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1298 let mut cursor = TabStopCursor::new(chunks);
1299
1300 let mut found_tab_stops = Vec::new();
1301 let mut position = distance;
1302 while let Some(tab_stop) = cursor.seek(position) {
1303 found_tab_stops.push(tab_stop);
1304 position = distance - tab_stop.byte_offset;
1305 }
1306
1307 let expected_found_tab_stops: Vec<_> = all_tab_stops
1308 .iter()
1309 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1310 .cloned()
1311 .collect();
1312
1313 pretty_assertions::assert_eq!(
1314 found_tab_stops,
1315 expected_found_tab_stops,
1316 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1317 distance,
1318 input
1319 );
1320
1321 let final_position = cursor.byte_offset();
1322 if !found_tab_stops.is_empty() {
1323 let last_tab_stop = found_tab_stops.last().unwrap();
1324 assert!(
1325 final_position >= last_tab_stop.byte_offset,
1326 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1327 final_position,
1328 last_tab_stop.byte_offset,
1329 input
1330 );
1331 }
1332 }
1333 }
1334
1335 #[gpui::test]
1336 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1337 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1338 let buffer = MultiBuffer::build_simple(text, cx);
1339 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1340 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot);
1341 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
1342 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1343 let chunks = fold_snapshot.chunks(
1344 FoldOffset(0)..fold_snapshot.len(),
1345 false,
1346 Default::default(),
1347 );
1348 let mut cursor = TabStopCursor::new(chunks);
1349 assert!(cursor.seek(0).is_none());
1350
1351 let mut expected_tab_stops = Vec::new();
1352 let mut byte_offset = 0;
1353 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1354 byte_offset += ch.len_utf8() as u32;
1355
1356 if ch == '\t' {
1357 expected_tab_stops.push(TabStop {
1358 byte_offset,
1359 char_offset: i as u32 + 1,
1360 });
1361 }
1362 }
1363
1364 let mut actual_tab_stops = Vec::new();
1365 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1366 actual_tab_stops.push(tab_stop);
1367 }
1368
1369 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1370
1371 assert_eq!(cursor.byte_offset(), byte_offset);
1372 }
1373
1374 #[gpui::test(iterations = 100)]
1375 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1376 // Generate random input string with up to 512 characters including tabs
1377 let len = rng.random_range(0..=2048);
1378 let input = util::RandomCharIter::new(&mut rng)
1379 .take(len)
1380 .collect::<String>();
1381
1382 // Build the buffer and create cursor
1383 let buffer = MultiBuffer::build_simple(&input, cx);
1384 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1385 let (_, filter_snapshot) = FilterMap::new(None, buffer_snapshot.clone());
1386 let (_, inlay_snapshot) = InlayMap::new(filter_snapshot);
1387 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1388
1389 // First, collect all expected tab positions
1390 let mut all_tab_stops = Vec::new();
1391 let mut byte_offset = 0;
1392 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1393 byte_offset += ch.len_utf8() as u32;
1394 if ch == '\t' {
1395 all_tab_stops.push(TabStop {
1396 byte_offset,
1397 char_offset: i as u32 + 1,
1398 });
1399 }
1400 }
1401
1402 // Test with various distances
1403 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1404 let distances = vec![150];
1405
1406 for distance in distances {
1407 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1408 let mut cursor = TabStopCursor::new(chunks);
1409
1410 let mut found_tab_stops = Vec::new();
1411 let mut position = distance;
1412 while let Some(tab_stop) = cursor.seek(position) {
1413 found_tab_stops.push(tab_stop);
1414 position = distance - tab_stop.byte_offset;
1415 }
1416
1417 let expected_found_tab_stops: Vec<_> = all_tab_stops
1418 .iter()
1419 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1420 .cloned()
1421 .collect();
1422
1423 pretty_assertions::assert_eq!(
1424 found_tab_stops,
1425 expected_found_tab_stops,
1426 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1427 distance,
1428 input
1429 );
1430
1431 let final_position = cursor.byte_offset();
1432 if !found_tab_stops.is_empty() {
1433 let last_tab_stop = found_tab_stops.last().unwrap();
1434 assert!(
1435 final_position >= last_tab_stop.byte_offset,
1436 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1437 final_position,
1438 last_tab_stop.byte_offset,
1439 input
1440 );
1441 }
1442 }
1443 }
1444}
1445
1446struct TabStopCursor<'a, I>
1447where
1448 I: Iterator<Item = Chunk<'a>>,
1449{
1450 chunks: I,
1451 byte_offset: u32,
1452 char_offset: u32,
1453 /// Chunk
1454 /// last tab position iterated through
1455 current_chunk: Option<(Chunk<'a>, u32)>,
1456}
1457
1458impl<'a, I> TabStopCursor<'a, I>
1459where
1460 I: Iterator<Item = Chunk<'a>>,
1461{
1462 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1463 Self {
1464 chunks: chunks.into_iter(),
1465 byte_offset: 0,
1466 char_offset: 0,
1467 current_chunk: None,
1468 }
1469 }
1470
1471 fn bytes_until_next_char(&self) -> Option<usize> {
1472 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1473 let mut idx = *idx;
1474 let mut diff = 0;
1475 while idx > 0 && chunk.chars & (1 << idx) == 0 {
1476 idx -= 1;
1477 diff += 1;
1478 }
1479
1480 if chunk.chars & (1 << idx) != 0 {
1481 Some(
1482 (chunk.text[idx as usize..].chars().next()?)
1483 .len_utf8()
1484 .saturating_sub(diff),
1485 )
1486 } else {
1487 None
1488 }
1489 })
1490 }
1491
1492 fn is_char_boundary(&self) -> bool {
1493 self.current_chunk
1494 .as_ref()
1495 .is_some_and(|(chunk, idx)| (chunk.chars & (1 << *idx.min(&127))) != 0)
1496 }
1497
1498 /// distance: length to move forward while searching for the next tab stop
1499 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1500 if distance == 0 {
1501 return None;
1502 }
1503
1504 let mut distance_traversed = 0;
1505
1506 while let Some((mut chunk, chunk_position)) = self
1507 .current_chunk
1508 .take()
1509 .or_else(|| self.chunks.next().zip(Some(0)))
1510 {
1511 if chunk.tabs == 0 {
1512 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1513 if chunk_distance + distance_traversed >= distance {
1514 let overshoot = distance_traversed.abs_diff(distance);
1515
1516 self.byte_offset += overshoot;
1517 self.char_offset += get_char_offset(
1518 chunk_position..(chunk_position + overshoot).saturating_sub(1).min(127),
1519 chunk.chars,
1520 );
1521
1522 self.current_chunk = Some((chunk, chunk_position + overshoot));
1523
1524 return None;
1525 }
1526
1527 self.byte_offset += chunk_distance;
1528 self.char_offset += get_char_offset(
1529 chunk_position..(chunk_position + chunk_distance).saturating_sub(1).min(127),
1530 chunk.chars,
1531 );
1532 distance_traversed += chunk_distance;
1533 continue;
1534 }
1535 let tab_position = chunk.tabs.trailing_zeros() + 1;
1536
1537 if distance_traversed + tab_position - chunk_position > distance {
1538 let cursor_position = distance_traversed.abs_diff(distance);
1539
1540 self.char_offset += get_char_offset(
1541 chunk_position..(chunk_position + cursor_position - 1),
1542 chunk.chars,
1543 );
1544 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1545 self.byte_offset += cursor_position;
1546
1547 return None;
1548 }
1549
1550 self.byte_offset += tab_position - chunk_position;
1551 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1552
1553 let tabstop = TabStop {
1554 char_offset: self.char_offset,
1555 byte_offset: self.byte_offset,
1556 };
1557
1558 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1559
1560 if tab_position as usize != chunk.text.len() {
1561 self.current_chunk = Some((chunk, tab_position));
1562 }
1563
1564 return Some(tabstop);
1565 }
1566
1567 None
1568 }
1569
1570 fn byte_offset(&self) -> u32 {
1571 self.byte_offset
1572 }
1573
1574 fn char_offset(&self) -> u32 {
1575 self.char_offset
1576 }
1577}
1578
1579#[inline(always)]
1580fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1581 // This edge case can happen when we're at chunk position 128
1582
1583 if range.start == range.end {
1584 return if (1u128 << range.start) & bit_map == 0 {
1585 0
1586 } else {
1587 1
1588 };
1589 }
1590 let end_shift: u128 = 127u128 - range.end.min(127) as u128;
1591 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1592 bit_mask = (bit_mask << end_shift) >> end_shift;
1593 let bit_map = bit_map & bit_mask;
1594
1595 bit_map.count_ones()
1596}
1597
1598#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1599struct TabStop {
1600 char_offset: u32,
1601 byte_offset: u32,
1602}