1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
24 let snapshot = TabSnapshot {
25 fold_snapshot,
26 tab_size: tab_size.min(MAX_TABS),
27 max_expansion_column: MAX_EXPANSION_COLUMN,
28 version: 0,
29 };
30 (Self(snapshot.clone()), snapshot)
31 }
32
33 #[cfg(test)]
34 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
35 self.0.max_expansion_column = column;
36 self.0.clone()
37 }
38
39 pub fn sync(
40 &mut self,
41 fold_snapshot: FoldSnapshot,
42 mut fold_edits: Vec<FoldEdit>,
43 tab_size: NonZeroU32,
44 ) -> (TabSnapshot, Vec<TabEdit>) {
45 let old_snapshot = &mut self.0;
46 let mut new_snapshot = TabSnapshot {
47 fold_snapshot,
48 tab_size: tab_size.min(MAX_TABS),
49 max_expansion_column: old_snapshot.max_expansion_column,
50 version: old_snapshot.version,
51 };
52
53 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
54 new_snapshot.version += 1;
55 }
56
57 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
58 // Expand each edit to include the next tab on the same line as the edit,
59 // and any subsequent tabs on that line that moved across the tab expansion
60 // boundary.
61 for fold_edit in &mut fold_edits {
62 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
63 let old_end_row_successor_offset = cmp::min(
64 FoldPoint::new(old_end.row() + 1, 0),
65 old_snapshot.fold_snapshot.max_point(),
66 )
67 .to_offset(&old_snapshot.fold_snapshot);
68 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
69
70 let mut offset_from_edit = 0;
71 let mut first_tab_offset = None;
72 let mut last_tab_with_changed_expansion_offset = None;
73 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
74 fold_edit.old.end..old_end_row_successor_offset,
75 false,
76 Highlights::default(),
77 ) {
78 // todo(performance use tabs bitmask)
79 for (ix, _) in chunk.text.match_indices('\t') {
80 let offset_from_edit = offset_from_edit + (ix as u32);
81 if first_tab_offset.is_none() {
82 first_tab_offset = Some(offset_from_edit);
83 }
84
85 let old_column = old_end.column() + offset_from_edit;
86 let new_column = new_end.column() + offset_from_edit;
87 let was_expanded = old_column < old_snapshot.max_expansion_column;
88 let is_expanded = new_column < new_snapshot.max_expansion_column;
89 if was_expanded != is_expanded {
90 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
91 } else if !was_expanded && !is_expanded {
92 break 'outer;
93 }
94 }
95
96 offset_from_edit += chunk.text.len() as u32;
97 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
98 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
99 {
100 break;
101 }
102 }
103
104 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
105 fold_edit.old.end.0 += offset as usize + 1;
106 fold_edit.new.end.0 += offset as usize + 1;
107 }
108 }
109
110 let _old_alloc_ptr = fold_edits.as_ptr();
111 // Combine any edits that overlap due to the expansion.
112 let mut fold_edits = fold_edits.into_iter();
113 if let Some(mut first_edit) = fold_edits.next() {
114 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
115 #[allow(clippy::filter_map_identity)]
116 let mut v: Vec<_> = fold_edits
117 .scan(&mut first_edit, |state, edit| {
118 if state.old.end >= edit.old.start {
119 state.old.end = edit.old.end;
120 state.new.end = edit.new.end;
121 Some(None) // Skip this edit, it's merged
122 } else {
123 let new_state = edit;
124 let result = Some(Some(state.clone())); // Yield the previous edit
125 **state = new_state;
126 result
127 }
128 })
129 .filter_map(|x| x)
130 .collect();
131 v.push(first_edit);
132 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
133 v.into_iter()
134 .map(|fold_edit| {
135 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
136 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
137 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
138 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
139 TabEdit {
140 old: old_snapshot.to_tab_point(old_start)
141 ..old_snapshot.to_tab_point(old_end),
142 new: new_snapshot.to_tab_point(new_start)
143 ..new_snapshot.to_tab_point(new_end),
144 }
145 })
146 .collect()
147 } else {
148 vec![]
149 }
150 } else {
151 new_snapshot.version += 1;
152 vec![TabEdit {
153 old: TabPoint::zero()..old_snapshot.max_point(),
154 new: TabPoint::zero()..new_snapshot.max_point(),
155 }]
156 };
157 *old_snapshot = new_snapshot;
158 (old_snapshot.clone(), tab_edits)
159 }
160}
161
162#[derive(Clone)]
163pub struct TabSnapshot {
164 pub fold_snapshot: FoldSnapshot,
165 pub tab_size: NonZeroU32,
166 pub max_expansion_column: u32,
167 pub version: usize,
168}
169
170impl std::ops::Deref for TabSnapshot {
171 type Target = FoldSnapshot;
172
173 fn deref(&self) -> &Self::Target {
174 &self.fold_snapshot
175 }
176}
177
178impl TabSnapshot {
179 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
180 &self.fold_snapshot.inlay_snapshot.buffer
181 }
182
183 pub fn line_len(&self, row: u32) -> u32 {
184 let max_point = self.max_point();
185 if row < max_point.row() {
186 self.to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
187 .0
188 .column
189 } else {
190 max_point.column()
191 }
192 }
193
194 pub fn text_summary(&self) -> TextSummary {
195 self.text_summary_for_range(TabPoint::zero()..self.max_point())
196 }
197
198 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
199 let input_start = self.to_fold_point(range.start, Bias::Left).0;
200 let input_end = self.to_fold_point(range.end, Bias::Right).0;
201 let input_summary = self
202 .fold_snapshot
203 .text_summary_for_range(input_start..input_end);
204
205 let line_end = if range.start.row() == range.end.row() {
206 range.end
207 } else {
208 self.max_point()
209 };
210 let first_line_chars = self
211 .chunks(range.start..line_end, false, Highlights::default())
212 .flat_map(|chunk| chunk.text.chars())
213 .take_while(|&c| c != '\n')
214 .count() as u32;
215
216 let last_line_chars = if range.start.row() == range.end.row() {
217 first_line_chars
218 } else {
219 self.chunks(
220 TabPoint::new(range.end.row(), 0)..range.end,
221 false,
222 Highlights::default(),
223 )
224 .flat_map(|chunk| chunk.text.chars())
225 .count() as u32
226 };
227
228 TextSummary {
229 lines: range.end.0 - range.start.0,
230 first_line_chars,
231 last_line_chars,
232 longest_row: input_summary.longest_row,
233 longest_row_chars: input_summary.longest_row_chars,
234 }
235 }
236
237 pub(crate) fn chunks<'a>(
238 &'a self,
239 range: Range<TabPoint>,
240 language_aware: bool,
241 highlights: Highlights<'a>,
242 ) -> TabChunks<'a> {
243 let (input_start, expanded_char_column, to_next_stop) =
244 self.to_fold_point(range.start, Bias::Left);
245 let input_column = input_start.column();
246 let input_start = input_start.to_offset(&self.fold_snapshot);
247 let input_end = self
248 .to_fold_point(range.end, Bias::Right)
249 .0
250 .to_offset(&self.fold_snapshot);
251 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
252 range.end.column() - range.start.column()
253 } else {
254 to_next_stop
255 };
256
257 TabChunks {
258 snapshot: self,
259 fold_chunks: self.fold_snapshot.chunks(
260 input_start..input_end,
261 language_aware,
262 highlights,
263 ),
264 input_column,
265 column: expanded_char_column,
266 max_expansion_column: self.max_expansion_column,
267 output_position: range.start.0,
268 max_output_position: range.end.0,
269 tab_size: self.tab_size,
270 chunk: Chunk {
271 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
272 is_tab: true,
273 ..Default::default()
274 },
275 inside_leading_tab: to_next_stop > 0,
276 }
277 }
278
279 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
280 self.fold_snapshot.row_infos(row)
281 }
282
283 #[cfg(test)]
284 pub fn text(&self) -> String {
285 self.chunks(
286 TabPoint::zero()..self.max_point(),
287 false,
288 Highlights::default(),
289 )
290 .map(|chunk| chunk.text)
291 .collect()
292 }
293
294 pub fn max_point(&self) -> TabPoint {
295 self.to_tab_point(self.fold_snapshot.max_point())
296 }
297
298 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
299 self.to_tab_point(
300 self.fold_snapshot
301 .clip_point(self.to_fold_point(point, bias).0, bias),
302 )
303 }
304
305 pub fn to_tab_point(&self, input: FoldPoint) -> TabPoint {
306 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
307 let tab_cursor = TabStopCursor::new(chunks);
308 let expanded = self.expand_tabs(tab_cursor, input.column());
309 TabPoint::new(input.row(), expanded)
310 }
311
312 pub fn to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
313 let chunks = self
314 .fold_snapshot
315 .chunks_at(FoldPoint::new(output.row(), 0));
316
317 let tab_cursor = TabStopCursor::new(chunks);
318 let expanded = output.column();
319 let (collapsed, expanded_char_column, to_next_stop) =
320 self.collapse_tabs(tab_cursor, expanded, bias);
321
322 (
323 FoldPoint::new(output.row(), collapsed),
324 expanded_char_column,
325 to_next_stop,
326 )
327 }
328
329 pub fn make_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
330 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
331 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
332 self.to_tab_point(fold_point)
333 }
334
335 pub fn to_point(&self, point: TabPoint, bias: Bias) -> Point {
336 let fold_point = self.to_fold_point(point, bias).0;
337 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
338 self.fold_snapshot
339 .inlay_snapshot
340 .to_buffer_point(inlay_point)
341 }
342
343 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
344 where
345 I: Iterator<Item = Chunk<'a>>,
346 {
347 let tab_size = self.tab_size.get();
348
349 let end_column = column.min(self.max_expansion_column);
350 let mut seek_target = end_column;
351 let mut tab_count = 0;
352 let mut expanded_tab_len = 0;
353
354 while let Some(tab_stop) = cursor.seek(seek_target) {
355 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
356 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
357 tab_count += 1;
358 expanded_tab_len += tab_len;
359
360 seek_target = end_column - cursor.byte_offset;
361 }
362
363 let left_over_char_bytes = if !cursor.is_char_boundary() {
364 cursor.bytes_until_next_char().unwrap_or(0) as u32
365 } else {
366 0
367 };
368
369 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
370 let expanded_bytes =
371 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
372
373 expanded_bytes + column.saturating_sub(collapsed_bytes)
374 }
375
376 fn collapse_tabs<'a, I>(
377 &self,
378 mut cursor: TabStopCursor<'a, I>,
379 column: u32,
380 bias: Bias,
381 ) -> (u32, u32, u32)
382 where
383 I: Iterator<Item = Chunk<'a>>,
384 {
385 let tab_size = self.tab_size.get();
386 let mut collapsed_column = column;
387 let mut seek_target = column.min(self.max_expansion_column);
388 let mut tab_count = 0;
389 let mut expanded_tab_len = 0;
390
391 while let Some(tab_stop) = cursor.seek(seek_target) {
392 // Calculate how much we want to expand this tab stop (into spaces)
393 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
394 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
395 // Increment tab count
396 tab_count += 1;
397 // The count of how many spaces we've added to this line in place of tab bytes
398 expanded_tab_len += tab_len;
399
400 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
401 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
402
403 // Did we expand past the search target?
404 if expanded_bytes > column {
405 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
406 // We expanded past the search target, so need to account for the offshoot
407 expanded_chars -= expanded_bytes - column;
408 return match bias {
409 Bias::Left => (
410 cursor.byte_offset() - 1,
411 expanded_chars,
412 expanded_bytes - column,
413 ),
414 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
415 };
416 } else {
417 // otherwise we only want to move the cursor collapse column forward
418 collapsed_column = collapsed_column - tab_len + 1;
419 seek_target = (collapsed_column - cursor.byte_offset)
420 .min(self.max_expansion_column - cursor.byte_offset);
421 }
422 }
423
424 let collapsed_bytes = cursor.byte_offset();
425 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
426 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
427 (
428 collapsed_bytes + column.saturating_sub(expanded_bytes),
429 expanded_chars,
430 0,
431 )
432 }
433}
434
435#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
436pub struct TabPoint(pub Point);
437
438impl TabPoint {
439 pub fn new(row: u32, column: u32) -> Self {
440 Self(Point::new(row, column))
441 }
442
443 pub fn zero() -> Self {
444 Self::new(0, 0)
445 }
446
447 pub fn row(self) -> u32 {
448 self.0.row
449 }
450
451 pub fn column(self) -> u32 {
452 self.0.column
453 }
454}
455
456impl From<Point> for TabPoint {
457 fn from(point: Point) -> Self {
458 Self(point)
459 }
460}
461
462pub type TabEdit = text::Edit<TabPoint>;
463
464#[derive(Clone, Debug, Default, Eq, PartialEq)]
465pub struct TextSummary {
466 pub lines: Point,
467 pub first_line_chars: u32,
468 pub last_line_chars: u32,
469 pub longest_row: u32,
470 pub longest_row_chars: u32,
471}
472
473impl<'a> From<&'a str> for TextSummary {
474 fn from(text: &'a str) -> Self {
475 let sum = text::TextSummary::from(text);
476
477 TextSummary {
478 lines: sum.lines,
479 first_line_chars: sum.first_line_chars,
480 last_line_chars: sum.last_line_chars,
481 longest_row: sum.longest_row,
482 longest_row_chars: sum.longest_row_chars,
483 }
484 }
485}
486
487impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
488 fn add_assign(&mut self, other: &'a Self) {
489 let joined_chars = self.last_line_chars + other.first_line_chars;
490 if joined_chars > self.longest_row_chars {
491 self.longest_row = self.lines.row;
492 self.longest_row_chars = joined_chars;
493 }
494 if other.longest_row_chars > self.longest_row_chars {
495 self.longest_row = self.lines.row + other.longest_row;
496 self.longest_row_chars = other.longest_row_chars;
497 }
498
499 if self.lines.row == 0 {
500 self.first_line_chars += other.first_line_chars;
501 }
502
503 if other.lines.row == 0 {
504 self.last_line_chars += other.first_line_chars;
505 } else {
506 self.last_line_chars = other.last_line_chars;
507 }
508
509 self.lines += &other.lines;
510 }
511}
512
513pub struct TabChunks<'a> {
514 snapshot: &'a TabSnapshot,
515 max_expansion_column: u32,
516 max_output_position: Point,
517 tab_size: NonZeroU32,
518 // region: iteration state
519 fold_chunks: FoldChunks<'a>,
520 chunk: Chunk<'a>,
521 column: u32,
522 output_position: Point,
523 input_column: u32,
524 inside_leading_tab: bool,
525 // endregion: iteration state
526}
527
528impl TabChunks<'_> {
529 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
530 let (input_start, expanded_char_column, to_next_stop) =
531 self.snapshot.to_fold_point(range.start, Bias::Left);
532 let input_column = input_start.column();
533 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
534 let input_end = self
535 .snapshot
536 .to_fold_point(range.end, Bias::Right)
537 .0
538 .to_offset(&self.snapshot.fold_snapshot);
539 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
540 range.end.column() - range.start.column()
541 } else {
542 to_next_stop
543 };
544
545 self.fold_chunks.seek(input_start..input_end);
546 self.input_column = input_column;
547 self.column = expanded_char_column;
548 self.output_position = range.start.0;
549 self.max_output_position = range.end.0;
550 self.chunk = Chunk {
551 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
552 is_tab: true,
553 chars: 1u128.unbounded_shl(to_next_stop) - 1,
554 ..Default::default()
555 };
556 self.inside_leading_tab = to_next_stop > 0;
557 }
558}
559
560impl<'a> Iterator for TabChunks<'a> {
561 type Item = Chunk<'a>;
562
563 fn next(&mut self) -> Option<Self::Item> {
564 if self.chunk.text.is_empty() {
565 if let Some(chunk) = self.fold_chunks.next() {
566 self.chunk = chunk;
567 if self.inside_leading_tab {
568 self.chunk.text = &self.chunk.text[1..];
569 self.inside_leading_tab = false;
570 self.input_column += 1;
571 }
572 } else {
573 return None;
574 }
575 }
576
577 //todo(improve performance by using tab cursor)
578 for (ix, c) in self.chunk.text.char_indices() {
579 match c {
580 '\t' if ix > 0 => {
581 let (prefix, suffix) = self.chunk.text.split_at(ix);
582
583 let mask = 1u128.unbounded_shl(ix as u32).wrapping_sub(1);
584 let chars = self.chunk.chars & mask;
585 let tabs = self.chunk.tabs & mask;
586 self.chunk.tabs = self.chunk.tabs.unbounded_shr(ix as u32);
587 self.chunk.chars = self.chunk.chars.unbounded_shr(ix as u32);
588 self.chunk.text = suffix;
589 return Some(Chunk {
590 text: prefix,
591 chars,
592 tabs,
593 ..self.chunk.clone()
594 });
595 }
596 '\t' => {
597 self.chunk.text = &self.chunk.text[1..];
598 self.chunk.tabs >>= 1;
599 self.chunk.chars >>= 1;
600 let tab_size = if self.input_column < self.max_expansion_column {
601 self.tab_size.get()
602 } else {
603 1
604 };
605 let mut len = tab_size - self.column % tab_size;
606 let next_output_position = cmp::min(
607 self.output_position + Point::new(0, len),
608 self.max_output_position,
609 );
610 len = next_output_position.column - self.output_position.column;
611 self.column += len;
612 self.input_column += 1;
613 self.output_position = next_output_position;
614 return Some(Chunk {
615 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
616 is_tab: true,
617 chars: 1u128.unbounded_shl(len) - 1,
618 tabs: 0,
619 ..self.chunk.clone()
620 });
621 }
622 '\n' => {
623 self.column = 0;
624 self.input_column = 0;
625 self.output_position += Point::new(1, 0);
626 }
627 _ => {
628 self.column += 1;
629 if !self.inside_leading_tab {
630 self.input_column += c.len_utf8() as u32;
631 }
632 self.output_position.column += c.len_utf8() as u32;
633 }
634 }
635 }
636
637 Some(mem::take(&mut self.chunk))
638 }
639}
640
641#[cfg(test)]
642mod tests {
643 use super::*;
644 use crate::{
645 MultiBuffer,
646 display_map::{
647 fold_map::{FoldMap, FoldOffset},
648 inlay_map::InlayMap,
649 },
650 };
651 use rand::{Rng, prelude::StdRng};
652 use util;
653
654 impl TabSnapshot {
655 fn expected_collapse_tabs(
656 &self,
657 chars: impl Iterator<Item = char>,
658 column: u32,
659 bias: Bias,
660 ) -> (u32, u32, u32) {
661 let tab_size = self.tab_size.get();
662
663 let mut expanded_bytes = 0;
664 let mut expanded_chars = 0;
665 let mut collapsed_bytes = 0;
666 for c in chars {
667 if expanded_bytes >= column {
668 break;
669 }
670 if collapsed_bytes >= self.max_expansion_column {
671 break;
672 }
673
674 if c == '\t' {
675 let tab_len = tab_size - (expanded_chars % tab_size);
676 expanded_chars += tab_len;
677 expanded_bytes += tab_len;
678 if expanded_bytes > column {
679 expanded_chars -= expanded_bytes - column;
680 return match bias {
681 Bias::Left => {
682 (collapsed_bytes, expanded_chars, expanded_bytes - column)
683 }
684 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
685 };
686 }
687 } else {
688 expanded_chars += 1;
689 expanded_bytes += c.len_utf8() as u32;
690 }
691
692 if expanded_bytes > column && matches!(bias, Bias::Left) {
693 expanded_chars -= 1;
694 break;
695 }
696
697 collapsed_bytes += c.len_utf8() as u32;
698 }
699
700 (
701 collapsed_bytes + column.saturating_sub(expanded_bytes),
702 expanded_chars,
703 0,
704 )
705 }
706
707 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
708 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
709 let expanded = self.expected_expand_tabs(chars, input.column());
710 TabPoint::new(input.row(), expanded)
711 }
712
713 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
714 let tab_size = self.tab_size.get();
715
716 let mut expanded_chars = 0;
717 let mut expanded_bytes = 0;
718 let mut collapsed_bytes = 0;
719 let end_column = column.min(self.max_expansion_column);
720 for c in chars {
721 if collapsed_bytes >= end_column {
722 break;
723 }
724 if c == '\t' {
725 let tab_len = tab_size - expanded_chars % tab_size;
726 expanded_bytes += tab_len;
727 expanded_chars += tab_len;
728 } else {
729 expanded_bytes += c.len_utf8() as u32;
730 expanded_chars += 1;
731 }
732 collapsed_bytes += c.len_utf8() as u32;
733 }
734
735 expanded_bytes + column.saturating_sub(collapsed_bytes)
736 }
737
738 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
739 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
740 let expanded = output.column();
741 let (collapsed, expanded_char_column, to_next_stop) =
742 self.expected_collapse_tabs(chars, expanded, bias);
743 (
744 FoldPoint::new(output.row(), collapsed),
745 expanded_char_column,
746 to_next_stop,
747 )
748 }
749 }
750
751 #[gpui::test]
752 fn test_expand_tabs(cx: &mut gpui::App) {
753 let test_values = [
754 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
755 (" \twςe", 4),
756 ("fε", 1),
757 ("i❎\t", 3),
758 ];
759 let buffer = MultiBuffer::build_simple("", cx);
760 let buffer_snapshot = buffer.read(cx).snapshot(cx);
761 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
762 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
763 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
764
765 for (text, column) in test_values {
766 let mut tabs = 0u128;
767 let mut chars = 0u128;
768 for (idx, c) in text.char_indices() {
769 if c == '\t' {
770 tabs |= 1 << idx;
771 }
772 chars |= 1 << idx;
773 }
774
775 let chunks = [Chunk {
776 text,
777 tabs,
778 chars,
779 ..Default::default()
780 }];
781
782 let cursor = TabStopCursor::new(chunks);
783
784 assert_eq!(
785 tab_snapshot.expected_expand_tabs(text.chars(), column),
786 tab_snapshot.expand_tabs(cursor, column)
787 );
788 }
789 }
790
791 #[gpui::test]
792 fn test_collapse_tabs(cx: &mut gpui::App) {
793 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
794
795 let buffer = MultiBuffer::build_simple(input, cx);
796 let buffer_snapshot = buffer.read(cx).snapshot(cx);
797 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
798 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
799 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
800
801 for (ix, _) in input.char_indices() {
802 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
803
804 assert_eq!(
805 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
806 tab_snapshot.to_fold_point(range.start, Bias::Left),
807 "Failed with tab_point at column {ix}"
808 );
809 assert_eq!(
810 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
811 tab_snapshot.to_fold_point(range.start, Bias::Right),
812 "Failed with tab_point at column {ix}"
813 );
814
815 assert_eq!(
816 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
817 tab_snapshot.to_fold_point(range.end, Bias::Left),
818 "Failed with tab_point at column {ix}"
819 );
820 assert_eq!(
821 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
822 tab_snapshot.to_fold_point(range.end, Bias::Right),
823 "Failed with tab_point at column {ix}"
824 );
825 }
826 }
827
828 #[gpui::test]
829 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
830 // This test reproduces a specific panic where to_fold_point returns incorrect results
831 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
832
833 let text = "γ\tw⭐\n🍐🍗 \t";
834 let buffer = MultiBuffer::build_simple(text, cx);
835 let buffer_snapshot = buffer.read(cx).snapshot(cx);
836 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
837 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
838 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
839
840 // This should panic with the expected vs actual mismatch
841 let tab_point = TabPoint::new(0, 9);
842 let result = tab_snapshot.to_fold_point(tab_point, Bias::Left);
843 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
844
845 assert_eq!(result, expected);
846 }
847
848 #[gpui::test(iterations = 100)]
849 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
850 // Generate random input string with up to 200 characters including tabs
851 // to stay within the MAX_EXPANSION_COLUMN limit of 256
852 let len = rng.random_range(0..=2048);
853 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
854 let mut input = String::with_capacity(len);
855
856 for _ in 0..len {
857 if rng.random_bool(0.1) {
858 // 10% chance of inserting a tab
859 input.push('\t');
860 } else {
861 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
862 let ch = loop {
863 let ascii_code = rng.random_range(32..=126); // printable ASCII range
864 let ch = ascii_code as u8 as char;
865 if ch != '\t' {
866 break ch;
867 }
868 };
869 input.push(ch);
870 }
871 }
872
873 let buffer = MultiBuffer::build_simple(&input, cx);
874 let buffer_snapshot = buffer.read(cx).snapshot(cx);
875 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
876 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
877 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
878 tab_snapshot.max_expansion_column = rng.random_range(0..323);
879 tab_snapshot.tab_size = tab_size;
880
881 for (ix, _) in input.char_indices() {
882 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
883
884 assert_eq!(
885 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
886 tab_snapshot.to_fold_point(range.start, Bias::Left),
887 "Failed with input: {}, with idx: {ix}",
888 input
889 );
890 assert_eq!(
891 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
892 tab_snapshot.to_fold_point(range.start, Bias::Right),
893 "Failed with input: {}, with idx: {ix}",
894 input
895 );
896
897 assert_eq!(
898 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
899 tab_snapshot.to_fold_point(range.end, Bias::Left),
900 "Failed with input: {}, with idx: {ix}",
901 input
902 );
903 assert_eq!(
904 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
905 tab_snapshot.to_fold_point(range.end, Bias::Right),
906 "Failed with input: {}, with idx: {ix}",
907 input
908 );
909 }
910 }
911
912 #[gpui::test]
913 fn test_long_lines(cx: &mut gpui::App) {
914 let max_expansion_column = 12;
915 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
916 let output = "A BC DEF G HI J K L M";
917
918 let buffer = MultiBuffer::build_simple(input, cx);
919 let buffer_snapshot = buffer.read(cx).snapshot(cx);
920 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
921 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
922 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
923
924 tab_snapshot.max_expansion_column = max_expansion_column;
925 assert_eq!(tab_snapshot.text(), output);
926
927 for (ix, c) in input.char_indices() {
928 assert_eq!(
929 tab_snapshot
930 .chunks(
931 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
932 false,
933 Highlights::default(),
934 )
935 .map(|c| c.text)
936 .collect::<String>(),
937 &output[ix..],
938 "text from index {ix}"
939 );
940
941 if c != '\t' {
942 let input_point = Point::new(0, ix as u32);
943 let output_point = Point::new(0, output.find(c).unwrap() as u32);
944 assert_eq!(
945 tab_snapshot.to_tab_point(FoldPoint(input_point)),
946 TabPoint(output_point),
947 "to_tab_point({input_point:?})"
948 );
949 assert_eq!(
950 tab_snapshot
951 .to_fold_point(TabPoint(output_point), Bias::Left)
952 .0,
953 FoldPoint(input_point),
954 "to_fold_point({output_point:?})"
955 );
956 }
957 }
958 }
959
960 #[gpui::test]
961 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
962 let max_expansion_column = 8;
963 let input = "abcdefg⋯hij";
964
965 let buffer = MultiBuffer::build_simple(input, cx);
966 let buffer_snapshot = buffer.read(cx).snapshot(cx);
967 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
968 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
969 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
970
971 tab_snapshot.max_expansion_column = max_expansion_column;
972 assert_eq!(tab_snapshot.text(), input);
973 }
974
975 #[gpui::test]
976 fn test_marking_tabs(cx: &mut gpui::App) {
977 let input = "\t \thello";
978
979 let buffer = MultiBuffer::build_simple(input, cx);
980 let buffer_snapshot = buffer.read(cx).snapshot(cx);
981 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
982 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
983 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
984
985 assert_eq!(
986 chunks(&tab_snapshot, TabPoint::zero()),
987 vec![
988 (" ".to_string(), true),
989 (" ".to_string(), false),
990 (" ".to_string(), true),
991 ("hello".to_string(), false),
992 ]
993 );
994 assert_eq!(
995 chunks(&tab_snapshot, TabPoint::new(0, 2)),
996 vec![
997 (" ".to_string(), true),
998 (" ".to_string(), false),
999 (" ".to_string(), true),
1000 ("hello".to_string(), false),
1001 ]
1002 );
1003
1004 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1005 let mut chunks = Vec::new();
1006 let mut was_tab = false;
1007 let mut text = String::new();
1008 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1009 {
1010 if chunk.is_tab != was_tab {
1011 if !text.is_empty() {
1012 chunks.push((mem::take(&mut text), was_tab));
1013 }
1014 was_tab = chunk.is_tab;
1015 }
1016 text.push_str(chunk.text);
1017 }
1018
1019 if !text.is_empty() {
1020 chunks.push((text, was_tab));
1021 }
1022 chunks
1023 }
1024 }
1025
1026 #[gpui::test(iterations = 100)]
1027 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1028 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1029 let len = rng.random_range(0..30);
1030 let buffer = if rng.random() {
1031 let text = util::RandomCharIter::new(&mut rng)
1032 .take(len)
1033 .collect::<String>();
1034 MultiBuffer::build_simple(&text, cx)
1035 } else {
1036 MultiBuffer::build_random(&mut rng, cx)
1037 };
1038 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1039 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1040
1041 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1042 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1043 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1044 fold_map.randomly_mutate(&mut rng);
1045 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1046 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1047 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1048 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1049
1050 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1051 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1052
1053 let text = text::Rope::from(tabs_snapshot.text().as_str());
1054 log::info!(
1055 "TabMap text (tab size: {}): {:?}",
1056 tab_size,
1057 tabs_snapshot.text(),
1058 );
1059
1060 for _ in 0..5 {
1061 let end_row = rng.random_range(0..=text.max_point().row);
1062 let end_column = rng.random_range(0..=text.line_len(end_row));
1063 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1064 let start_row = rng.random_range(0..=text.max_point().row);
1065 let start_column = rng.random_range(0..=text.line_len(start_row));
1066 let mut start =
1067 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1068 if start > end {
1069 mem::swap(&mut start, &mut end);
1070 }
1071
1072 let expected_text = text
1073 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1074 .collect::<String>();
1075 let expected_summary = TextSummary::from(expected_text.as_str());
1076 assert_eq!(
1077 tabs_snapshot
1078 .chunks(start..end, false, Highlights::default())
1079 .map(|c| c.text)
1080 .collect::<String>(),
1081 expected_text,
1082 "chunks({:?}..{:?})",
1083 start,
1084 end
1085 );
1086
1087 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1088 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1089 actual_summary.longest_row = expected_summary.longest_row;
1090 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1091 }
1092 assert_eq!(actual_summary, expected_summary);
1093 }
1094
1095 for row in 0..=text.max_point().row {
1096 assert_eq!(
1097 tabs_snapshot.line_len(row),
1098 text.line_len(row),
1099 "line_len({row})"
1100 );
1101 }
1102 }
1103
1104 #[gpui::test(iterations = 100)]
1105 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1106 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1107 let len = rng.random_range(0..=2000);
1108
1109 // Generate random text using RandomCharIter
1110 let text = util::RandomCharIter::new(&mut rng)
1111 .take(len)
1112 .collect::<String>();
1113
1114 // Create buffer and tab map
1115 let buffer = MultiBuffer::build_simple(&text, cx);
1116 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1117 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1118 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1119 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1120
1121 let mut next_inlay_id = 0;
1122 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1123 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1124 let max_fold_point = fold_snapshot.max_point();
1125 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1126
1127 // Test random fold points
1128 for _ in 0..50 {
1129 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1130 // Generate random fold point
1131 let row = rng.random_range(0..=max_fold_point.row());
1132 let max_column = if row < max_fold_point.row() {
1133 fold_snapshot.line_len(row)
1134 } else {
1135 max_fold_point.column()
1136 };
1137 let column = rng.random_range(0..=max_column + 10);
1138 let fold_point = FoldPoint::new(row, column);
1139
1140 let actual = tab_snapshot.to_tab_point(fold_point);
1141 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1142
1143 assert_eq!(
1144 actual, expected,
1145 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1146 fold_point, text
1147 );
1148 }
1149 }
1150
1151 #[gpui::test]
1152 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1153 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1154 let buffer = MultiBuffer::build_simple(text, cx);
1155 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1156 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1157 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1158 let chunks = fold_snapshot.chunks(
1159 FoldOffset(0)..fold_snapshot.len(),
1160 false,
1161 Default::default(),
1162 );
1163 let mut cursor = TabStopCursor::new(chunks);
1164 assert!(cursor.seek(0).is_none());
1165 let mut tab_stops = Vec::new();
1166
1167 let mut all_tab_stops = Vec::new();
1168 let mut byte_offset = 0;
1169 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1170 byte_offset += ch.len_utf8() as u32;
1171
1172 if ch == '\t' {
1173 all_tab_stops.push(TabStop {
1174 byte_offset,
1175 char_offset: offset as u32 + 1,
1176 });
1177 }
1178 }
1179
1180 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1181 tab_stops.push(tab_stop);
1182 }
1183 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1184
1185 assert_eq!(cursor.byte_offset(), byte_offset);
1186 }
1187
1188 #[gpui::test]
1189 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1190 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1191
1192 let buffer = MultiBuffer::build_simple(input, cx);
1193 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1194 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1195 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1196
1197 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1198 let mut cursor = TabStopCursor::new(chunks);
1199
1200 let mut actual_tab_stops = Vec::new();
1201
1202 let mut expected_tab_stops = Vec::new();
1203 let mut byte_offset = 0;
1204 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1205 byte_offset += ch.len_utf8() as u32;
1206
1207 if ch == '\t' {
1208 expected_tab_stops.push(TabStop {
1209 byte_offset,
1210 char_offset: offset as u32 + 1,
1211 });
1212 }
1213 }
1214
1215 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1216 actual_tab_stops.push(tab_stop);
1217 }
1218 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1219
1220 assert_eq!(cursor.byte_offset(), byte_offset);
1221 }
1222
1223 #[gpui::test(iterations = 100)]
1224 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1225 // Generate random input string with up to 512 characters including tabs
1226 let len = rng.random_range(0..=2048);
1227 let mut input = String::with_capacity(len);
1228
1229 let mut skip_tabs = rng.random_bool(0.10);
1230 for idx in 0..len {
1231 if idx % 128 == 0 {
1232 skip_tabs = rng.random_bool(0.10);
1233 }
1234
1235 if rng.random_bool(0.15) && !skip_tabs {
1236 input.push('\t');
1237 } else {
1238 let ch = loop {
1239 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1240 let ch = ascii_code as u8 as char;
1241 if ch != '\t' {
1242 break ch;
1243 }
1244 };
1245 input.push(ch);
1246 }
1247 }
1248
1249 // Build the buffer and create cursor
1250 let buffer = MultiBuffer::build_simple(&input, cx);
1251 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1252 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1253 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1254
1255 // First, collect all expected tab positions
1256 let mut all_tab_stops = Vec::new();
1257 let mut byte_offset = 1;
1258 let mut char_offset = 1;
1259 for ch in buffer_snapshot.text().chars() {
1260 if ch == '\t' {
1261 all_tab_stops.push(TabStop {
1262 byte_offset,
1263 char_offset,
1264 });
1265 }
1266 byte_offset += ch.len_utf8() as u32;
1267 char_offset += 1;
1268 }
1269
1270 // Test with various distances
1271 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1272 // let distances = vec![150];
1273
1274 for distance in distances {
1275 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1276 let mut cursor = TabStopCursor::new(chunks);
1277
1278 let mut found_tab_stops = Vec::new();
1279 let mut position = distance;
1280 while let Some(tab_stop) = cursor.seek(position) {
1281 found_tab_stops.push(tab_stop);
1282 position = distance - tab_stop.byte_offset;
1283 }
1284
1285 let expected_found_tab_stops: Vec<_> = all_tab_stops
1286 .iter()
1287 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1288 .cloned()
1289 .collect();
1290
1291 pretty_assertions::assert_eq!(
1292 found_tab_stops,
1293 expected_found_tab_stops,
1294 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1295 distance,
1296 input
1297 );
1298
1299 let final_position = cursor.byte_offset();
1300 if !found_tab_stops.is_empty() {
1301 let last_tab_stop = found_tab_stops.last().unwrap();
1302 assert!(
1303 final_position >= last_tab_stop.byte_offset,
1304 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1305 final_position,
1306 last_tab_stop.byte_offset,
1307 input
1308 );
1309 }
1310 }
1311 }
1312
1313 #[gpui::test]
1314 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1315 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1316 let buffer = MultiBuffer::build_simple(text, cx);
1317 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1318 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1319 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1320 let chunks = fold_snapshot.chunks(
1321 FoldOffset(0)..fold_snapshot.len(),
1322 false,
1323 Default::default(),
1324 );
1325 let mut cursor = TabStopCursor::new(chunks);
1326 assert!(cursor.seek(0).is_none());
1327
1328 let mut expected_tab_stops = Vec::new();
1329 let mut byte_offset = 0;
1330 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1331 byte_offset += ch.len_utf8() as u32;
1332
1333 if ch == '\t' {
1334 expected_tab_stops.push(TabStop {
1335 byte_offset,
1336 char_offset: i as u32 + 1,
1337 });
1338 }
1339 }
1340
1341 let mut actual_tab_stops = Vec::new();
1342 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1343 actual_tab_stops.push(tab_stop);
1344 }
1345
1346 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1347
1348 assert_eq!(cursor.byte_offset(), byte_offset);
1349 }
1350
1351 #[gpui::test(iterations = 100)]
1352 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1353 // Generate random input string with up to 512 characters including tabs
1354 let len = rng.random_range(0..=2048);
1355 let input = util::RandomCharIter::new(&mut rng)
1356 .take(len)
1357 .collect::<String>();
1358
1359 // Build the buffer and create cursor
1360 let buffer = MultiBuffer::build_simple(&input, cx);
1361 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1362 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1363 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1364
1365 // First, collect all expected tab positions
1366 let mut all_tab_stops = Vec::new();
1367 let mut byte_offset = 0;
1368 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1369 byte_offset += ch.len_utf8() as u32;
1370 if ch == '\t' {
1371 all_tab_stops.push(TabStop {
1372 byte_offset,
1373 char_offset: i as u32 + 1,
1374 });
1375 }
1376 }
1377
1378 // Test with various distances
1379 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1380 let distances = vec![150];
1381
1382 for distance in distances {
1383 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1384 let mut cursor = TabStopCursor::new(chunks);
1385
1386 let mut found_tab_stops = Vec::new();
1387 let mut position = distance;
1388 while let Some(tab_stop) = cursor.seek(position) {
1389 found_tab_stops.push(tab_stop);
1390 position = distance - tab_stop.byte_offset;
1391 }
1392
1393 let expected_found_tab_stops: Vec<_> = all_tab_stops
1394 .iter()
1395 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1396 .cloned()
1397 .collect();
1398
1399 pretty_assertions::assert_eq!(
1400 found_tab_stops,
1401 expected_found_tab_stops,
1402 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1403 distance,
1404 input
1405 );
1406
1407 let final_position = cursor.byte_offset();
1408 if !found_tab_stops.is_empty() {
1409 let last_tab_stop = found_tab_stops.last().unwrap();
1410 assert!(
1411 final_position >= last_tab_stop.byte_offset,
1412 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1413 final_position,
1414 last_tab_stop.byte_offset,
1415 input
1416 );
1417 }
1418 }
1419 }
1420}
1421
1422struct TabStopCursor<'a, I>
1423where
1424 I: Iterator<Item = Chunk<'a>>,
1425{
1426 chunks: I,
1427 byte_offset: u32,
1428 char_offset: u32,
1429 /// Chunk
1430 /// last tab position iterated through
1431 current_chunk: Option<(Chunk<'a>, u32)>,
1432}
1433
1434impl<'a, I> TabStopCursor<'a, I>
1435where
1436 I: Iterator<Item = Chunk<'a>>,
1437{
1438 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1439 Self {
1440 chunks: chunks.into_iter(),
1441 byte_offset: 0,
1442 char_offset: 0,
1443 current_chunk: None,
1444 }
1445 }
1446
1447 fn bytes_until_next_char(&self) -> Option<usize> {
1448 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1449 let mut idx = *idx;
1450 let mut diff = 0;
1451 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1452 idx -= 1;
1453 diff += 1;
1454 }
1455
1456 if chunk.chars & (1 << idx) != 0 {
1457 Some(
1458 (chunk.text[idx as usize..].chars().next()?)
1459 .len_utf8()
1460 .saturating_sub(diff),
1461 )
1462 } else {
1463 None
1464 }
1465 })
1466 }
1467
1468 fn is_char_boundary(&self) -> bool {
1469 self.current_chunk
1470 .as_ref()
1471 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1472 }
1473
1474 /// distance: length to move forward while searching for the next tab stop
1475 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1476 if distance == 0 {
1477 return None;
1478 }
1479
1480 let mut distance_traversed = 0;
1481
1482 while let Some((mut chunk, chunk_position)) = self
1483 .current_chunk
1484 .take()
1485 .or_else(|| self.chunks.next().zip(Some(0)))
1486 {
1487 if chunk.tabs == 0 {
1488 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1489 if chunk_distance + distance_traversed >= distance {
1490 let overshoot = distance_traversed.abs_diff(distance);
1491
1492 self.byte_offset += overshoot;
1493 self.char_offset += get_char_offset(
1494 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1495 chunk.chars,
1496 );
1497
1498 if chunk_position + overshoot < 128 {
1499 self.current_chunk = Some((chunk, chunk_position + overshoot));
1500 }
1501
1502 return None;
1503 }
1504
1505 self.byte_offset += chunk_distance;
1506 self.char_offset += get_char_offset(
1507 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1508 chunk.chars,
1509 );
1510 distance_traversed += chunk_distance;
1511 continue;
1512 }
1513 let tab_position = chunk.tabs.trailing_zeros() + 1;
1514
1515 if distance_traversed + tab_position - chunk_position > distance {
1516 let cursor_position = distance_traversed.abs_diff(distance);
1517
1518 self.char_offset += get_char_offset(
1519 chunk_position..(chunk_position + cursor_position - 1),
1520 chunk.chars,
1521 );
1522 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1523 self.byte_offset += cursor_position;
1524
1525 return None;
1526 }
1527
1528 self.byte_offset += tab_position - chunk_position;
1529 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1530
1531 let tabstop = TabStop {
1532 char_offset: self.char_offset,
1533 byte_offset: self.byte_offset,
1534 };
1535
1536 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1537
1538 if tab_position as usize != chunk.text.len() {
1539 self.current_chunk = Some((chunk, tab_position));
1540 }
1541
1542 return Some(tabstop);
1543 }
1544
1545 None
1546 }
1547
1548 fn byte_offset(&self) -> u32 {
1549 self.byte_offset
1550 }
1551
1552 fn char_offset(&self) -> u32 {
1553 self.char_offset
1554 }
1555}
1556
1557#[inline(always)]
1558fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1559 if range.start == range.end {
1560 return if (1u128 << range.start) & bit_map == 0 {
1561 0
1562 } else {
1563 1
1564 };
1565 }
1566 let end_shift: u128 = 127u128 - range.end as u128;
1567 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1568 bit_mask = (bit_mask << end_shift) >> end_shift;
1569 let bit_map = bit_map & bit_mask;
1570
1571 bit_map.count_ones()
1572}
1573
1574#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1575struct TabStop {
1576 char_offset: u32,
1577 byte_offset: u32,
1578}