1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; u128::BITS as usize] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
24 let snapshot = TabSnapshot {
25 fold_snapshot,
26 tab_size: tab_size.min(MAX_TABS),
27 max_expansion_column: MAX_EXPANSION_COLUMN,
28 version: 0,
29 };
30 (Self(snapshot.clone()), snapshot)
31 }
32
33 #[cfg(test)]
34 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
35 self.0.max_expansion_column = column;
36 self.0.clone()
37 }
38
39 pub fn sync(
40 &mut self,
41 fold_snapshot: FoldSnapshot,
42 mut fold_edits: Vec<FoldEdit>,
43 tab_size: NonZeroU32,
44 ) -> (TabSnapshot, Vec<TabEdit>) {
45 let old_snapshot = &mut self.0;
46 let mut new_snapshot = TabSnapshot {
47 fold_snapshot,
48 tab_size: tab_size.min(MAX_TABS),
49 max_expansion_column: old_snapshot.max_expansion_column,
50 version: old_snapshot.version,
51 };
52
53 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
54 new_snapshot.version += 1;
55 }
56
57 let mut tab_edits = Vec::with_capacity(fold_edits.len());
58
59 if old_snapshot.tab_size == new_snapshot.tab_size {
60 // Expand each edit to include the next tab on the same line as the edit,
61 // and any subsequent tabs on that line that moved across the tab expansion
62 // boundary.
63 for fold_edit in &mut fold_edits {
64 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
65 let old_end_row_successor_offset = cmp::min(
66 FoldPoint::new(old_end.row() + 1, 0),
67 old_snapshot.fold_snapshot.max_point(),
68 )
69 .to_offset(&old_snapshot.fold_snapshot);
70 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
71
72 let mut offset_from_edit = 0;
73 let mut first_tab_offset = None;
74 let mut last_tab_with_changed_expansion_offset = None;
75 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
76 fold_edit.old.end..old_end_row_successor_offset,
77 false,
78 Highlights::default(),
79 ) {
80 // todo(performance use tabs bitmask)
81 for (ix, _) in chunk.text.match_indices('\t') {
82 let offset_from_edit = offset_from_edit + (ix as u32);
83 if first_tab_offset.is_none() {
84 first_tab_offset = Some(offset_from_edit);
85 }
86
87 let old_column = old_end.column() + offset_from_edit;
88 let new_column = new_end.column() + offset_from_edit;
89 let was_expanded = old_column < old_snapshot.max_expansion_column;
90 let is_expanded = new_column < new_snapshot.max_expansion_column;
91 if was_expanded != is_expanded {
92 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
93 } else if !was_expanded && !is_expanded {
94 break 'outer;
95 }
96 }
97
98 offset_from_edit += chunk.text.len() as u32;
99 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
100 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
101 {
102 break;
103 }
104 }
105
106 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
107 fold_edit.old.end.0 += offset as usize + 1;
108 fold_edit.new.end.0 += offset as usize + 1;
109 }
110 }
111
112 let _old_alloc_ptr = fold_edits.as_ptr();
113 // Combine any edits that overlap due to the expansion.
114 let mut fold_edits = fold_edits.into_iter();
115 let fold_edits = if let Some(mut first_edit) = fold_edits.next() {
116 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
117 #[allow(clippy::filter_map_identity)]
118 let mut v: Vec<_> = fold_edits
119 .scan(&mut first_edit, |state, edit| {
120 if state.old.end >= edit.old.start {
121 state.old.end = edit.old.end;
122 state.new.end = edit.new.end;
123 Some(None) // Skip this edit, it's merged
124 } else {
125 let new_state = edit;
126 let result = Some(Some(state.clone())); // Yield the previous edit
127 **state = new_state;
128 result
129 }
130 })
131 .filter_map(|x| x)
132 .collect();
133 v.push(first_edit);
134 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
135 v
136 } else {
137 vec![]
138 };
139
140 for fold_edit in fold_edits {
141 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
142 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
143 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
144 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
145 tab_edits.push(TabEdit {
146 old: old_snapshot.to_tab_point(old_start)..old_snapshot.to_tab_point(old_end),
147 new: new_snapshot.to_tab_point(new_start)..new_snapshot.to_tab_point(new_end),
148 });
149 }
150 } else {
151 new_snapshot.version += 1;
152 tab_edits.push(TabEdit {
153 old: TabPoint::zero()..old_snapshot.max_point(),
154 new: TabPoint::zero()..new_snapshot.max_point(),
155 });
156 }
157
158 *old_snapshot = new_snapshot;
159 (old_snapshot.clone(), tab_edits)
160 }
161}
162
163#[derive(Clone)]
164pub struct TabSnapshot {
165 pub fold_snapshot: FoldSnapshot,
166 pub tab_size: NonZeroU32,
167 pub max_expansion_column: u32,
168 pub version: usize,
169}
170
171impl TabSnapshot {
172 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
173 &self.fold_snapshot.inlay_snapshot.buffer
174 }
175
176 pub fn line_len(&self, row: u32) -> u32 {
177 let max_point = self.max_point();
178 if row < max_point.row() {
179 self.to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
180 .0
181 .column
182 } else {
183 max_point.column()
184 }
185 }
186
187 pub fn text_summary(&self) -> TextSummary {
188 self.text_summary_for_range(TabPoint::zero()..self.max_point())
189 }
190
191 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
192 let input_start = self.to_fold_point(range.start, Bias::Left).0;
193 let input_end = self.to_fold_point(range.end, Bias::Right).0;
194 let input_summary = self
195 .fold_snapshot
196 .text_summary_for_range(input_start..input_end);
197
198 let mut first_line_chars = 0;
199 let line_end = if range.start.row() == range.end.row() {
200 range.end
201 } else {
202 self.max_point()
203 };
204 for c in self
205 .chunks(range.start..line_end, false, Highlights::default())
206 .flat_map(|chunk| chunk.text.chars())
207 {
208 if c == '\n' {
209 break;
210 }
211 first_line_chars += 1;
212 }
213
214 let mut last_line_chars = 0;
215 if range.start.row() == range.end.row() {
216 last_line_chars = first_line_chars;
217 } else {
218 for _ in self
219 .chunks(
220 TabPoint::new(range.end.row(), 0)..range.end,
221 false,
222 Highlights::default(),
223 )
224 .flat_map(|chunk| chunk.text.chars())
225 {
226 last_line_chars += 1;
227 }
228 }
229
230 TextSummary {
231 lines: range.end.0 - range.start.0,
232 first_line_chars,
233 last_line_chars,
234 longest_row: input_summary.longest_row,
235 longest_row_chars: input_summary.longest_row_chars,
236 }
237 }
238
239 pub(crate) fn chunks<'a>(
240 &'a self,
241 range: Range<TabPoint>,
242 language_aware: bool,
243 highlights: Highlights<'a>,
244 ) -> TabChunks<'a> {
245 let (input_start, expanded_char_column, to_next_stop) =
246 self.to_fold_point(range.start, Bias::Left);
247 let input_column = input_start.column();
248 let input_start = input_start.to_offset(&self.fold_snapshot);
249 let input_end = self
250 .to_fold_point(range.end, Bias::Right)
251 .0
252 .to_offset(&self.fold_snapshot);
253 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
254 range.end.column() - range.start.column()
255 } else {
256 to_next_stop
257 };
258
259 TabChunks {
260 snapshot: self,
261 fold_chunks: self.fold_snapshot.chunks(
262 input_start..input_end,
263 language_aware,
264 highlights,
265 ),
266 input_column,
267 column: expanded_char_column,
268 max_expansion_column: self.max_expansion_column,
269 output_position: range.start.0,
270 max_output_position: range.end.0,
271 tab_size: self.tab_size,
272 chunk: Chunk {
273 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
274 is_tab: true,
275 ..Default::default()
276 },
277 inside_leading_tab: to_next_stop > 0,
278 }
279 }
280
281 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
282 self.fold_snapshot.row_infos(row)
283 }
284
285 #[cfg(test)]
286 pub fn text(&self) -> String {
287 self.chunks(
288 TabPoint::zero()..self.max_point(),
289 false,
290 Highlights::default(),
291 )
292 .map(|chunk| chunk.text)
293 .collect()
294 }
295
296 pub fn max_point(&self) -> TabPoint {
297 self.to_tab_point(self.fold_snapshot.max_point())
298 }
299
300 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
301 self.to_tab_point(
302 self.fold_snapshot
303 .clip_point(self.to_fold_point(point, bias).0, bias),
304 )
305 }
306
307 pub fn to_tab_point(&self, input: FoldPoint) -> TabPoint {
308 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
309 let tab_cursor = TabStopCursor::new(chunks);
310 let expanded = self.expand_tabs(tab_cursor, input.column());
311 TabPoint::new(input.row(), expanded)
312 }
313
314 pub fn to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
315 let chunks = self
316 .fold_snapshot
317 .chunks_at(FoldPoint::new(output.row(), 0));
318
319 let tab_cursor = TabStopCursor::new(chunks);
320 let expanded = output.column();
321 let (collapsed, expanded_char_column, to_next_stop) =
322 self.collapse_tabs(tab_cursor, expanded, bias);
323
324 (
325 FoldPoint::new(output.row(), collapsed),
326 expanded_char_column,
327 to_next_stop,
328 )
329 }
330
331 pub fn make_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
332 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
333 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
334 self.to_tab_point(fold_point)
335 }
336
337 pub fn to_point(&self, point: TabPoint, bias: Bias) -> Point {
338 let fold_point = self.to_fold_point(point, bias).0;
339 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
340 self.fold_snapshot
341 .inlay_snapshot
342 .to_buffer_point(inlay_point)
343 }
344
345 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
346 where
347 I: Iterator<Item = Chunk<'a>>,
348 {
349 let tab_size = self.tab_size.get();
350
351 let end_column = column.min(self.max_expansion_column);
352 let mut seek_target = end_column;
353 let mut tab_count = 0;
354 let mut expanded_tab_len = 0;
355
356 while let Some(tab_stop) = cursor.seek(seek_target) {
357 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
358 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
359 tab_count += 1;
360 expanded_tab_len += tab_len;
361
362 seek_target = end_column - cursor.byte_offset;
363 }
364
365 let left_over_char_bytes = if !cursor.is_char_boundary() {
366 cursor.bytes_until_next_char().unwrap_or(0) as u32
367 } else {
368 0
369 };
370
371 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
372 let expanded_bytes =
373 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
374
375 expanded_bytes + column.saturating_sub(collapsed_bytes)
376 }
377
378 fn collapse_tabs<'a, I>(
379 &self,
380 mut cursor: TabStopCursor<'a, I>,
381 column: u32,
382 bias: Bias,
383 ) -> (u32, u32, u32)
384 where
385 I: Iterator<Item = Chunk<'a>>,
386 {
387 let tab_size = self.tab_size.get();
388 let mut collapsed_column = column;
389 let mut seek_target = column.min(self.max_expansion_column);
390 let mut tab_count = 0;
391 let mut expanded_tab_len = 0;
392
393 while let Some(tab_stop) = cursor.seek(seek_target) {
394 // Calculate how much we want to expand this tab stop (into spaces)
395 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
396 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
397 // Increment tab count
398 tab_count += 1;
399 // The count of how many spaces we've added to this line in place of tab bytes
400 expanded_tab_len += tab_len;
401
402 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
403 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
404
405 // Did we expand past the search target?
406 if expanded_bytes > column {
407 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
408 // We expanded past the search target, so need to account for the offshoot
409 expanded_chars -= expanded_bytes - column;
410 return match bias {
411 Bias::Left => (
412 cursor.byte_offset() - 1,
413 expanded_chars,
414 expanded_bytes - column,
415 ),
416 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
417 };
418 } else {
419 // otherwise we only want to move the cursor collapse column forward
420 collapsed_column = collapsed_column - tab_len + 1;
421 seek_target = (collapsed_column - cursor.byte_offset)
422 .min(self.max_expansion_column - cursor.byte_offset);
423 }
424 }
425
426 let collapsed_bytes = cursor.byte_offset();
427 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
428 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
429 (
430 collapsed_bytes + column.saturating_sub(expanded_bytes),
431 expanded_chars,
432 0,
433 )
434 }
435}
436
437#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
438pub struct TabPoint(pub Point);
439
440impl TabPoint {
441 pub fn new(row: u32, column: u32) -> Self {
442 Self(Point::new(row, column))
443 }
444
445 pub fn zero() -> Self {
446 Self::new(0, 0)
447 }
448
449 pub fn row(self) -> u32 {
450 self.0.row
451 }
452
453 pub fn column(self) -> u32 {
454 self.0.column
455 }
456}
457
458impl From<Point> for TabPoint {
459 fn from(point: Point) -> Self {
460 Self(point)
461 }
462}
463
464pub type TabEdit = text::Edit<TabPoint>;
465
466#[derive(Clone, Debug, Default, Eq, PartialEq)]
467pub struct TextSummary {
468 pub lines: Point,
469 pub first_line_chars: u32,
470 pub last_line_chars: u32,
471 pub longest_row: u32,
472 pub longest_row_chars: u32,
473}
474
475impl<'a> From<&'a str> for TextSummary {
476 fn from(text: &'a str) -> Self {
477 let sum = text::TextSummary::from(text);
478
479 TextSummary {
480 lines: sum.lines,
481 first_line_chars: sum.first_line_chars,
482 last_line_chars: sum.last_line_chars,
483 longest_row: sum.longest_row,
484 longest_row_chars: sum.longest_row_chars,
485 }
486 }
487}
488
489impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
490 fn add_assign(&mut self, other: &'a Self) {
491 let joined_chars = self.last_line_chars + other.first_line_chars;
492 if joined_chars > self.longest_row_chars {
493 self.longest_row = self.lines.row;
494 self.longest_row_chars = joined_chars;
495 }
496 if other.longest_row_chars > self.longest_row_chars {
497 self.longest_row = self.lines.row + other.longest_row;
498 self.longest_row_chars = other.longest_row_chars;
499 }
500
501 if self.lines.row == 0 {
502 self.first_line_chars += other.first_line_chars;
503 }
504
505 if other.lines.row == 0 {
506 self.last_line_chars += other.first_line_chars;
507 } else {
508 self.last_line_chars = other.last_line_chars;
509 }
510
511 self.lines += &other.lines;
512 }
513}
514
515pub struct TabChunks<'a> {
516 snapshot: &'a TabSnapshot,
517 fold_chunks: FoldChunks<'a>,
518 chunk: Chunk<'a>,
519 column: u32,
520 max_expansion_column: u32,
521 output_position: Point,
522 input_column: u32,
523 max_output_position: Point,
524 tab_size: NonZeroU32,
525 inside_leading_tab: bool,
526}
527
528impl TabChunks<'_> {
529 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
530 let (input_start, expanded_char_column, to_next_stop) =
531 self.snapshot.to_fold_point(range.start, Bias::Left);
532 let input_column = input_start.column();
533 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
534 let input_end = self
535 .snapshot
536 .to_fold_point(range.end, Bias::Right)
537 .0
538 .to_offset(&self.snapshot.fold_snapshot);
539 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
540 range.end.column() - range.start.column()
541 } else {
542 to_next_stop
543 };
544
545 self.fold_chunks.seek(input_start..input_end);
546 self.input_column = input_column;
547 self.column = expanded_char_column;
548 self.output_position = range.start.0;
549 self.max_output_position = range.end.0;
550 self.chunk = Chunk {
551 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
552 is_tab: true,
553 chars: 1u128.unbounded_shl(to_next_stop) - 1,
554 ..Default::default()
555 };
556 self.inside_leading_tab = to_next_stop > 0;
557 }
558}
559
560impl<'a> Iterator for TabChunks<'a> {
561 type Item = Chunk<'a>;
562
563 fn next(&mut self) -> Option<Self::Item> {
564 if self.chunk.text.is_empty() {
565 if let Some(chunk) = self.fold_chunks.next() {
566 self.chunk = chunk;
567 if self.inside_leading_tab {
568 self.chunk.text = &self.chunk.text[1..];
569 self.inside_leading_tab = false;
570 self.input_column += 1;
571 }
572 } else {
573 return None;
574 }
575 }
576
577 //todo(improve performance by using tab cursor)
578 for (ix, c) in self.chunk.text.char_indices() {
579 match c {
580 '\t' => {
581 if ix > 0 {
582 let (prefix, suffix) = self.chunk.text.split_at(ix);
583
584 let (chars, tabs) = if ix == 128 {
585 let output = (self.chunk.chars, self.chunk.tabs);
586 self.chunk.chars = 0;
587 self.chunk.tabs = 0;
588 output
589 } else {
590 let mask = (1 << ix) - 1;
591 let output = (self.chunk.chars & mask, self.chunk.tabs & mask);
592 self.chunk.chars = self.chunk.chars >> ix;
593 self.chunk.tabs = self.chunk.tabs >> ix;
594 output
595 };
596
597 self.chunk.text = suffix;
598 return Some(Chunk {
599 text: prefix,
600 chars,
601 tabs,
602 ..self.chunk.clone()
603 });
604 } else {
605 self.chunk.text = &self.chunk.text[1..];
606 self.chunk.tabs >>= 1;
607 self.chunk.chars >>= 1;
608 let tab_size = if self.input_column < self.max_expansion_column {
609 self.tab_size.get()
610 } else {
611 1
612 };
613 let mut len = tab_size - self.column % tab_size;
614 let next_output_position = cmp::min(
615 self.output_position + Point::new(0, len),
616 self.max_output_position,
617 );
618 len = next_output_position.column - self.output_position.column;
619 self.column += len;
620 self.input_column += 1;
621 self.output_position = next_output_position;
622 return Some(Chunk {
623 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
624 is_tab: true,
625 chars: 1u128.unbounded_shl(len) - 1,
626 tabs: 0,
627 ..self.chunk.clone()
628 });
629 }
630 }
631 '\n' => {
632 self.column = 0;
633 self.input_column = 0;
634 self.output_position += Point::new(1, 0);
635 }
636 _ => {
637 self.column += 1;
638 if !self.inside_leading_tab {
639 self.input_column += c.len_utf8() as u32;
640 }
641 self.output_position.column += c.len_utf8() as u32;
642 }
643 }
644 }
645
646 Some(mem::take(&mut self.chunk))
647 }
648}
649
650#[cfg(test)]
651mod tests {
652 use super::*;
653 use crate::{
654 MultiBuffer,
655 display_map::{
656 fold_map::{FoldMap, FoldOffset},
657 inlay_map::InlayMap,
658 },
659 };
660 use rand::{Rng, prelude::StdRng};
661 use util;
662
663 impl TabSnapshot {
664 fn expected_collapse_tabs(
665 &self,
666 chars: impl Iterator<Item = char>,
667 column: u32,
668 bias: Bias,
669 ) -> (u32, u32, u32) {
670 let tab_size = self.tab_size.get();
671
672 let mut expanded_bytes = 0;
673 let mut expanded_chars = 0;
674 let mut collapsed_bytes = 0;
675 for c in chars {
676 if expanded_bytes >= column {
677 break;
678 }
679 if collapsed_bytes >= self.max_expansion_column {
680 break;
681 }
682
683 if c == '\t' {
684 let tab_len = tab_size - (expanded_chars % tab_size);
685 expanded_chars += tab_len;
686 expanded_bytes += tab_len;
687 if expanded_bytes > column {
688 expanded_chars -= expanded_bytes - column;
689 return match bias {
690 Bias::Left => {
691 (collapsed_bytes, expanded_chars, expanded_bytes - column)
692 }
693 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
694 };
695 }
696 } else {
697 expanded_chars += 1;
698 expanded_bytes += c.len_utf8() as u32;
699 }
700
701 if expanded_bytes > column && matches!(bias, Bias::Left) {
702 expanded_chars -= 1;
703 break;
704 }
705
706 collapsed_bytes += c.len_utf8() as u32;
707 }
708
709 (
710 collapsed_bytes + column.saturating_sub(expanded_bytes),
711 expanded_chars,
712 0,
713 )
714 }
715
716 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
717 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
718 let expanded = self.expected_expand_tabs(chars, input.column());
719 TabPoint::new(input.row(), expanded)
720 }
721
722 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
723 let tab_size = self.tab_size.get();
724
725 let mut expanded_chars = 0;
726 let mut expanded_bytes = 0;
727 let mut collapsed_bytes = 0;
728 let end_column = column.min(self.max_expansion_column);
729 for c in chars {
730 if collapsed_bytes >= end_column {
731 break;
732 }
733 if c == '\t' {
734 let tab_len = tab_size - expanded_chars % tab_size;
735 expanded_bytes += tab_len;
736 expanded_chars += tab_len;
737 } else {
738 expanded_bytes += c.len_utf8() as u32;
739 expanded_chars += 1;
740 }
741 collapsed_bytes += c.len_utf8() as u32;
742 }
743
744 expanded_bytes + column.saturating_sub(collapsed_bytes)
745 }
746
747 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
748 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
749 let expanded = output.column();
750 let (collapsed, expanded_char_column, to_next_stop) =
751 self.expected_collapse_tabs(chars, expanded, bias);
752 (
753 FoldPoint::new(output.row(), collapsed),
754 expanded_char_column,
755 to_next_stop,
756 )
757 }
758 }
759
760 #[gpui::test]
761 fn test_expand_tabs(cx: &mut gpui::App) {
762 let test_values = [
763 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
764 (" \twςe", 4),
765 ("fε", 1),
766 ("i❎\t", 3),
767 ];
768 let buffer = MultiBuffer::build_simple("", cx);
769 let buffer_snapshot = buffer.read(cx).snapshot(cx);
770 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
771 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
772 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
773
774 for (text, column) in test_values {
775 let mut tabs = 0u128;
776 let mut chars = 0u128;
777 for (idx, c) in text.char_indices() {
778 if c == '\t' {
779 tabs |= 1 << idx;
780 }
781 chars |= 1 << idx;
782 }
783
784 let chunks = [Chunk {
785 text,
786 tabs,
787 chars,
788 ..Default::default()
789 }];
790
791 let cursor = TabStopCursor::new(chunks);
792
793 assert_eq!(
794 tab_snapshot.expected_expand_tabs(text.chars(), column),
795 tab_snapshot.expand_tabs(cursor, column)
796 );
797 }
798 }
799
800 #[gpui::test]
801 fn test_collapse_tabs(cx: &mut gpui::App) {
802 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
803
804 let buffer = MultiBuffer::build_simple(input, cx);
805 let buffer_snapshot = buffer.read(cx).snapshot(cx);
806 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
807 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
808 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
809
810 for (ix, _) in input.char_indices() {
811 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
812
813 assert_eq!(
814 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
815 tab_snapshot.to_fold_point(range.start, Bias::Left),
816 "Failed with tab_point at column {ix}"
817 );
818 assert_eq!(
819 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
820 tab_snapshot.to_fold_point(range.start, Bias::Right),
821 "Failed with tab_point at column {ix}"
822 );
823
824 assert_eq!(
825 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
826 tab_snapshot.to_fold_point(range.end, Bias::Left),
827 "Failed with tab_point at column {ix}"
828 );
829 assert_eq!(
830 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
831 tab_snapshot.to_fold_point(range.end, Bias::Right),
832 "Failed with tab_point at column {ix}"
833 );
834 }
835 }
836
837 #[gpui::test]
838 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
839 // This test reproduces a specific panic where to_fold_point returns incorrect results
840 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
841
842 let text = "γ\tw⭐\n🍐🍗 \t";
843 let buffer = MultiBuffer::build_simple(text, cx);
844 let buffer_snapshot = buffer.read(cx).snapshot(cx);
845 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
846 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
847 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
848
849 // This should panic with the expected vs actual mismatch
850 let tab_point = TabPoint::new(0, 9);
851 let result = tab_snapshot.to_fold_point(tab_point, Bias::Left);
852 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
853
854 assert_eq!(result, expected);
855 }
856
857 #[gpui::test(iterations = 100)]
858 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
859 // Generate random input string with up to 200 characters including tabs
860 // to stay within the MAX_EXPANSION_COLUMN limit of 256
861 let len = rng.random_range(0..=2048);
862 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
863 let mut input = String::with_capacity(len);
864
865 for _ in 0..len {
866 if rng.random_bool(0.1) {
867 // 10% chance of inserting a tab
868 input.push('\t');
869 } else {
870 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
871 let ch = loop {
872 let ascii_code = rng.random_range(32..=126); // printable ASCII range
873 let ch = ascii_code as u8 as char;
874 if ch != '\t' {
875 break ch;
876 }
877 };
878 input.push(ch);
879 }
880 }
881
882 let buffer = MultiBuffer::build_simple(&input, cx);
883 let buffer_snapshot = buffer.read(cx).snapshot(cx);
884 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
885 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
886 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
887 tab_snapshot.max_expansion_column = rng.random_range(0..323);
888 tab_snapshot.tab_size = tab_size;
889
890 for (ix, _) in input.char_indices() {
891 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
892
893 assert_eq!(
894 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
895 tab_snapshot.to_fold_point(range.start, Bias::Left),
896 "Failed with input: {}, with idx: {ix}",
897 input
898 );
899 assert_eq!(
900 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
901 tab_snapshot.to_fold_point(range.start, Bias::Right),
902 "Failed with input: {}, with idx: {ix}",
903 input
904 );
905
906 assert_eq!(
907 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
908 tab_snapshot.to_fold_point(range.end, Bias::Left),
909 "Failed with input: {}, with idx: {ix}",
910 input
911 );
912 assert_eq!(
913 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
914 tab_snapshot.to_fold_point(range.end, Bias::Right),
915 "Failed with input: {}, with idx: {ix}",
916 input
917 );
918 }
919 }
920
921 #[gpui::test]
922 fn test_long_lines(cx: &mut gpui::App) {
923 let max_expansion_column = 12;
924 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
925 let output = "A BC DEF G HI J K L M";
926
927 let buffer = MultiBuffer::build_simple(input, cx);
928 let buffer_snapshot = buffer.read(cx).snapshot(cx);
929 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
930 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
931 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
932
933 tab_snapshot.max_expansion_column = max_expansion_column;
934 assert_eq!(tab_snapshot.text(), output);
935
936 for (ix, c) in input.char_indices() {
937 assert_eq!(
938 tab_snapshot
939 .chunks(
940 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
941 false,
942 Highlights::default(),
943 )
944 .map(|c| c.text)
945 .collect::<String>(),
946 &output[ix..],
947 "text from index {ix}"
948 );
949
950 if c != '\t' {
951 let input_point = Point::new(0, ix as u32);
952 let output_point = Point::new(0, output.find(c).unwrap() as u32);
953 assert_eq!(
954 tab_snapshot.to_tab_point(FoldPoint(input_point)),
955 TabPoint(output_point),
956 "to_tab_point({input_point:?})"
957 );
958 assert_eq!(
959 tab_snapshot
960 .to_fold_point(TabPoint(output_point), Bias::Left)
961 .0,
962 FoldPoint(input_point),
963 "to_fold_point({output_point:?})"
964 );
965 }
966 }
967 }
968
969 #[gpui::test]
970 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
971 let max_expansion_column = 8;
972 let input = "abcdefg⋯hij";
973
974 let buffer = MultiBuffer::build_simple(input, cx);
975 let buffer_snapshot = buffer.read(cx).snapshot(cx);
976 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
977 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
978 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
979
980 tab_snapshot.max_expansion_column = max_expansion_column;
981 assert_eq!(tab_snapshot.text(), input);
982 }
983
984 #[gpui::test]
985 fn test_marking_tabs(cx: &mut gpui::App) {
986 let input = "\t \thello";
987
988 let buffer = MultiBuffer::build_simple(input, cx);
989 let buffer_snapshot = buffer.read(cx).snapshot(cx);
990 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
991 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
992 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
993
994 assert_eq!(
995 chunks(&tab_snapshot, TabPoint::zero()),
996 vec![
997 (" ".to_string(), true),
998 (" ".to_string(), false),
999 (" ".to_string(), true),
1000 ("hello".to_string(), false),
1001 ]
1002 );
1003 assert_eq!(
1004 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1005 vec![
1006 (" ".to_string(), true),
1007 (" ".to_string(), false),
1008 (" ".to_string(), true),
1009 ("hello".to_string(), false),
1010 ]
1011 );
1012
1013 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1014 let mut chunks = Vec::new();
1015 let mut was_tab = false;
1016 let mut text = String::new();
1017 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1018 {
1019 if chunk.is_tab != was_tab {
1020 if !text.is_empty() {
1021 chunks.push((mem::take(&mut text), was_tab));
1022 }
1023 was_tab = chunk.is_tab;
1024 }
1025 text.push_str(chunk.text);
1026 }
1027
1028 if !text.is_empty() {
1029 chunks.push((text, was_tab));
1030 }
1031 chunks
1032 }
1033 }
1034
1035 #[gpui::test(iterations = 100)]
1036 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1037 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1038 let len = rng.random_range(0..30);
1039 let buffer = if rng.random() {
1040 let text = util::RandomCharIter::new(&mut rng)
1041 .take(len)
1042 .collect::<String>();
1043 MultiBuffer::build_simple(&text, cx)
1044 } else {
1045 MultiBuffer::build_random(&mut rng, cx)
1046 };
1047 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1048 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1049
1050 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1051 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1052 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1053 fold_map.randomly_mutate(&mut rng);
1054 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1055 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1056 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1057 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1058
1059 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1060 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1061
1062 let text = text::Rope::from(tabs_snapshot.text().as_str());
1063 log::info!(
1064 "TabMap text (tab size: {}): {:?}",
1065 tab_size,
1066 tabs_snapshot.text(),
1067 );
1068
1069 for _ in 0..5 {
1070 let end_row = rng.random_range(0..=text.max_point().row);
1071 let end_column = rng.random_range(0..=text.line_len(end_row));
1072 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1073 let start_row = rng.random_range(0..=text.max_point().row);
1074 let start_column = rng.random_range(0..=text.line_len(start_row));
1075 let mut start =
1076 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1077 if start > end {
1078 mem::swap(&mut start, &mut end);
1079 }
1080
1081 let expected_text = text
1082 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1083 .collect::<String>();
1084 let expected_summary = TextSummary::from(expected_text.as_str());
1085 assert_eq!(
1086 tabs_snapshot
1087 .chunks(start..end, false, Highlights::default())
1088 .map(|c| c.text)
1089 .collect::<String>(),
1090 expected_text,
1091 "chunks({:?}..{:?})",
1092 start,
1093 end
1094 );
1095
1096 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1097 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1098 actual_summary.longest_row = expected_summary.longest_row;
1099 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1100 }
1101 assert_eq!(actual_summary, expected_summary);
1102 }
1103
1104 for row in 0..=text.max_point().row {
1105 assert_eq!(
1106 tabs_snapshot.line_len(row),
1107 text.line_len(row),
1108 "line_len({row})"
1109 );
1110 }
1111 }
1112
1113 #[gpui::test(iterations = 100)]
1114 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1115 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1116 let len = rng.random_range(0..=2000);
1117
1118 // Generate random text using RandomCharIter
1119 let text = util::RandomCharIter::new(&mut rng)
1120 .take(len)
1121 .collect::<String>();
1122
1123 // Create buffer and tab map
1124 let buffer = MultiBuffer::build_simple(&text, cx);
1125 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1126 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1127 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1128 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1129
1130 let mut next_inlay_id = 0;
1131 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1132 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1133 let max_fold_point = fold_snapshot.max_point();
1134 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1135
1136 // Test random fold points
1137 for _ in 0..50 {
1138 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1139 // Generate random fold point
1140 let row = rng.random_range(0..=max_fold_point.row());
1141 let max_column = if row < max_fold_point.row() {
1142 fold_snapshot.line_len(row)
1143 } else {
1144 max_fold_point.column()
1145 };
1146 let column = rng.random_range(0..=max_column + 10);
1147 let fold_point = FoldPoint::new(row, column);
1148
1149 let actual = tab_snapshot.to_tab_point(fold_point);
1150 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1151
1152 assert_eq!(
1153 actual, expected,
1154 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1155 fold_point, text
1156 );
1157 }
1158 }
1159
1160 #[gpui::test]
1161 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1162 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1163 let buffer = MultiBuffer::build_simple(text, cx);
1164 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1165 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1166 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1167 let chunks = fold_snapshot.chunks(
1168 FoldOffset(0)..fold_snapshot.len(),
1169 false,
1170 Default::default(),
1171 );
1172 let mut cursor = TabStopCursor::new(chunks);
1173 assert!(cursor.seek(0).is_none());
1174 let mut tab_stops = Vec::new();
1175
1176 let mut all_tab_stops = Vec::new();
1177 let mut byte_offset = 0;
1178 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1179 byte_offset += ch.len_utf8() as u32;
1180
1181 if ch == '\t' {
1182 all_tab_stops.push(TabStop {
1183 byte_offset,
1184 char_offset: offset as u32 + 1,
1185 });
1186 }
1187 }
1188
1189 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1190 tab_stops.push(tab_stop);
1191 }
1192 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1193
1194 assert_eq!(cursor.byte_offset(), byte_offset);
1195 }
1196
1197 #[gpui::test]
1198 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1199 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1200
1201 let buffer = MultiBuffer::build_simple(input, cx);
1202 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1203 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1204 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1205
1206 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1207 let mut cursor = TabStopCursor::new(chunks);
1208
1209 let mut actual_tab_stops = Vec::new();
1210
1211 let mut expected_tab_stops = Vec::new();
1212 let mut byte_offset = 0;
1213 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1214 byte_offset += ch.len_utf8() as u32;
1215
1216 if ch == '\t' {
1217 expected_tab_stops.push(TabStop {
1218 byte_offset,
1219 char_offset: offset as u32 + 1,
1220 });
1221 }
1222 }
1223
1224 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1225 actual_tab_stops.push(tab_stop);
1226 }
1227 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1228
1229 assert_eq!(cursor.byte_offset(), byte_offset);
1230 }
1231
1232 #[gpui::test(iterations = 100)]
1233 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1234 // Generate random input string with up to 512 characters including tabs
1235 let len = rng.random_range(0..=2048);
1236 let mut input = String::with_capacity(len);
1237
1238 let mut skip_tabs = rng.random_bool(0.10);
1239 for idx in 0..len {
1240 if idx % 128 == 0 {
1241 skip_tabs = rng.random_bool(0.10);
1242 }
1243
1244 if rng.random_bool(0.15) && !skip_tabs {
1245 input.push('\t');
1246 } else {
1247 let ch = loop {
1248 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1249 let ch = ascii_code as u8 as char;
1250 if ch != '\t' {
1251 break ch;
1252 }
1253 };
1254 input.push(ch);
1255 }
1256 }
1257
1258 // Build the buffer and create cursor
1259 let buffer = MultiBuffer::build_simple(&input, cx);
1260 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1261 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1262 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1263
1264 // First, collect all expected tab positions
1265 let mut all_tab_stops = Vec::new();
1266 let mut byte_offset = 1;
1267 let mut char_offset = 1;
1268 for ch in buffer_snapshot.text().chars() {
1269 if ch == '\t' {
1270 all_tab_stops.push(TabStop {
1271 byte_offset,
1272 char_offset,
1273 });
1274 }
1275 byte_offset += ch.len_utf8() as u32;
1276 char_offset += 1;
1277 }
1278
1279 // Test with various distances
1280 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1281 // let distances = vec![150];
1282
1283 for distance in distances {
1284 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1285 let mut cursor = TabStopCursor::new(chunks);
1286
1287 let mut found_tab_stops = Vec::new();
1288 let mut position = distance;
1289 while let Some(tab_stop) = cursor.seek(position) {
1290 found_tab_stops.push(tab_stop);
1291 position = distance - tab_stop.byte_offset;
1292 }
1293
1294 let expected_found_tab_stops: Vec<_> = all_tab_stops
1295 .iter()
1296 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1297 .cloned()
1298 .collect();
1299
1300 pretty_assertions::assert_eq!(
1301 found_tab_stops,
1302 expected_found_tab_stops,
1303 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1304 distance,
1305 input
1306 );
1307
1308 let final_position = cursor.byte_offset();
1309 if !found_tab_stops.is_empty() {
1310 let last_tab_stop = found_tab_stops.last().unwrap();
1311 assert!(
1312 final_position >= last_tab_stop.byte_offset,
1313 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1314 final_position,
1315 last_tab_stop.byte_offset,
1316 input
1317 );
1318 }
1319 }
1320 }
1321
1322 #[gpui::test]
1323 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1324 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1325 let buffer = MultiBuffer::build_simple(text, cx);
1326 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1327 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1328 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1329 let chunks = fold_snapshot.chunks(
1330 FoldOffset(0)..fold_snapshot.len(),
1331 false,
1332 Default::default(),
1333 );
1334 let mut cursor = TabStopCursor::new(chunks);
1335 assert!(cursor.seek(0).is_none());
1336
1337 let mut expected_tab_stops = Vec::new();
1338 let mut byte_offset = 0;
1339 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1340 byte_offset += ch.len_utf8() as u32;
1341
1342 if ch == '\t' {
1343 expected_tab_stops.push(TabStop {
1344 byte_offset,
1345 char_offset: i as u32 + 1,
1346 });
1347 }
1348 }
1349
1350 let mut actual_tab_stops = Vec::new();
1351 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1352 actual_tab_stops.push(tab_stop);
1353 }
1354
1355 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1356
1357 assert_eq!(cursor.byte_offset(), byte_offset);
1358 }
1359
1360 #[gpui::test(iterations = 100)]
1361 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1362 // Generate random input string with up to 512 characters including tabs
1363 let len = rng.random_range(0..=2048);
1364 let input = util::RandomCharIter::new(&mut rng)
1365 .take(len)
1366 .collect::<String>();
1367
1368 // Build the buffer and create cursor
1369 let buffer = MultiBuffer::build_simple(&input, cx);
1370 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1371 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1372 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1373
1374 // First, collect all expected tab positions
1375 let mut all_tab_stops = Vec::new();
1376 let mut byte_offset = 0;
1377 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1378 byte_offset += ch.len_utf8() as u32;
1379 if ch == '\t' {
1380 all_tab_stops.push(TabStop {
1381 byte_offset,
1382 char_offset: i as u32 + 1,
1383 });
1384 }
1385 }
1386
1387 // Test with various distances
1388 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1389 let distances = vec![150];
1390
1391 for distance in distances {
1392 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1393 let mut cursor = TabStopCursor::new(chunks);
1394
1395 let mut found_tab_stops = Vec::new();
1396 let mut position = distance;
1397 while let Some(tab_stop) = cursor.seek(position) {
1398 found_tab_stops.push(tab_stop);
1399 position = distance - tab_stop.byte_offset;
1400 }
1401
1402 let expected_found_tab_stops: Vec<_> = all_tab_stops
1403 .iter()
1404 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1405 .cloned()
1406 .collect();
1407
1408 pretty_assertions::assert_eq!(
1409 found_tab_stops,
1410 expected_found_tab_stops,
1411 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1412 distance,
1413 input
1414 );
1415
1416 let final_position = cursor.byte_offset();
1417 if !found_tab_stops.is_empty() {
1418 let last_tab_stop = found_tab_stops.last().unwrap();
1419 assert!(
1420 final_position >= last_tab_stop.byte_offset,
1421 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1422 final_position,
1423 last_tab_stop.byte_offset,
1424 input
1425 );
1426 }
1427 }
1428 }
1429}
1430
1431struct TabStopCursor<'a, I>
1432where
1433 I: Iterator<Item = Chunk<'a>>,
1434{
1435 chunks: I,
1436 byte_offset: u32,
1437 char_offset: u32,
1438 /// Chunk
1439 /// last tab position iterated through
1440 current_chunk: Option<(Chunk<'a>, u32)>,
1441}
1442
1443impl<'a, I> TabStopCursor<'a, I>
1444where
1445 I: Iterator<Item = Chunk<'a>>,
1446{
1447 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1448 Self {
1449 chunks: chunks.into_iter(),
1450 byte_offset: 0,
1451 char_offset: 0,
1452 current_chunk: None,
1453 }
1454 }
1455
1456 fn bytes_until_next_char(&self) -> Option<usize> {
1457 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1458 let mut idx = *idx;
1459 let mut diff = 0;
1460 while idx > 0 && chunk.chars & (1 << idx) == 0 {
1461 idx -= 1;
1462 diff += 1;
1463 }
1464
1465 if chunk.chars & (1 << idx) != 0 {
1466 Some(
1467 (chunk.text[idx as usize..].chars().next()?)
1468 .len_utf8()
1469 .saturating_sub(diff),
1470 )
1471 } else {
1472 None
1473 }
1474 })
1475 }
1476
1477 fn is_char_boundary(&self) -> bool {
1478 self.current_chunk
1479 .as_ref()
1480 .is_some_and(|(chunk, idx)| (chunk.chars & (1 << *idx.min(&127))) != 0)
1481 }
1482
1483 /// distance: length to move forward while searching for the next tab stop
1484 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1485 if distance == 0 {
1486 return None;
1487 }
1488
1489 let mut distance_traversed = 0;
1490
1491 while let Some((mut chunk, chunk_position)) = self
1492 .current_chunk
1493 .take()
1494 .or_else(|| self.chunks.next().zip(Some(0)))
1495 {
1496 if chunk.tabs == 0 {
1497 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1498 if chunk_distance + distance_traversed >= distance {
1499 let overshoot = distance_traversed.abs_diff(distance);
1500
1501 self.byte_offset += overshoot;
1502 self.char_offset += get_char_offset(
1503 chunk_position..(chunk_position + overshoot).saturating_sub(1).min(127),
1504 chunk.chars,
1505 );
1506
1507 self.current_chunk = Some((chunk, chunk_position + overshoot));
1508
1509 return None;
1510 }
1511
1512 self.byte_offset += chunk_distance;
1513 self.char_offset += get_char_offset(
1514 chunk_position..(chunk_position + chunk_distance).saturating_sub(1).min(127),
1515 chunk.chars,
1516 );
1517 distance_traversed += chunk_distance;
1518 continue;
1519 }
1520 let tab_position = chunk.tabs.trailing_zeros() + 1;
1521
1522 if distance_traversed + tab_position - chunk_position > distance {
1523 let cursor_position = distance_traversed.abs_diff(distance);
1524
1525 self.char_offset += get_char_offset(
1526 chunk_position..(chunk_position + cursor_position - 1),
1527 chunk.chars,
1528 );
1529 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1530 self.byte_offset += cursor_position;
1531
1532 return None;
1533 }
1534
1535 self.byte_offset += tab_position - chunk_position;
1536 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1537
1538 let tabstop = TabStop {
1539 char_offset: self.char_offset,
1540 byte_offset: self.byte_offset,
1541 };
1542
1543 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1544
1545 if tab_position as usize != chunk.text.len() {
1546 self.current_chunk = Some((chunk, tab_position));
1547 }
1548
1549 return Some(tabstop);
1550 }
1551
1552 None
1553 }
1554
1555 fn byte_offset(&self) -> u32 {
1556 self.byte_offset
1557 }
1558
1559 fn char_offset(&self) -> u32 {
1560 self.char_offset
1561 }
1562}
1563
1564#[inline(always)]
1565fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1566 // This edge case can happen when we're at chunk position 128
1567
1568 if range.start == range.end {
1569 return if (1u128 << range.start) & bit_map == 0 {
1570 0
1571 } else {
1572 1
1573 };
1574 }
1575 let end_shift: u128 = 127u128 - range.end.min(127) as u128;
1576 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1577 bit_mask = (bit_mask << end_shift) >> end_shift;
1578 let bit_map = bit_map & bit_mask;
1579
1580 bit_map.count_ones()
1581}
1582
1583#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1584struct TabStop {
1585 char_offset: u32,
1586 byte_offset: u32,
1587}