1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13/// Keeps track of hard tabs in a text buffer.
14///
15/// See the [`display_map` module documentation](crate::display_map) for more information.
16pub struct TabMap(TabSnapshot);
17
18impl TabMap {
19 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
20 let snapshot = TabSnapshot {
21 fold_snapshot,
22 tab_size,
23 max_expansion_column: MAX_EXPANSION_COLUMN,
24 version: 0,
25 };
26 (Self(snapshot.clone()), snapshot)
27 }
28
29 #[cfg(test)]
30 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
31 self.0.max_expansion_column = column;
32 self.0.clone()
33 }
34
35 pub fn sync(
36 &mut self,
37 fold_snapshot: FoldSnapshot,
38 mut fold_edits: Vec<FoldEdit>,
39 tab_size: NonZeroU32,
40 ) -> (TabSnapshot, Vec<TabEdit>) {
41 let old_snapshot = &mut self.0;
42 let mut new_snapshot = TabSnapshot {
43 fold_snapshot,
44 tab_size,
45 max_expansion_column: old_snapshot.max_expansion_column,
46 version: old_snapshot.version,
47 };
48
49 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
50 new_snapshot.version += 1;
51 }
52
53 let mut tab_edits = Vec::with_capacity(fold_edits.len());
54
55 if old_snapshot.tab_size == new_snapshot.tab_size {
56 // Expand each edit to include the next tab on the same line as the edit,
57 // and any subsequent tabs on that line that moved across the tab expansion
58 // boundary.
59 for fold_edit in &mut fold_edits {
60 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
61 let old_end_row_successor_offset = cmp::min(
62 FoldPoint::new(old_end.row() + 1, 0),
63 old_snapshot.fold_snapshot.max_point(),
64 )
65 .to_offset(&old_snapshot.fold_snapshot);
66 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
67
68 let mut offset_from_edit = 0;
69 let mut first_tab_offset = None;
70 let mut last_tab_with_changed_expansion_offset = None;
71 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
72 fold_edit.old.end..old_end_row_successor_offset,
73 false,
74 Highlights::default(),
75 ) {
76 // todo(performance use tabs bitmask)
77 for (ix, _) in chunk.text.match_indices('\t') {
78 let offset_from_edit = offset_from_edit + (ix as u32);
79 if first_tab_offset.is_none() {
80 first_tab_offset = Some(offset_from_edit);
81 }
82
83 let old_column = old_end.column() + offset_from_edit;
84 let new_column = new_end.column() + offset_from_edit;
85 let was_expanded = old_column < old_snapshot.max_expansion_column;
86 let is_expanded = new_column < new_snapshot.max_expansion_column;
87 if was_expanded != is_expanded {
88 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
89 } else if !was_expanded && !is_expanded {
90 break 'outer;
91 }
92 }
93
94 offset_from_edit += chunk.text.len() as u32;
95 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
96 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
97 {
98 break;
99 }
100 }
101
102 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
103 fold_edit.old.end.0 += offset as usize + 1;
104 fold_edit.new.end.0 += offset as usize + 1;
105 }
106 }
107
108 let _old_alloc_ptr = fold_edits.as_ptr();
109 // Combine any edits that overlap due to the expansion.
110 let mut fold_edits = fold_edits.into_iter();
111 let fold_edits = if let Some(mut first_edit) = fold_edits.next() {
112 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
113 #[allow(clippy::filter_map_identity)]
114 let mut v: Vec<_> = fold_edits
115 .scan(&mut first_edit, |state, edit| {
116 if state.old.end >= edit.old.start {
117 state.old.end = edit.old.end;
118 state.new.end = edit.new.end;
119 Some(None) // Skip this edit, it's merged
120 } else {
121 let new_state = edit;
122 let result = Some(Some(state.clone())); // Yield the previous edit
123 **state = new_state;
124 result
125 }
126 })
127 .filter_map(|x| x)
128 .collect();
129 v.push(first_edit);
130 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
131 v
132 } else {
133 vec![]
134 };
135
136 for fold_edit in fold_edits {
137 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
138 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
139 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
140 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
141 tab_edits.push(TabEdit {
142 old: old_snapshot.to_tab_point(old_start)..old_snapshot.to_tab_point(old_end),
143 new: new_snapshot.to_tab_point(new_start)..new_snapshot.to_tab_point(new_end),
144 });
145 }
146 } else {
147 new_snapshot.version += 1;
148 tab_edits.push(TabEdit {
149 old: TabPoint::zero()..old_snapshot.max_point(),
150 new: TabPoint::zero()..new_snapshot.max_point(),
151 });
152 }
153
154 *old_snapshot = new_snapshot;
155 (old_snapshot.clone(), tab_edits)
156 }
157}
158
159#[derive(Clone)]
160pub struct TabSnapshot {
161 pub fold_snapshot: FoldSnapshot,
162 pub tab_size: NonZeroU32,
163 pub max_expansion_column: u32,
164 pub version: usize,
165}
166
167impl TabSnapshot {
168 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
169 &self.fold_snapshot.inlay_snapshot.buffer
170 }
171
172 pub fn line_len(&self, row: u32) -> u32 {
173 let max_point = self.max_point();
174 if row < max_point.row() {
175 self.to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
176 .0
177 .column
178 } else {
179 max_point.column()
180 }
181 }
182
183 pub fn text_summary(&self) -> TextSummary {
184 self.text_summary_for_range(TabPoint::zero()..self.max_point())
185 }
186
187 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
188 let input_start = self.to_fold_point(range.start, Bias::Left).0;
189 let input_end = self.to_fold_point(range.end, Bias::Right).0;
190 let input_summary = self
191 .fold_snapshot
192 .text_summary_for_range(input_start..input_end);
193
194 let mut first_line_chars = 0;
195 let line_end = if range.start.row() == range.end.row() {
196 range.end
197 } else {
198 self.max_point()
199 };
200 for c in self
201 .chunks(range.start..line_end, false, Highlights::default())
202 .flat_map(|chunk| chunk.text.chars())
203 {
204 if c == '\n' {
205 break;
206 }
207 first_line_chars += 1;
208 }
209
210 let mut last_line_chars = 0;
211 if range.start.row() == range.end.row() {
212 last_line_chars = first_line_chars;
213 } else {
214 for _ in self
215 .chunks(
216 TabPoint::new(range.end.row(), 0)..range.end,
217 false,
218 Highlights::default(),
219 )
220 .flat_map(|chunk| chunk.text.chars())
221 {
222 last_line_chars += 1;
223 }
224 }
225
226 TextSummary {
227 lines: range.end.0 - range.start.0,
228 first_line_chars,
229 last_line_chars,
230 longest_row: input_summary.longest_row,
231 longest_row_chars: input_summary.longest_row_chars,
232 }
233 }
234
235 pub(crate) fn chunks<'a>(
236 &'a self,
237 range: Range<TabPoint>,
238 language_aware: bool,
239 highlights: Highlights<'a>,
240 ) -> TabChunks<'a> {
241 let (input_start, expanded_char_column, to_next_stop) =
242 self.to_fold_point(range.start, Bias::Left);
243 let input_column = input_start.column();
244 let input_start = input_start.to_offset(&self.fold_snapshot);
245 let input_end = self
246 .to_fold_point(range.end, Bias::Right)
247 .0
248 .to_offset(&self.fold_snapshot);
249 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
250 range.end.column() - range.start.column()
251 } else {
252 to_next_stop
253 };
254
255 TabChunks {
256 snapshot: self,
257 fold_chunks: self.fold_snapshot.chunks(
258 input_start..input_end,
259 language_aware,
260 highlights,
261 ),
262 input_column,
263 column: expanded_char_column,
264 max_expansion_column: self.max_expansion_column,
265 output_position: range.start.0,
266 max_output_position: range.end.0,
267 tab_size: self.tab_size,
268 chunk: Chunk {
269 text: &SPACES[0..(to_next_stop as usize)],
270 is_tab: true,
271 ..Default::default()
272 },
273 inside_leading_tab: to_next_stop > 0,
274 }
275 }
276
277 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
278 self.fold_snapshot.row_infos(row)
279 }
280
281 #[cfg(test)]
282 pub fn text(&self) -> String {
283 self.chunks(
284 TabPoint::zero()..self.max_point(),
285 false,
286 Highlights::default(),
287 )
288 .map(|chunk| chunk.text)
289 .collect()
290 }
291
292 pub fn max_point(&self) -> TabPoint {
293 self.to_tab_point(self.fold_snapshot.max_point())
294 }
295
296 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
297 self.to_tab_point(
298 self.fold_snapshot
299 .clip_point(self.to_fold_point(point, bias).0, bias),
300 )
301 }
302
303 pub fn to_tab_point(&self, input: FoldPoint) -> TabPoint {
304 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
305 let tab_cursor = TabStopCursor::new(chunks);
306 let expanded = self.expand_tabs(tab_cursor, input.column());
307 TabPoint::new(input.row(), expanded)
308 }
309
310 pub fn to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
311 let chunks = self
312 .fold_snapshot
313 .chunks_at(FoldPoint::new(output.row(), 0));
314
315 let tab_cursor = TabStopCursor::new(chunks);
316 let expanded = output.column();
317 let (collapsed, expanded_char_column, to_next_stop) =
318 self.collapse_tabs(tab_cursor, expanded, bias);
319
320 let result = (
321 FoldPoint::new(output.row(), collapsed),
322 expanded_char_column,
323 to_next_stop,
324 );
325
326 result
327 }
328
329 pub fn make_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
330 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
331 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
332 self.to_tab_point(fold_point)
333 }
334
335 pub fn to_point(&self, point: TabPoint, bias: Bias) -> Point {
336 let fold_point = self.to_fold_point(point, bias).0;
337 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
338 self.fold_snapshot
339 .inlay_snapshot
340 .to_buffer_point(inlay_point)
341 }
342
343 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
344 where
345 I: Iterator<Item = Chunk<'a>>,
346 {
347 let tab_size = self.tab_size.get();
348
349 let end_column = column.min(self.max_expansion_column);
350 let mut seek_target = end_column;
351 let mut tab_count = 0;
352 let mut expanded_tab_len = 0;
353
354 while let Some(tab_stop) = cursor.seek(seek_target) {
355 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
356 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
357 tab_count += 1;
358 expanded_tab_len += tab_len;
359
360 seek_target = end_column - cursor.byte_offset;
361 }
362
363 let left_over_char_bytes = if !cursor.is_char_boundary() {
364 cursor.bytes_until_next_char().unwrap_or(0) as u32
365 } else {
366 0
367 };
368
369 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
370 let expanded_bytes =
371 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
372
373 expanded_bytes + column.saturating_sub(collapsed_bytes)
374 }
375
376 fn collapse_tabs<'a, I>(
377 &self,
378 mut cursor: TabStopCursor<'a, I>,
379 column: u32,
380 bias: Bias,
381 ) -> (u32, u32, u32)
382 where
383 I: Iterator<Item = Chunk<'a>>,
384 {
385 let tab_size = self.tab_size.get();
386 let mut collapsed_column = column;
387 let mut seek_target = column.min(self.max_expansion_column);
388 let mut tab_count = 0;
389 let mut expanded_tab_len = 0;
390
391 while let Some(tab_stop) = cursor.seek(seek_target) {
392 // Calculate how much we want to expand this tab stop (into spaces)
393 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
394 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
395 // Increment tab count
396 tab_count += 1;
397 // The count of how many spaces we've added to this line in place of tab bytes
398 expanded_tab_len += tab_len;
399
400 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
401 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
402
403 // Did we expand past the search target?
404 if expanded_bytes > column {
405 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
406 // We expanded past the search target, so need to account for the offshoot
407 expanded_chars -= expanded_bytes - column;
408 return match bias {
409 Bias::Left => (
410 cursor.byte_offset() - 1,
411 expanded_chars,
412 expanded_bytes - column,
413 ),
414 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
415 };
416 } else {
417 // otherwise we only want to move the cursor collapse column forward
418 collapsed_column = collapsed_column - tab_len + 1;
419 seek_target = (collapsed_column - cursor.byte_offset)
420 .min(self.max_expansion_column - cursor.byte_offset);
421 }
422 }
423
424 let collapsed_bytes = cursor.byte_offset();
425 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
426 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
427 (
428 collapsed_bytes + column.saturating_sub(expanded_bytes),
429 expanded_chars,
430 0,
431 )
432 }
433}
434
435#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
436pub struct TabPoint(pub Point);
437
438impl TabPoint {
439 pub fn new(row: u32, column: u32) -> Self {
440 Self(Point::new(row, column))
441 }
442
443 pub fn zero() -> Self {
444 Self::new(0, 0)
445 }
446
447 pub fn row(self) -> u32 {
448 self.0.row
449 }
450
451 pub fn column(self) -> u32 {
452 self.0.column
453 }
454}
455
456impl From<Point> for TabPoint {
457 fn from(point: Point) -> Self {
458 Self(point)
459 }
460}
461
462pub type TabEdit = text::Edit<TabPoint>;
463
464#[derive(Clone, Debug, Default, Eq, PartialEq)]
465pub struct TextSummary {
466 pub lines: Point,
467 pub first_line_chars: u32,
468 pub last_line_chars: u32,
469 pub longest_row: u32,
470 pub longest_row_chars: u32,
471}
472
473impl<'a> From<&'a str> for TextSummary {
474 fn from(text: &'a str) -> Self {
475 let sum = text::TextSummary::from(text);
476
477 TextSummary {
478 lines: sum.lines,
479 first_line_chars: sum.first_line_chars,
480 last_line_chars: sum.last_line_chars,
481 longest_row: sum.longest_row,
482 longest_row_chars: sum.longest_row_chars,
483 }
484 }
485}
486
487impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
488 fn add_assign(&mut self, other: &'a Self) {
489 let joined_chars = self.last_line_chars + other.first_line_chars;
490 if joined_chars > self.longest_row_chars {
491 self.longest_row = self.lines.row;
492 self.longest_row_chars = joined_chars;
493 }
494 if other.longest_row_chars > self.longest_row_chars {
495 self.longest_row = self.lines.row + other.longest_row;
496 self.longest_row_chars = other.longest_row_chars;
497 }
498
499 if self.lines.row == 0 {
500 self.first_line_chars += other.first_line_chars;
501 }
502
503 if other.lines.row == 0 {
504 self.last_line_chars += other.first_line_chars;
505 } else {
506 self.last_line_chars = other.last_line_chars;
507 }
508
509 self.lines += &other.lines;
510 }
511}
512
513// Handles a tab width <= 16
514const SPACES: &str = " ";
515
516pub struct TabChunks<'a> {
517 snapshot: &'a TabSnapshot,
518 fold_chunks: FoldChunks<'a>,
519 chunk: Chunk<'a>,
520 column: u32,
521 max_expansion_column: u32,
522 output_position: Point,
523 input_column: u32,
524 max_output_position: Point,
525 tab_size: NonZeroU32,
526 inside_leading_tab: bool,
527}
528
529impl TabChunks<'_> {
530 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
531 let (input_start, expanded_char_column, to_next_stop) =
532 self.snapshot.to_fold_point(range.start, Bias::Left);
533 let input_column = input_start.column();
534 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
535 let input_end = self
536 .snapshot
537 .to_fold_point(range.end, Bias::Right)
538 .0
539 .to_offset(&self.snapshot.fold_snapshot);
540 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
541 range.end.column() - range.start.column()
542 } else {
543 to_next_stop
544 };
545
546 self.fold_chunks.seek(input_start..input_end);
547 self.input_column = input_column;
548 self.column = expanded_char_column;
549 self.output_position = range.start.0;
550 self.max_output_position = range.end.0;
551 self.chunk = Chunk {
552 text: &SPACES[0..(to_next_stop as usize)],
553 is_tab: true,
554 chars: 1u128.unbounded_shl(to_next_stop) - 1,
555 ..Default::default()
556 };
557 self.inside_leading_tab = to_next_stop > 0;
558 }
559}
560
561impl<'a> Iterator for TabChunks<'a> {
562 type Item = Chunk<'a>;
563
564 fn next(&mut self) -> Option<Self::Item> {
565 if self.chunk.text.is_empty() {
566 if let Some(chunk) = self.fold_chunks.next() {
567 self.chunk = chunk;
568 if self.inside_leading_tab {
569 self.chunk.text = &self.chunk.text[1..];
570 self.inside_leading_tab = false;
571 self.input_column += 1;
572 }
573 } else {
574 return None;
575 }
576 }
577
578 //todo(improve performance by using tab cursor)
579 for (ix, c) in self.chunk.text.char_indices() {
580 match c {
581 '\t' => {
582 if ix > 0 {
583 let (prefix, suffix) = self.chunk.text.split_at(ix);
584
585 let (chars, tabs) = if ix == 128 {
586 let output = (self.chunk.chars, self.chunk.tabs);
587 self.chunk.chars = 0;
588 self.chunk.tabs = 0;
589 output
590 } else {
591 let mask = (1 << ix) - 1;
592 let output = (self.chunk.chars & mask, self.chunk.tabs & mask);
593 self.chunk.chars = self.chunk.chars >> ix;
594 self.chunk.tabs = self.chunk.tabs >> ix;
595 output
596 };
597
598 self.chunk.text = suffix;
599 return Some(Chunk {
600 text: prefix,
601 chars,
602 tabs,
603 ..self.chunk.clone()
604 });
605 } else {
606 self.chunk.text = &self.chunk.text[1..];
607 self.chunk.tabs >>= 1;
608 self.chunk.chars >>= 1;
609 let tab_size = if self.input_column < self.max_expansion_column {
610 self.tab_size.get()
611 } else {
612 1
613 };
614 let mut len = tab_size - self.column % tab_size;
615 let next_output_position = cmp::min(
616 self.output_position + Point::new(0, len),
617 self.max_output_position,
618 );
619 len = next_output_position.column - self.output_position.column;
620 self.column += len;
621 self.input_column += 1;
622 self.output_position = next_output_position;
623 return Some(Chunk {
624 text: &SPACES[..len as usize],
625 is_tab: true,
626 chars: 1u128.unbounded_shl(len) - 1,
627 tabs: 0,
628 ..self.chunk.clone()
629 });
630 }
631 }
632 '\n' => {
633 self.column = 0;
634 self.input_column = 0;
635 self.output_position += Point::new(1, 0);
636 }
637 _ => {
638 self.column += 1;
639 if !self.inside_leading_tab {
640 self.input_column += c.len_utf8() as u32;
641 }
642 self.output_position.column += c.len_utf8() as u32;
643 }
644 }
645 }
646
647 Some(mem::take(&mut self.chunk))
648 }
649}
650
651#[cfg(test)]
652mod tests {
653 use super::*;
654 use crate::{
655 MultiBuffer,
656 display_map::{
657 fold_map::{FoldMap, FoldOffset},
658 inlay_map::InlayMap,
659 },
660 };
661 use rand::{Rng, prelude::StdRng};
662 use util;
663
664 impl TabSnapshot {
665 fn expected_collapse_tabs(
666 &self,
667 chars: impl Iterator<Item = char>,
668 column: u32,
669 bias: Bias,
670 ) -> (u32, u32, u32) {
671 let tab_size = self.tab_size.get();
672
673 let mut expanded_bytes = 0;
674 let mut expanded_chars = 0;
675 let mut collapsed_bytes = 0;
676 for c in chars {
677 if expanded_bytes >= column {
678 break;
679 }
680 if collapsed_bytes >= self.max_expansion_column {
681 break;
682 }
683
684 if c == '\t' {
685 let tab_len = tab_size - (expanded_chars % tab_size);
686 expanded_chars += tab_len;
687 expanded_bytes += tab_len;
688 if expanded_bytes > column {
689 expanded_chars -= expanded_bytes - column;
690 return match bias {
691 Bias::Left => {
692 (collapsed_bytes, expanded_chars, expanded_bytes - column)
693 }
694 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
695 };
696 }
697 } else {
698 expanded_chars += 1;
699 expanded_bytes += c.len_utf8() as u32;
700 }
701
702 if expanded_bytes > column && matches!(bias, Bias::Left) {
703 expanded_chars -= 1;
704 break;
705 }
706
707 collapsed_bytes += c.len_utf8() as u32;
708 }
709
710 (
711 collapsed_bytes + column.saturating_sub(expanded_bytes),
712 expanded_chars,
713 0,
714 )
715 }
716
717 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
718 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
719 let expanded = self.expected_expand_tabs(chars, input.column());
720 TabPoint::new(input.row(), expanded)
721 }
722
723 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
724 let tab_size = self.tab_size.get();
725
726 let mut expanded_chars = 0;
727 let mut expanded_bytes = 0;
728 let mut collapsed_bytes = 0;
729 let end_column = column.min(self.max_expansion_column);
730 for c in chars {
731 if collapsed_bytes >= end_column {
732 break;
733 }
734 if c == '\t' {
735 let tab_len = tab_size - expanded_chars % tab_size;
736 expanded_bytes += tab_len;
737 expanded_chars += tab_len;
738 } else {
739 expanded_bytes += c.len_utf8() as u32;
740 expanded_chars += 1;
741 }
742 collapsed_bytes += c.len_utf8() as u32;
743 }
744
745 expanded_bytes + column.saturating_sub(collapsed_bytes)
746 }
747
748 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
749 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
750 let expanded = output.column();
751 let (collapsed, expanded_char_column, to_next_stop) =
752 self.expected_collapse_tabs(chars, expanded, bias);
753 (
754 FoldPoint::new(output.row(), collapsed),
755 expanded_char_column,
756 to_next_stop,
757 )
758 }
759 }
760
761 #[gpui::test]
762 fn test_expand_tabs(cx: &mut gpui::App) {
763 let test_values = [
764 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
765 (" \twςe", 4),
766 ("fε", 1),
767 ("i❎\t", 3),
768 ];
769 let buffer = MultiBuffer::build_simple("", cx);
770 let buffer_snapshot = buffer.read(cx).snapshot(cx);
771 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
772 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
773 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
774
775 for (text, column) in test_values {
776 let mut tabs = 0u128;
777 let mut chars = 0u128;
778 for (idx, c) in text.char_indices() {
779 if c == '\t' {
780 tabs |= 1 << idx;
781 }
782 chars |= 1 << idx;
783 }
784
785 let chunks = [Chunk {
786 text,
787 tabs,
788 chars,
789 ..Default::default()
790 }];
791
792 let cursor = TabStopCursor::new(chunks);
793
794 assert_eq!(
795 tab_snapshot.expected_expand_tabs(text.chars(), column),
796 tab_snapshot.expand_tabs(cursor, column)
797 );
798 }
799 }
800
801 #[gpui::test]
802 fn test_collapse_tabs(cx: &mut gpui::App) {
803 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
804
805 let buffer = MultiBuffer::build_simple(input, cx);
806 let buffer_snapshot = buffer.read(cx).snapshot(cx);
807 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
808 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
809 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
810
811 for (ix, _) in input.char_indices() {
812 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
813
814 assert_eq!(
815 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
816 tab_snapshot.to_fold_point(range.start, Bias::Left),
817 "Failed with tab_point at column {ix}"
818 );
819 assert_eq!(
820 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
821 tab_snapshot.to_fold_point(range.start, Bias::Right),
822 "Failed with tab_point at column {ix}"
823 );
824
825 assert_eq!(
826 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
827 tab_snapshot.to_fold_point(range.end, Bias::Left),
828 "Failed with tab_point at column {ix}"
829 );
830 assert_eq!(
831 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
832 tab_snapshot.to_fold_point(range.end, Bias::Right),
833 "Failed with tab_point at column {ix}"
834 );
835 }
836 }
837
838 #[gpui::test]
839 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
840 // This test reproduces a specific panic where to_fold_point returns incorrect results
841 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
842
843 let text = "γ\tw⭐\n🍐🍗 \t";
844 let buffer = MultiBuffer::build_simple(text, cx);
845 let buffer_snapshot = buffer.read(cx).snapshot(cx);
846 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
847 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
848 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
849
850 // This should panic with the expected vs actual mismatch
851 let tab_point = TabPoint::new(0, 9);
852 let result = tab_snapshot.to_fold_point(tab_point, Bias::Left);
853 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
854
855 assert_eq!(result, expected);
856 }
857
858 #[gpui::test(iterations = 100)]
859 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
860 // Generate random input string with up to 200 characters including tabs
861 // to stay within the MAX_EXPANSION_COLUMN limit of 256
862 let len = rng.random_range(0..=2048);
863 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
864 let mut input = String::with_capacity(len);
865
866 for _ in 0..len {
867 if rng.random_bool(0.1) {
868 // 10% chance of inserting a tab
869 input.push('\t');
870 } else {
871 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
872 let ch = loop {
873 let ascii_code = rng.random_range(32..=126); // printable ASCII range
874 let ch = ascii_code as u8 as char;
875 if ch != '\t' {
876 break ch;
877 }
878 };
879 input.push(ch);
880 }
881 }
882
883 let buffer = MultiBuffer::build_simple(&input, cx);
884 let buffer_snapshot = buffer.read(cx).snapshot(cx);
885 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
886 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
887 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
888 tab_snapshot.max_expansion_column = rng.random_range(0..323);
889 tab_snapshot.tab_size = tab_size;
890
891 for (ix, _) in input.char_indices() {
892 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
893
894 assert_eq!(
895 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
896 tab_snapshot.to_fold_point(range.start, Bias::Left),
897 "Failed with input: {}, with idx: {ix}",
898 input
899 );
900 assert_eq!(
901 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
902 tab_snapshot.to_fold_point(range.start, Bias::Right),
903 "Failed with input: {}, with idx: {ix}",
904 input
905 );
906
907 assert_eq!(
908 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
909 tab_snapshot.to_fold_point(range.end, Bias::Left),
910 "Failed with input: {}, with idx: {ix}",
911 input
912 );
913 assert_eq!(
914 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
915 tab_snapshot.to_fold_point(range.end, Bias::Right),
916 "Failed with input: {}, with idx: {ix}",
917 input
918 );
919 }
920 }
921
922 #[gpui::test]
923 fn test_long_lines(cx: &mut gpui::App) {
924 let max_expansion_column = 12;
925 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
926 let output = "A BC DEF G HI J K L M";
927
928 let buffer = MultiBuffer::build_simple(input, cx);
929 let buffer_snapshot = buffer.read(cx).snapshot(cx);
930 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
931 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
932 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
933
934 tab_snapshot.max_expansion_column = max_expansion_column;
935 assert_eq!(tab_snapshot.text(), output);
936
937 for (ix, c) in input.char_indices() {
938 assert_eq!(
939 tab_snapshot
940 .chunks(
941 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
942 false,
943 Highlights::default(),
944 )
945 .map(|c| c.text)
946 .collect::<String>(),
947 &output[ix..],
948 "text from index {ix}"
949 );
950
951 if c != '\t' {
952 let input_point = Point::new(0, ix as u32);
953 let output_point = Point::new(0, output.find(c).unwrap() as u32);
954 assert_eq!(
955 tab_snapshot.to_tab_point(FoldPoint(input_point)),
956 TabPoint(output_point),
957 "to_tab_point({input_point:?})"
958 );
959 assert_eq!(
960 tab_snapshot
961 .to_fold_point(TabPoint(output_point), Bias::Left)
962 .0,
963 FoldPoint(input_point),
964 "to_fold_point({output_point:?})"
965 );
966 }
967 }
968 }
969
970 #[gpui::test]
971 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
972 let max_expansion_column = 8;
973 let input = "abcdefg⋯hij";
974
975 let buffer = MultiBuffer::build_simple(input, cx);
976 let buffer_snapshot = buffer.read(cx).snapshot(cx);
977 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
978 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
979 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
980
981 tab_snapshot.max_expansion_column = max_expansion_column;
982 assert_eq!(tab_snapshot.text(), input);
983 }
984
985 #[gpui::test]
986 fn test_marking_tabs(cx: &mut gpui::App) {
987 let input = "\t \thello";
988
989 let buffer = MultiBuffer::build_simple(input, cx);
990 let buffer_snapshot = buffer.read(cx).snapshot(cx);
991 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
992 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
993 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
994
995 assert_eq!(
996 chunks(&tab_snapshot, TabPoint::zero()),
997 vec![
998 (" ".to_string(), true),
999 (" ".to_string(), false),
1000 (" ".to_string(), true),
1001 ("hello".to_string(), false),
1002 ]
1003 );
1004 assert_eq!(
1005 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1006 vec![
1007 (" ".to_string(), true),
1008 (" ".to_string(), false),
1009 (" ".to_string(), true),
1010 ("hello".to_string(), false),
1011 ]
1012 );
1013
1014 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1015 let mut chunks = Vec::new();
1016 let mut was_tab = false;
1017 let mut text = String::new();
1018 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1019 {
1020 if chunk.is_tab != was_tab {
1021 if !text.is_empty() {
1022 chunks.push((mem::take(&mut text), was_tab));
1023 }
1024 was_tab = chunk.is_tab;
1025 }
1026 text.push_str(chunk.text);
1027 }
1028
1029 if !text.is_empty() {
1030 chunks.push((text, was_tab));
1031 }
1032 chunks
1033 }
1034 }
1035
1036 #[gpui::test(iterations = 100)]
1037 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1038 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1039 let len = rng.random_range(0..30);
1040 let buffer = if rng.random() {
1041 let text = util::RandomCharIter::new(&mut rng)
1042 .take(len)
1043 .collect::<String>();
1044 MultiBuffer::build_simple(&text, cx)
1045 } else {
1046 MultiBuffer::build_random(&mut rng, cx)
1047 };
1048 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1049 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1050
1051 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1052 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1053 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1054 fold_map.randomly_mutate(&mut rng);
1055 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1056 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1057 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1058 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1059
1060 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1061 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1062
1063 let text = text::Rope::from(tabs_snapshot.text().as_str());
1064 log::info!(
1065 "TabMap text (tab size: {}): {:?}",
1066 tab_size,
1067 tabs_snapshot.text(),
1068 );
1069
1070 for _ in 0..5 {
1071 let end_row = rng.random_range(0..=text.max_point().row);
1072 let end_column = rng.random_range(0..=text.line_len(end_row));
1073 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1074 let start_row = rng.random_range(0..=text.max_point().row);
1075 let start_column = rng.random_range(0..=text.line_len(start_row));
1076 let mut start =
1077 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1078 if start > end {
1079 mem::swap(&mut start, &mut end);
1080 }
1081
1082 let expected_text = text
1083 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1084 .collect::<String>();
1085 let expected_summary = TextSummary::from(expected_text.as_str());
1086 assert_eq!(
1087 tabs_snapshot
1088 .chunks(start..end, false, Highlights::default())
1089 .map(|c| c.text)
1090 .collect::<String>(),
1091 expected_text,
1092 "chunks({:?}..{:?})",
1093 start,
1094 end
1095 );
1096
1097 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1098 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1099 actual_summary.longest_row = expected_summary.longest_row;
1100 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1101 }
1102 assert_eq!(actual_summary, expected_summary);
1103 }
1104
1105 for row in 0..=text.max_point().row {
1106 assert_eq!(
1107 tabs_snapshot.line_len(row),
1108 text.line_len(row),
1109 "line_len({row})"
1110 );
1111 }
1112 }
1113
1114 #[gpui::test(iterations = 100)]
1115 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1116 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1117 let len = rng.random_range(0..=2000);
1118
1119 // Generate random text using RandomCharIter
1120 let text = util::RandomCharIter::new(&mut rng)
1121 .take(len)
1122 .collect::<String>();
1123
1124 // Create buffer and tab map
1125 let buffer = MultiBuffer::build_simple(&text, cx);
1126 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1127 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1128 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1129 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1130
1131 let mut next_inlay_id = 0;
1132 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1133 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1134 let max_fold_point = fold_snapshot.max_point();
1135 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1136
1137 // Test random fold points
1138 for _ in 0..50 {
1139 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1140 // Generate random fold point
1141 let row = rng.random_range(0..=max_fold_point.row());
1142 let max_column = if row < max_fold_point.row() {
1143 fold_snapshot.line_len(row)
1144 } else {
1145 max_fold_point.column()
1146 };
1147 let column = rng.random_range(0..=max_column + 10);
1148 let fold_point = FoldPoint::new(row, column);
1149
1150 let actual = tab_snapshot.to_tab_point(fold_point);
1151 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1152
1153 assert_eq!(
1154 actual, expected,
1155 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1156 fold_point, text
1157 );
1158 }
1159 }
1160
1161 #[gpui::test]
1162 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1163 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1164 let buffer = MultiBuffer::build_simple(text, cx);
1165 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1166 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1167 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1168 let chunks = fold_snapshot.chunks(
1169 FoldOffset(0)..fold_snapshot.len(),
1170 false,
1171 Default::default(),
1172 );
1173 let mut cursor = TabStopCursor::new(chunks);
1174 assert!(cursor.seek(0).is_none());
1175 let mut tab_stops = Vec::new();
1176
1177 let mut all_tab_stops = Vec::new();
1178 let mut byte_offset = 0;
1179 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1180 byte_offset += ch.len_utf8() as u32;
1181
1182 if ch == '\t' {
1183 all_tab_stops.push(TabStop {
1184 byte_offset,
1185 char_offset: offset as u32 + 1,
1186 });
1187 }
1188 }
1189
1190 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1191 tab_stops.push(tab_stop);
1192 }
1193 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1194
1195 assert_eq!(cursor.byte_offset(), byte_offset);
1196 }
1197
1198 #[gpui::test]
1199 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1200 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1201
1202 let buffer = MultiBuffer::build_simple(input, cx);
1203 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1204 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1205 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1206
1207 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1208 let mut cursor = TabStopCursor::new(chunks);
1209
1210 let mut actual_tab_stops = Vec::new();
1211
1212 let mut expected_tab_stops = Vec::new();
1213 let mut byte_offset = 0;
1214 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1215 byte_offset += ch.len_utf8() as u32;
1216
1217 if ch == '\t' {
1218 expected_tab_stops.push(TabStop {
1219 byte_offset,
1220 char_offset: offset as u32 + 1,
1221 });
1222 }
1223 }
1224
1225 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1226 actual_tab_stops.push(tab_stop);
1227 }
1228 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1229
1230 assert_eq!(cursor.byte_offset(), byte_offset);
1231 }
1232
1233 #[gpui::test(iterations = 100)]
1234 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1235 // Generate random input string with up to 512 characters including tabs
1236 let len = rng.random_range(0..=2048);
1237 let mut input = String::with_capacity(len);
1238
1239 let mut skip_tabs = rng.random_bool(0.10);
1240 for idx in 0..len {
1241 if idx % 128 == 0 {
1242 skip_tabs = rng.random_bool(0.10);
1243 }
1244
1245 if rng.random_bool(0.15) && !skip_tabs {
1246 input.push('\t');
1247 } else {
1248 let ch = loop {
1249 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1250 let ch = ascii_code as u8 as char;
1251 if ch != '\t' {
1252 break ch;
1253 }
1254 };
1255 input.push(ch);
1256 }
1257 }
1258
1259 // Build the buffer and create cursor
1260 let buffer = MultiBuffer::build_simple(&input, cx);
1261 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1262 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1263 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1264
1265 // First, collect all expected tab positions
1266 let mut all_tab_stops = Vec::new();
1267 let mut byte_offset = 1;
1268 let mut char_offset = 1;
1269 for ch in buffer_snapshot.text().chars() {
1270 if ch == '\t' {
1271 all_tab_stops.push(TabStop {
1272 byte_offset,
1273 char_offset,
1274 });
1275 }
1276 byte_offset += ch.len_utf8() as u32;
1277 char_offset += 1;
1278 }
1279
1280 // Test with various distances
1281 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1282 // let distances = vec![150];
1283
1284 for distance in distances {
1285 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1286 let mut cursor = TabStopCursor::new(chunks);
1287
1288 let mut found_tab_stops = Vec::new();
1289 let mut position = distance;
1290 while let Some(tab_stop) = cursor.seek(position) {
1291 found_tab_stops.push(tab_stop);
1292 position = distance - tab_stop.byte_offset;
1293 }
1294
1295 let expected_found_tab_stops: Vec<_> = all_tab_stops
1296 .iter()
1297 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1298 .cloned()
1299 .collect();
1300
1301 pretty_assertions::assert_eq!(
1302 found_tab_stops,
1303 expected_found_tab_stops,
1304 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1305 distance,
1306 input
1307 );
1308
1309 let final_position = cursor.byte_offset();
1310 if !found_tab_stops.is_empty() {
1311 let last_tab_stop = found_tab_stops.last().unwrap();
1312 assert!(
1313 final_position >= last_tab_stop.byte_offset,
1314 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1315 final_position,
1316 last_tab_stop.byte_offset,
1317 input
1318 );
1319 }
1320 }
1321 }
1322
1323 #[gpui::test]
1324 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1325 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1326 let buffer = MultiBuffer::build_simple(text, cx);
1327 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1328 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1329 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1330 let chunks = fold_snapshot.chunks(
1331 FoldOffset(0)..fold_snapshot.len(),
1332 false,
1333 Default::default(),
1334 );
1335 let mut cursor = TabStopCursor::new(chunks);
1336 assert!(cursor.seek(0).is_none());
1337
1338 let mut expected_tab_stops = Vec::new();
1339 let mut byte_offset = 0;
1340 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1341 byte_offset += ch.len_utf8() as u32;
1342
1343 if ch == '\t' {
1344 expected_tab_stops.push(TabStop {
1345 byte_offset,
1346 char_offset: i as u32 + 1,
1347 });
1348 }
1349 }
1350
1351 let mut actual_tab_stops = Vec::new();
1352 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1353 actual_tab_stops.push(tab_stop);
1354 }
1355
1356 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1357
1358 assert_eq!(cursor.byte_offset(), byte_offset);
1359 }
1360
1361 #[gpui::test(iterations = 100)]
1362 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1363 // Generate random input string with up to 512 characters including tabs
1364 let len = rng.random_range(0..=2048);
1365 let input = util::RandomCharIter::new(&mut rng)
1366 .take(len)
1367 .collect::<String>();
1368
1369 // Build the buffer and create cursor
1370 let buffer = MultiBuffer::build_simple(&input, cx);
1371 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1372 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1373 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1374
1375 // First, collect all expected tab positions
1376 let mut all_tab_stops = Vec::new();
1377 let mut byte_offset = 0;
1378 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1379 byte_offset += ch.len_utf8() as u32;
1380 if ch == '\t' {
1381 all_tab_stops.push(TabStop {
1382 byte_offset,
1383 char_offset: i as u32 + 1,
1384 });
1385 }
1386 }
1387
1388 // Test with various distances
1389 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1390 let distances = vec![150];
1391
1392 for distance in distances {
1393 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1394 let mut cursor = TabStopCursor::new(chunks);
1395
1396 let mut found_tab_stops = Vec::new();
1397 let mut position = distance;
1398 while let Some(tab_stop) = cursor.seek(position) {
1399 found_tab_stops.push(tab_stop);
1400 position = distance - tab_stop.byte_offset;
1401 }
1402
1403 let expected_found_tab_stops: Vec<_> = all_tab_stops
1404 .iter()
1405 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1406 .cloned()
1407 .collect();
1408
1409 pretty_assertions::assert_eq!(
1410 found_tab_stops,
1411 expected_found_tab_stops,
1412 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1413 distance,
1414 input
1415 );
1416
1417 let final_position = cursor.byte_offset();
1418 if !found_tab_stops.is_empty() {
1419 let last_tab_stop = found_tab_stops.last().unwrap();
1420 assert!(
1421 final_position >= last_tab_stop.byte_offset,
1422 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1423 final_position,
1424 last_tab_stop.byte_offset,
1425 input
1426 );
1427 }
1428 }
1429 }
1430}
1431
1432struct TabStopCursor<'a, I>
1433where
1434 I: Iterator<Item = Chunk<'a>>,
1435{
1436 chunks: I,
1437 byte_offset: u32,
1438 char_offset: u32,
1439 /// Chunk
1440 /// last tab position iterated through
1441 current_chunk: Option<(Chunk<'a>, u32)>,
1442}
1443
1444impl<'a, I> TabStopCursor<'a, I>
1445where
1446 I: Iterator<Item = Chunk<'a>>,
1447{
1448 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1449 Self {
1450 chunks: chunks.into_iter(),
1451 byte_offset: 0,
1452 char_offset: 0,
1453 current_chunk: None,
1454 }
1455 }
1456
1457 fn bytes_until_next_char(&self) -> Option<usize> {
1458 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1459 let mut idx = *idx;
1460 let mut diff = 0;
1461 while idx > 0 && chunk.chars & (1 << idx) == 0 {
1462 idx -= 1;
1463 diff += 1;
1464 }
1465
1466 if chunk.chars & (1 << idx) != 0 {
1467 Some(
1468 (chunk.text[idx as usize..].chars().next()?)
1469 .len_utf8()
1470 .saturating_sub(diff),
1471 )
1472 } else {
1473 None
1474 }
1475 })
1476 }
1477
1478 fn is_char_boundary(&self) -> bool {
1479 self.current_chunk
1480 .as_ref()
1481 .is_some_and(|(chunk, idx)| (chunk.chars & (1 << *idx.min(&127))) != 0)
1482 }
1483
1484 /// distance: length to move forward while searching for the next tab stop
1485 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1486 if distance == 0 {
1487 return None;
1488 }
1489
1490 let mut distance_traversed = 0;
1491
1492 while let Some((mut chunk, chunk_position)) = self
1493 .current_chunk
1494 .take()
1495 .or_else(|| self.chunks.next().zip(Some(0)))
1496 {
1497 if chunk.tabs == 0 {
1498 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1499 if chunk_distance + distance_traversed >= distance {
1500 let overshoot = distance_traversed.abs_diff(distance);
1501
1502 self.byte_offset += overshoot;
1503 self.char_offset += get_char_offset(
1504 chunk_position..(chunk_position + overshoot).saturating_sub(1).min(127),
1505 chunk.chars,
1506 );
1507
1508 self.current_chunk = Some((chunk, chunk_position + overshoot));
1509
1510 return None;
1511 }
1512
1513 self.byte_offset += chunk_distance;
1514 self.char_offset += get_char_offset(
1515 chunk_position..(chunk_position + chunk_distance).saturating_sub(1).min(127),
1516 chunk.chars,
1517 );
1518 distance_traversed += chunk_distance;
1519 continue;
1520 }
1521 let tab_position = chunk.tabs.trailing_zeros() + 1;
1522
1523 if distance_traversed + tab_position - chunk_position > distance {
1524 let cursor_position = distance_traversed.abs_diff(distance);
1525
1526 self.char_offset += get_char_offset(
1527 chunk_position..(chunk_position + cursor_position - 1),
1528 chunk.chars,
1529 );
1530 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1531 self.byte_offset += cursor_position;
1532
1533 return None;
1534 }
1535
1536 self.byte_offset += tab_position - chunk_position;
1537 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1538
1539 let tabstop = TabStop {
1540 char_offset: self.char_offset,
1541 byte_offset: self.byte_offset,
1542 };
1543
1544 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1545
1546 if tab_position as usize != chunk.text.len() {
1547 self.current_chunk = Some((chunk, tab_position));
1548 }
1549
1550 return Some(tabstop);
1551 }
1552
1553 None
1554 }
1555
1556 fn byte_offset(&self) -> u32 {
1557 self.byte_offset
1558 }
1559
1560 fn char_offset(&self) -> u32 {
1561 self.char_offset
1562 }
1563}
1564
1565#[inline(always)]
1566fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1567 // This edge case can happen when we're at chunk position 128
1568
1569 if range.start == range.end {
1570 return if (1u128 << range.start) & bit_map == 0 {
1571 0
1572 } else {
1573 1
1574 };
1575 }
1576 let end_shift: u128 = 127u128 - range.end.min(127) as u128;
1577 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1578 bit_mask = (bit_mask << end_shift) >> end_shift;
1579 let bit_map = bit_map & bit_mask;
1580
1581 bit_map.count_ones()
1582}
1583
1584#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1585struct TabStop {
1586 char_offset: u32,
1587 byte_offset: u32,
1588}