1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
24 let snapshot = TabSnapshot {
25 fold_snapshot,
26 tab_size: tab_size.min(MAX_TABS),
27 max_expansion_column: MAX_EXPANSION_COLUMN,
28 version: 0,
29 };
30 (Self(snapshot.clone()), snapshot)
31 }
32
33 #[cfg(test)]
34 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
35 self.0.max_expansion_column = column;
36 self.0.clone()
37 }
38
39 pub fn sync(
40 &mut self,
41 fold_snapshot: FoldSnapshot,
42 mut fold_edits: Vec<FoldEdit>,
43 tab_size: NonZeroU32,
44 ) -> (TabSnapshot, Vec<TabEdit>) {
45 let old_snapshot = &mut self.0;
46 let mut new_snapshot = TabSnapshot {
47 fold_snapshot,
48 tab_size: tab_size.min(MAX_TABS),
49 max_expansion_column: old_snapshot.max_expansion_column,
50 version: old_snapshot.version,
51 };
52
53 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
54 new_snapshot.version += 1;
55 }
56
57 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
58 // Expand each edit to include the next tab on the same line as the edit,
59 // and any subsequent tabs on that line that moved across the tab expansion
60 // boundary.
61 for fold_edit in &mut fold_edits {
62 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
63 let old_end_row_successor_offset = cmp::min(
64 FoldPoint::new(old_end.row() + 1, 0),
65 old_snapshot.fold_snapshot.max_point(),
66 )
67 .to_offset(&old_snapshot.fold_snapshot);
68 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
69
70 let mut offset_from_edit = 0;
71 let mut first_tab_offset = None;
72 let mut last_tab_with_changed_expansion_offset = None;
73 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
74 fold_edit.old.end..old_end_row_successor_offset,
75 false,
76 Highlights::default(),
77 ) {
78 // todo(performance use tabs bitmask)
79 for (ix, _) in chunk.text.match_indices('\t') {
80 let offset_from_edit = offset_from_edit + (ix as u32);
81 if first_tab_offset.is_none() {
82 first_tab_offset = Some(offset_from_edit);
83 }
84
85 let old_column = old_end.column() + offset_from_edit;
86 let new_column = new_end.column() + offset_from_edit;
87 let was_expanded = old_column < old_snapshot.max_expansion_column;
88 let is_expanded = new_column < new_snapshot.max_expansion_column;
89 if was_expanded != is_expanded {
90 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
91 } else if !was_expanded && !is_expanded {
92 break 'outer;
93 }
94 }
95
96 offset_from_edit += chunk.text.len() as u32;
97 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
98 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
99 {
100 break;
101 }
102 }
103
104 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
105 fold_edit.old.end.0 += offset as usize + 1;
106 fold_edit.new.end.0 += offset as usize + 1;
107 }
108 }
109
110 let _old_alloc_ptr = fold_edits.as_ptr();
111 // Combine any edits that overlap due to the expansion.
112 let mut fold_edits = fold_edits.into_iter();
113 if let Some(mut first_edit) = fold_edits.next() {
114 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
115 #[allow(clippy::filter_map_identity)]
116 let mut v: Vec<_> = fold_edits
117 .scan(&mut first_edit, |state, edit| {
118 if state.old.end >= edit.old.start {
119 state.old.end = edit.old.end;
120 state.new.end = edit.new.end;
121 Some(None) // Skip this edit, it's merged
122 } else {
123 let new_state = edit;
124 let result = Some(Some(state.clone())); // Yield the previous edit
125 **state = new_state;
126 result
127 }
128 })
129 .filter_map(|x| x)
130 .collect();
131 v.push(first_edit);
132 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
133 v.into_iter()
134 .map(|fold_edit| {
135 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
136 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
137 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
138 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
139 TabEdit {
140 old: old_snapshot.fold_point_to_tab_point(old_start)
141 ..old_snapshot.fold_point_to_tab_point(old_end),
142 new: new_snapshot.fold_point_to_tab_point(new_start)
143 ..new_snapshot.fold_point_to_tab_point(new_end),
144 }
145 })
146 .collect()
147 } else {
148 vec![]
149 }
150 } else {
151 new_snapshot.version += 1;
152 vec![TabEdit {
153 old: TabPoint::zero()..old_snapshot.max_point(),
154 new: TabPoint::zero()..new_snapshot.max_point(),
155 }]
156 };
157 *old_snapshot = new_snapshot;
158 (old_snapshot.clone(), tab_edits)
159 }
160}
161
162#[derive(Clone)]
163pub struct TabSnapshot {
164 pub fold_snapshot: FoldSnapshot,
165 pub tab_size: NonZeroU32,
166 pub max_expansion_column: u32,
167 pub version: usize,
168}
169
170impl std::ops::Deref for TabSnapshot {
171 type Target = FoldSnapshot;
172
173 fn deref(&self) -> &Self::Target {
174 &self.fold_snapshot
175 }
176}
177
178impl TabSnapshot {
179 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
180 &self.fold_snapshot.inlay_snapshot.buffer
181 }
182
183 pub fn line_len(&self, row: u32) -> u32 {
184 let max_point = self.max_point();
185 if row < max_point.row() {
186 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
187 .0
188 .column
189 } else {
190 max_point.column()
191 }
192 }
193
194 pub fn text_summary(&self) -> TextSummary {
195 self.text_summary_for_range(TabPoint::zero()..self.max_point())
196 }
197
198 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
199 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
200 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
201 let input_summary = self
202 .fold_snapshot
203 .text_summary_for_range(input_start..input_end);
204
205 let line_end = if range.start.row() == range.end.row() {
206 range.end
207 } else {
208 self.max_point()
209 };
210 let first_line_chars = self
211 .chunks(range.start..line_end, false, Highlights::default())
212 .flat_map(|chunk| chunk.text.chars())
213 .take_while(|&c| c != '\n')
214 .count() as u32;
215
216 let last_line_chars = if range.start.row() == range.end.row() {
217 first_line_chars
218 } else {
219 self.chunks(
220 TabPoint::new(range.end.row(), 0)..range.end,
221 false,
222 Highlights::default(),
223 )
224 .flat_map(|chunk| chunk.text.chars())
225 .count() as u32
226 };
227
228 TextSummary {
229 lines: range.end.0 - range.start.0,
230 first_line_chars,
231 last_line_chars,
232 longest_row: input_summary.longest_row,
233 longest_row_chars: input_summary.longest_row_chars,
234 }
235 }
236
237 pub(crate) fn chunks<'a>(
238 &'a self,
239 range: Range<TabPoint>,
240 language_aware: bool,
241 highlights: Highlights<'a>,
242 ) -> TabChunks<'a> {
243 let (input_start, expanded_char_column, to_next_stop) =
244 self.tab_point_to_fold_point(range.start, Bias::Left);
245 let input_column = input_start.column();
246 let input_start = input_start.to_offset(&self.fold_snapshot);
247 let input_end = self
248 .tab_point_to_fold_point(range.end, Bias::Right)
249 .0
250 .to_offset(&self.fold_snapshot);
251 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
252 range.end.column() - range.start.column()
253 } else {
254 to_next_stop
255 };
256
257 TabChunks {
258 snapshot: self,
259 fold_chunks: self.fold_snapshot.chunks(
260 input_start..input_end,
261 language_aware,
262 highlights,
263 ),
264 input_column,
265 column: expanded_char_column,
266 max_expansion_column: self.max_expansion_column,
267 output_position: range.start.0,
268 max_output_position: range.end.0,
269 tab_size: self.tab_size,
270 chunk: Chunk {
271 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
272 is_tab: true,
273 ..Default::default()
274 },
275 inside_leading_tab: to_next_stop > 0,
276 }
277 }
278
279 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
280 self.fold_snapshot.row_infos(row)
281 }
282
283 #[cfg(test)]
284 pub fn text(&self) -> String {
285 self.chunks(
286 TabPoint::zero()..self.max_point(),
287 false,
288 Highlights::default(),
289 )
290 .map(|chunk| chunk.text)
291 .collect()
292 }
293
294 pub fn max_point(&self) -> TabPoint {
295 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
296 }
297
298 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
299 self.fold_point_to_tab_point(
300 self.fold_snapshot
301 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
302 )
303 }
304
305 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
306 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
307 let tab_cursor = TabStopCursor::new(chunks);
308 let expanded = self.expand_tabs(tab_cursor, input.column());
309 TabPoint::new(input.row(), expanded)
310 }
311
312 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
313 TabPointCursor { this: self }
314 }
315
316 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
317 let chunks = self
318 .fold_snapshot
319 .chunks_at(FoldPoint::new(output.row(), 0));
320
321 let tab_cursor = TabStopCursor::new(chunks);
322 let expanded = output.column();
323 let (collapsed, expanded_char_column, to_next_stop) =
324 self.collapse_tabs(tab_cursor, expanded, bias);
325
326 (
327 FoldPoint::new(output.row(), collapsed),
328 expanded_char_column,
329 to_next_stop,
330 )
331 }
332
333 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
334 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
335 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
336 self.fold_point_to_tab_point(fold_point)
337 }
338
339 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
340 let fold_point = self.tab_point_to_fold_point(point, bias).0;
341 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
342 self.fold_snapshot
343 .inlay_snapshot
344 .to_buffer_point(inlay_point)
345 }
346
347 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
348 where
349 I: Iterator<Item = Chunk<'a>>,
350 {
351 let tab_size = self.tab_size.get();
352
353 let end_column = column.min(self.max_expansion_column);
354 let mut seek_target = end_column;
355 let mut tab_count = 0;
356 let mut expanded_tab_len = 0;
357
358 while let Some(tab_stop) = cursor.seek(seek_target) {
359 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
360 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
361 tab_count += 1;
362 expanded_tab_len += tab_len;
363
364 seek_target = end_column - cursor.byte_offset;
365 }
366
367 let left_over_char_bytes = if !cursor.is_char_boundary() {
368 cursor.bytes_until_next_char().unwrap_or(0) as u32
369 } else {
370 0
371 };
372
373 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
374 let expanded_bytes =
375 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
376
377 expanded_bytes + column.saturating_sub(collapsed_bytes)
378 }
379
380 fn collapse_tabs<'a, I>(
381 &self,
382 mut cursor: TabStopCursor<'a, I>,
383 column: u32,
384 bias: Bias,
385 ) -> (u32, u32, u32)
386 where
387 I: Iterator<Item = Chunk<'a>>,
388 {
389 let tab_size = self.tab_size.get();
390 let mut collapsed_column = column;
391 let mut seek_target = column.min(self.max_expansion_column);
392 let mut tab_count = 0;
393 let mut expanded_tab_len = 0;
394
395 while let Some(tab_stop) = cursor.seek(seek_target) {
396 // Calculate how much we want to expand this tab stop (into spaces)
397 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
398 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
399 // Increment tab count
400 tab_count += 1;
401 // The count of how many spaces we've added to this line in place of tab bytes
402 expanded_tab_len += tab_len;
403
404 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
405 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
406
407 // Did we expand past the search target?
408 if expanded_bytes > column {
409 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
410 // We expanded past the search target, so need to account for the offshoot
411 expanded_chars -= expanded_bytes - column;
412 return match bias {
413 Bias::Left => (
414 cursor.byte_offset() - 1,
415 expanded_chars,
416 expanded_bytes - column,
417 ),
418 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
419 };
420 } else {
421 // otherwise we only want to move the cursor collapse column forward
422 collapsed_column = collapsed_column - tab_len + 1;
423 seek_target = (collapsed_column - cursor.byte_offset)
424 .min(self.max_expansion_column - cursor.byte_offset);
425 }
426 }
427
428 let collapsed_bytes = cursor.byte_offset();
429 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
430 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
431 (
432 collapsed_bytes + column.saturating_sub(expanded_bytes),
433 expanded_chars,
434 0,
435 )
436 }
437}
438
439// todo(lw): Implement TabPointCursor properly
440pub struct TabPointCursor<'this> {
441 this: &'this TabSnapshot,
442}
443
444impl TabPointCursor<'_> {
445 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
446 self.this.fold_point_to_tab_point(point)
447 }
448}
449
450#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
451pub struct TabPoint(pub Point);
452
453impl TabPoint {
454 pub fn new(row: u32, column: u32) -> Self {
455 Self(Point::new(row, column))
456 }
457
458 pub fn zero() -> Self {
459 Self::new(0, 0)
460 }
461
462 pub fn row(self) -> u32 {
463 self.0.row
464 }
465
466 pub fn column(self) -> u32 {
467 self.0.column
468 }
469}
470
471impl From<Point> for TabPoint {
472 fn from(point: Point) -> Self {
473 Self(point)
474 }
475}
476
477pub type TabEdit = text::Edit<TabPoint>;
478
479#[derive(Clone, Debug, Default, Eq, PartialEq)]
480pub struct TextSummary {
481 pub lines: Point,
482 pub first_line_chars: u32,
483 pub last_line_chars: u32,
484 pub longest_row: u32,
485 pub longest_row_chars: u32,
486}
487
488impl<'a> From<&'a str> for TextSummary {
489 fn from(text: &'a str) -> Self {
490 let sum = text::TextSummary::from(text);
491
492 TextSummary {
493 lines: sum.lines,
494 first_line_chars: sum.first_line_chars,
495 last_line_chars: sum.last_line_chars,
496 longest_row: sum.longest_row,
497 longest_row_chars: sum.longest_row_chars,
498 }
499 }
500}
501
502impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
503 fn add_assign(&mut self, other: &'a Self) {
504 let joined_chars = self.last_line_chars + other.first_line_chars;
505 if joined_chars > self.longest_row_chars {
506 self.longest_row = self.lines.row;
507 self.longest_row_chars = joined_chars;
508 }
509 if other.longest_row_chars > self.longest_row_chars {
510 self.longest_row = self.lines.row + other.longest_row;
511 self.longest_row_chars = other.longest_row_chars;
512 }
513
514 if self.lines.row == 0 {
515 self.first_line_chars += other.first_line_chars;
516 }
517
518 if other.lines.row == 0 {
519 self.last_line_chars += other.first_line_chars;
520 } else {
521 self.last_line_chars = other.last_line_chars;
522 }
523
524 self.lines += &other.lines;
525 }
526}
527
528pub struct TabChunks<'a> {
529 snapshot: &'a TabSnapshot,
530 max_expansion_column: u32,
531 max_output_position: Point,
532 tab_size: NonZeroU32,
533 // region: iteration state
534 fold_chunks: FoldChunks<'a>,
535 chunk: Chunk<'a>,
536 column: u32,
537 output_position: Point,
538 input_column: u32,
539 inside_leading_tab: bool,
540 // endregion: iteration state
541}
542
543impl TabChunks<'_> {
544 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
545 let (input_start, expanded_char_column, to_next_stop) = self
546 .snapshot
547 .tab_point_to_fold_point(range.start, Bias::Left);
548 let input_column = input_start.column();
549 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
550 let input_end = self
551 .snapshot
552 .tab_point_to_fold_point(range.end, Bias::Right)
553 .0
554 .to_offset(&self.snapshot.fold_snapshot);
555 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
556 range.end.column() - range.start.column()
557 } else {
558 to_next_stop
559 };
560
561 self.fold_chunks.seek(input_start..input_end);
562 self.input_column = input_column;
563 self.column = expanded_char_column;
564 self.output_position = range.start.0;
565 self.max_output_position = range.end.0;
566 self.chunk = Chunk {
567 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
568 is_tab: true,
569 chars: 1u128.unbounded_shl(to_next_stop) - 1,
570 ..Default::default()
571 };
572 self.inside_leading_tab = to_next_stop > 0;
573 }
574}
575
576impl<'a> Iterator for TabChunks<'a> {
577 type Item = Chunk<'a>;
578
579 fn next(&mut self) -> Option<Self::Item> {
580 if self.chunk.text.is_empty() {
581 if let Some(chunk) = self.fold_chunks.next() {
582 self.chunk = chunk;
583 if self.inside_leading_tab {
584 self.chunk.text = &self.chunk.text[1..];
585 self.inside_leading_tab = false;
586 self.input_column += 1;
587 }
588 } else {
589 return None;
590 }
591 }
592
593 //todo(improve performance by using tab cursor)
594 for (ix, c) in self.chunk.text.char_indices() {
595 match c {
596 '\t' if ix > 0 => {
597 let (prefix, suffix) = self.chunk.text.split_at(ix);
598
599 let mask = 1u128.unbounded_shl(ix as u32).wrapping_sub(1);
600 let chars = self.chunk.chars & mask;
601 let tabs = self.chunk.tabs & mask;
602 self.chunk.tabs = self.chunk.tabs.unbounded_shr(ix as u32);
603 self.chunk.chars = self.chunk.chars.unbounded_shr(ix as u32);
604 self.chunk.text = suffix;
605 return Some(Chunk {
606 text: prefix,
607 chars,
608 tabs,
609 ..self.chunk.clone()
610 });
611 }
612 '\t' => {
613 self.chunk.text = &self.chunk.text[1..];
614 self.chunk.tabs >>= 1;
615 self.chunk.chars >>= 1;
616 let tab_size = if self.input_column < self.max_expansion_column {
617 self.tab_size.get()
618 } else {
619 1
620 };
621 let mut len = tab_size - self.column % tab_size;
622 let next_output_position = cmp::min(
623 self.output_position + Point::new(0, len),
624 self.max_output_position,
625 );
626 len = next_output_position.column - self.output_position.column;
627 self.column += len;
628 self.input_column += 1;
629 self.output_position = next_output_position;
630 return Some(Chunk {
631 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
632 is_tab: true,
633 chars: 1u128.unbounded_shl(len) - 1,
634 tabs: 0,
635 ..self.chunk.clone()
636 });
637 }
638 '\n' => {
639 self.column = 0;
640 self.input_column = 0;
641 self.output_position += Point::new(1, 0);
642 }
643 _ => {
644 self.column += 1;
645 if !self.inside_leading_tab {
646 self.input_column += c.len_utf8() as u32;
647 }
648 self.output_position.column += c.len_utf8() as u32;
649 }
650 }
651 }
652
653 Some(mem::take(&mut self.chunk))
654 }
655}
656
657#[cfg(test)]
658mod tests {
659 use super::*;
660 use crate::{
661 MultiBuffer,
662 display_map::{
663 fold_map::{FoldMap, FoldOffset},
664 inlay_map::InlayMap,
665 },
666 };
667 use multi_buffer::MultiBufferOffset;
668 use rand::{Rng, prelude::StdRng};
669 use util;
670
671 impl TabSnapshot {
672 fn expected_collapse_tabs(
673 &self,
674 chars: impl Iterator<Item = char>,
675 column: u32,
676 bias: Bias,
677 ) -> (u32, u32, u32) {
678 let tab_size = self.tab_size.get();
679
680 let mut expanded_bytes = 0;
681 let mut expanded_chars = 0;
682 let mut collapsed_bytes = 0;
683 for c in chars {
684 if expanded_bytes >= column {
685 break;
686 }
687 if collapsed_bytes >= self.max_expansion_column {
688 break;
689 }
690
691 if c == '\t' {
692 let tab_len = tab_size - (expanded_chars % tab_size);
693 expanded_chars += tab_len;
694 expanded_bytes += tab_len;
695 if expanded_bytes > column {
696 expanded_chars -= expanded_bytes - column;
697 return match bias {
698 Bias::Left => {
699 (collapsed_bytes, expanded_chars, expanded_bytes - column)
700 }
701 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
702 };
703 }
704 } else {
705 expanded_chars += 1;
706 expanded_bytes += c.len_utf8() as u32;
707 }
708
709 if expanded_bytes > column && matches!(bias, Bias::Left) {
710 expanded_chars -= 1;
711 break;
712 }
713
714 collapsed_bytes += c.len_utf8() as u32;
715 }
716
717 (
718 collapsed_bytes + column.saturating_sub(expanded_bytes),
719 expanded_chars,
720 0,
721 )
722 }
723
724 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
725 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
726 let expanded = self.expected_expand_tabs(chars, input.column());
727 TabPoint::new(input.row(), expanded)
728 }
729
730 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
731 let tab_size = self.tab_size.get();
732
733 let mut expanded_chars = 0;
734 let mut expanded_bytes = 0;
735 let mut collapsed_bytes = 0;
736 let end_column = column.min(self.max_expansion_column);
737 for c in chars {
738 if collapsed_bytes >= end_column {
739 break;
740 }
741 if c == '\t' {
742 let tab_len = tab_size - expanded_chars % tab_size;
743 expanded_bytes += tab_len;
744 expanded_chars += tab_len;
745 } else {
746 expanded_bytes += c.len_utf8() as u32;
747 expanded_chars += 1;
748 }
749 collapsed_bytes += c.len_utf8() as u32;
750 }
751
752 expanded_bytes + column.saturating_sub(collapsed_bytes)
753 }
754
755 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
756 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
757 let expanded = output.column();
758 let (collapsed, expanded_char_column, to_next_stop) =
759 self.expected_collapse_tabs(chars, expanded, bias);
760 (
761 FoldPoint::new(output.row(), collapsed),
762 expanded_char_column,
763 to_next_stop,
764 )
765 }
766 }
767
768 #[gpui::test]
769 fn test_expand_tabs(cx: &mut gpui::App) {
770 let test_values = [
771 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
772 (" \twςe", 4),
773 ("fε", 1),
774 ("i❎\t", 3),
775 ];
776 let buffer = MultiBuffer::build_simple("", cx);
777 let buffer_snapshot = buffer.read(cx).snapshot(cx);
778 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
779 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
780 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
781
782 for (text, column) in test_values {
783 let mut tabs = 0u128;
784 let mut chars = 0u128;
785 for (idx, c) in text.char_indices() {
786 if c == '\t' {
787 tabs |= 1 << idx;
788 }
789 chars |= 1 << idx;
790 }
791
792 let chunks = [Chunk {
793 text,
794 tabs,
795 chars,
796 ..Default::default()
797 }];
798
799 let cursor = TabStopCursor::new(chunks);
800
801 assert_eq!(
802 tab_snapshot.expected_expand_tabs(text.chars(), column),
803 tab_snapshot.expand_tabs(cursor, column)
804 );
805 }
806 }
807
808 #[gpui::test]
809 fn test_collapse_tabs(cx: &mut gpui::App) {
810 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
811
812 let buffer = MultiBuffer::build_simple(input, cx);
813 let buffer_snapshot = buffer.read(cx).snapshot(cx);
814 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
815 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
816 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
817
818 for (ix, _) in input.char_indices() {
819 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
820
821 assert_eq!(
822 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
823 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
824 "Failed with tab_point at column {ix}"
825 );
826 assert_eq!(
827 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
828 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
829 "Failed with tab_point at column {ix}"
830 );
831
832 assert_eq!(
833 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
834 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
835 "Failed with tab_point at column {ix}"
836 );
837 assert_eq!(
838 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
839 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
840 "Failed with tab_point at column {ix}"
841 );
842 }
843 }
844
845 #[gpui::test]
846 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
847 // This test reproduces a specific panic where to_fold_point returns incorrect results
848 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
849
850 let text = "γ\tw⭐\n🍐🍗 \t";
851 let buffer = MultiBuffer::build_simple(text, cx);
852 let buffer_snapshot = buffer.read(cx).snapshot(cx);
853 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
854 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
855 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
856
857 // This should panic with the expected vs actual mismatch
858 let tab_point = TabPoint::new(0, 9);
859 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
860 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
861
862 assert_eq!(result, expected);
863 }
864
865 #[gpui::test(iterations = 100)]
866 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
867 // Generate random input string with up to 200 characters including tabs
868 // to stay within the MAX_EXPANSION_COLUMN limit of 256
869 let len = rng.random_range(0..=2048);
870 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
871 let mut input = String::with_capacity(len);
872
873 for _ in 0..len {
874 if rng.random_bool(0.1) {
875 // 10% chance of inserting a tab
876 input.push('\t');
877 } else {
878 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
879 let ch = loop {
880 let ascii_code = rng.random_range(32..=126); // printable ASCII range
881 let ch = ascii_code as u8 as char;
882 if ch != '\t' {
883 break ch;
884 }
885 };
886 input.push(ch);
887 }
888 }
889
890 let buffer = MultiBuffer::build_simple(&input, cx);
891 let buffer_snapshot = buffer.read(cx).snapshot(cx);
892 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
893 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
894 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
895 tab_snapshot.max_expansion_column = rng.random_range(0..323);
896 tab_snapshot.tab_size = tab_size;
897
898 for (ix, _) in input.char_indices() {
899 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
900
901 assert_eq!(
902 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
903 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
904 "Failed with input: {}, with idx: {ix}",
905 input
906 );
907 assert_eq!(
908 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
909 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
910 "Failed with input: {}, with idx: {ix}",
911 input
912 );
913
914 assert_eq!(
915 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
916 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
917 "Failed with input: {}, with idx: {ix}",
918 input
919 );
920 assert_eq!(
921 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
922 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
923 "Failed with input: {}, with idx: {ix}",
924 input
925 );
926 }
927 }
928
929 #[gpui::test]
930 fn test_long_lines(cx: &mut gpui::App) {
931 let max_expansion_column = 12;
932 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
933 let output = "A BC DEF G HI J K L M";
934
935 let buffer = MultiBuffer::build_simple(input, cx);
936 let buffer_snapshot = buffer.read(cx).snapshot(cx);
937 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
938 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
939 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
940
941 tab_snapshot.max_expansion_column = max_expansion_column;
942 assert_eq!(tab_snapshot.text(), output);
943
944 for (ix, c) in input.char_indices() {
945 assert_eq!(
946 tab_snapshot
947 .chunks(
948 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
949 false,
950 Highlights::default(),
951 )
952 .map(|c| c.text)
953 .collect::<String>(),
954 &output[ix..],
955 "text from index {ix}"
956 );
957
958 if c != '\t' {
959 let input_point = Point::new(0, ix as u32);
960 let output_point = Point::new(0, output.find(c).unwrap() as u32);
961 assert_eq!(
962 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
963 TabPoint(output_point),
964 "to_tab_point({input_point:?})"
965 );
966 assert_eq!(
967 tab_snapshot
968 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
969 .0,
970 FoldPoint(input_point),
971 "to_fold_point({output_point:?})"
972 );
973 }
974 }
975 }
976
977 #[gpui::test]
978 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
979 let max_expansion_column = 8;
980 let input = "abcdefg⋯hij";
981
982 let buffer = MultiBuffer::build_simple(input, cx);
983 let buffer_snapshot = buffer.read(cx).snapshot(cx);
984 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
985 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
986 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
987
988 tab_snapshot.max_expansion_column = max_expansion_column;
989 assert_eq!(tab_snapshot.text(), input);
990 }
991
992 #[gpui::test]
993 fn test_marking_tabs(cx: &mut gpui::App) {
994 let input = "\t \thello";
995
996 let buffer = MultiBuffer::build_simple(input, cx);
997 let buffer_snapshot = buffer.read(cx).snapshot(cx);
998 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
999 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1000 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1001
1002 assert_eq!(
1003 chunks(&tab_snapshot, TabPoint::zero()),
1004 vec![
1005 (" ".to_string(), true),
1006 (" ".to_string(), false),
1007 (" ".to_string(), true),
1008 ("hello".to_string(), false),
1009 ]
1010 );
1011 assert_eq!(
1012 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1013 vec![
1014 (" ".to_string(), true),
1015 (" ".to_string(), false),
1016 (" ".to_string(), true),
1017 ("hello".to_string(), false),
1018 ]
1019 );
1020
1021 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1022 let mut chunks = Vec::new();
1023 let mut was_tab = false;
1024 let mut text = String::new();
1025 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1026 {
1027 if chunk.is_tab != was_tab {
1028 if !text.is_empty() {
1029 chunks.push((mem::take(&mut text), was_tab));
1030 }
1031 was_tab = chunk.is_tab;
1032 }
1033 text.push_str(chunk.text);
1034 }
1035
1036 if !text.is_empty() {
1037 chunks.push((text, was_tab));
1038 }
1039 chunks
1040 }
1041 }
1042
1043 #[gpui::test(iterations = 100)]
1044 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1045 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1046 let len = rng.random_range(0..30);
1047 let buffer = if rng.random() {
1048 let text = util::RandomCharIter::new(&mut rng)
1049 .take(len)
1050 .collect::<String>();
1051 MultiBuffer::build_simple(&text, cx)
1052 } else {
1053 MultiBuffer::build_random(&mut rng, cx)
1054 };
1055 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1056 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1057
1058 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1059 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1060 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1061 fold_map.randomly_mutate(&mut rng);
1062 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1063 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1064 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1065 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1066
1067 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1068 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1069
1070 let text = text::Rope::from(tabs_snapshot.text().as_str());
1071 log::info!(
1072 "TabMap text (tab size: {}): {:?}",
1073 tab_size,
1074 tabs_snapshot.text(),
1075 );
1076
1077 for _ in 0..5 {
1078 let end_row = rng.random_range(0..=text.max_point().row);
1079 let end_column = rng.random_range(0..=text.line_len(end_row));
1080 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1081 let start_row = rng.random_range(0..=text.max_point().row);
1082 let start_column = rng.random_range(0..=text.line_len(start_row));
1083 let mut start =
1084 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1085 if start > end {
1086 mem::swap(&mut start, &mut end);
1087 }
1088
1089 let expected_text = text
1090 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1091 .collect::<String>();
1092 let expected_summary = TextSummary::from(expected_text.as_str());
1093 assert_eq!(
1094 tabs_snapshot
1095 .chunks(start..end, false, Highlights::default())
1096 .map(|c| c.text)
1097 .collect::<String>(),
1098 expected_text,
1099 "chunks({:?}..{:?})",
1100 start,
1101 end
1102 );
1103
1104 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1105 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1106 actual_summary.longest_row = expected_summary.longest_row;
1107 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1108 }
1109 assert_eq!(actual_summary, expected_summary);
1110 }
1111
1112 for row in 0..=text.max_point().row {
1113 assert_eq!(
1114 tabs_snapshot.line_len(row),
1115 text.line_len(row),
1116 "line_len({row})"
1117 );
1118 }
1119 }
1120
1121 #[gpui::test(iterations = 100)]
1122 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1123 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1124 let len = rng.random_range(0..=2000);
1125
1126 // Generate random text using RandomCharIter
1127 let text = util::RandomCharIter::new(&mut rng)
1128 .take(len)
1129 .collect::<String>();
1130
1131 // Create buffer and tab map
1132 let buffer = MultiBuffer::build_simple(&text, cx);
1133 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1134 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1135 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1136 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1137
1138 let mut next_inlay_id = 0;
1139 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1140 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1141 let max_fold_point = fold_snapshot.max_point();
1142 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1143
1144 // Test random fold points
1145 for _ in 0..50 {
1146 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1147 // Generate random fold point
1148 let row = rng.random_range(0..=max_fold_point.row());
1149 let max_column = if row < max_fold_point.row() {
1150 fold_snapshot.line_len(row)
1151 } else {
1152 max_fold_point.column()
1153 };
1154 let column = rng.random_range(0..=max_column + 10);
1155 let fold_point = FoldPoint::new(row, column);
1156
1157 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1158 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1159
1160 assert_eq!(
1161 actual, expected,
1162 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1163 fold_point, text
1164 );
1165 }
1166 }
1167
1168 #[gpui::test]
1169 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1170 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1171 let buffer = MultiBuffer::build_simple(text, cx);
1172 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1173 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1174 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1175 let chunks = fold_snapshot.chunks(
1176 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1177 false,
1178 Default::default(),
1179 );
1180 let mut cursor = TabStopCursor::new(chunks);
1181 assert!(cursor.seek(0).is_none());
1182 let mut tab_stops = Vec::new();
1183
1184 let mut all_tab_stops = Vec::new();
1185 let mut byte_offset = 0;
1186 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1187 byte_offset += ch.len_utf8() as u32;
1188
1189 if ch == '\t' {
1190 all_tab_stops.push(TabStop {
1191 byte_offset,
1192 char_offset: offset as u32 + 1,
1193 });
1194 }
1195 }
1196
1197 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1198 tab_stops.push(tab_stop);
1199 }
1200 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1201
1202 assert_eq!(cursor.byte_offset(), byte_offset);
1203 }
1204
1205 #[gpui::test]
1206 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1207 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1208
1209 let buffer = MultiBuffer::build_simple(input, cx);
1210 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1211 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1212 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1213
1214 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1215 let mut cursor = TabStopCursor::new(chunks);
1216
1217 let mut actual_tab_stops = Vec::new();
1218
1219 let mut expected_tab_stops = Vec::new();
1220 let mut byte_offset = 0;
1221 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1222 byte_offset += ch.len_utf8() as u32;
1223
1224 if ch == '\t' {
1225 expected_tab_stops.push(TabStop {
1226 byte_offset,
1227 char_offset: offset as u32 + 1,
1228 });
1229 }
1230 }
1231
1232 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1233 actual_tab_stops.push(tab_stop);
1234 }
1235 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1236
1237 assert_eq!(cursor.byte_offset(), byte_offset);
1238 }
1239
1240 #[gpui::test(iterations = 100)]
1241 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1242 // Generate random input string with up to 512 characters including tabs
1243 let len = rng.random_range(0..=2048);
1244 let mut input = String::with_capacity(len);
1245
1246 let mut skip_tabs = rng.random_bool(0.10);
1247 for idx in 0..len {
1248 if idx % 128 == 0 {
1249 skip_tabs = rng.random_bool(0.10);
1250 }
1251
1252 if rng.random_bool(0.15) && !skip_tabs {
1253 input.push('\t');
1254 } else {
1255 let ch = loop {
1256 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1257 let ch = ascii_code as u8 as char;
1258 if ch != '\t' {
1259 break ch;
1260 }
1261 };
1262 input.push(ch);
1263 }
1264 }
1265
1266 // Build the buffer and create cursor
1267 let buffer = MultiBuffer::build_simple(&input, cx);
1268 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1269 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1270 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1271
1272 // First, collect all expected tab positions
1273 let mut all_tab_stops = Vec::new();
1274 let mut byte_offset = 1;
1275 let mut char_offset = 1;
1276 for ch in buffer_snapshot.text().chars() {
1277 if ch == '\t' {
1278 all_tab_stops.push(TabStop {
1279 byte_offset,
1280 char_offset,
1281 });
1282 }
1283 byte_offset += ch.len_utf8() as u32;
1284 char_offset += 1;
1285 }
1286
1287 // Test with various distances
1288 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1289 // let distances = vec![150];
1290
1291 for distance in distances {
1292 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1293 let mut cursor = TabStopCursor::new(chunks);
1294
1295 let mut found_tab_stops = Vec::new();
1296 let mut position = distance;
1297 while let Some(tab_stop) = cursor.seek(position) {
1298 found_tab_stops.push(tab_stop);
1299 position = distance - tab_stop.byte_offset;
1300 }
1301
1302 let expected_found_tab_stops: Vec<_> = all_tab_stops
1303 .iter()
1304 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1305 .cloned()
1306 .collect();
1307
1308 pretty_assertions::assert_eq!(
1309 found_tab_stops,
1310 expected_found_tab_stops,
1311 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1312 distance,
1313 input
1314 );
1315
1316 let final_position = cursor.byte_offset();
1317 if !found_tab_stops.is_empty() {
1318 let last_tab_stop = found_tab_stops.last().unwrap();
1319 assert!(
1320 final_position >= last_tab_stop.byte_offset,
1321 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1322 final_position,
1323 last_tab_stop.byte_offset,
1324 input
1325 );
1326 }
1327 }
1328 }
1329
1330 #[gpui::test]
1331 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1332 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1333 let buffer = MultiBuffer::build_simple(text, cx);
1334 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1335 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1336 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1337 let chunks = fold_snapshot.chunks(
1338 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1339 false,
1340 Default::default(),
1341 );
1342 let mut cursor = TabStopCursor::new(chunks);
1343 assert!(cursor.seek(0).is_none());
1344
1345 let mut expected_tab_stops = Vec::new();
1346 let mut byte_offset = 0;
1347 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1348 byte_offset += ch.len_utf8() as u32;
1349
1350 if ch == '\t' {
1351 expected_tab_stops.push(TabStop {
1352 byte_offset,
1353 char_offset: i as u32 + 1,
1354 });
1355 }
1356 }
1357
1358 let mut actual_tab_stops = Vec::new();
1359 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1360 actual_tab_stops.push(tab_stop);
1361 }
1362
1363 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1364
1365 assert_eq!(cursor.byte_offset(), byte_offset);
1366 }
1367
1368 #[gpui::test(iterations = 100)]
1369 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1370 // Generate random input string with up to 512 characters including tabs
1371 let len = rng.random_range(0..=2048);
1372 let input = util::RandomCharIter::new(&mut rng)
1373 .take(len)
1374 .collect::<String>();
1375
1376 // Build the buffer and create cursor
1377 let buffer = MultiBuffer::build_simple(&input, cx);
1378 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1379 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1380 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1381
1382 // First, collect all expected tab positions
1383 let mut all_tab_stops = Vec::new();
1384 let mut byte_offset = 0;
1385 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1386 byte_offset += ch.len_utf8() as u32;
1387 if ch == '\t' {
1388 all_tab_stops.push(TabStop {
1389 byte_offset,
1390 char_offset: i as u32 + 1,
1391 });
1392 }
1393 }
1394
1395 // Test with various distances
1396 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1397 let distances = vec![150];
1398
1399 for distance in distances {
1400 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1401 let mut cursor = TabStopCursor::new(chunks);
1402
1403 let mut found_tab_stops = Vec::new();
1404 let mut position = distance;
1405 while let Some(tab_stop) = cursor.seek(position) {
1406 found_tab_stops.push(tab_stop);
1407 position = distance - tab_stop.byte_offset;
1408 }
1409
1410 let expected_found_tab_stops: Vec<_> = all_tab_stops
1411 .iter()
1412 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1413 .cloned()
1414 .collect();
1415
1416 pretty_assertions::assert_eq!(
1417 found_tab_stops,
1418 expected_found_tab_stops,
1419 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1420 distance,
1421 input
1422 );
1423
1424 let final_position = cursor.byte_offset();
1425 if !found_tab_stops.is_empty() {
1426 let last_tab_stop = found_tab_stops.last().unwrap();
1427 assert!(
1428 final_position >= last_tab_stop.byte_offset,
1429 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1430 final_position,
1431 last_tab_stop.byte_offset,
1432 input
1433 );
1434 }
1435 }
1436 }
1437}
1438
1439struct TabStopCursor<'a, I>
1440where
1441 I: Iterator<Item = Chunk<'a>>,
1442{
1443 chunks: I,
1444 byte_offset: u32,
1445 char_offset: u32,
1446 /// Chunk
1447 /// last tab position iterated through
1448 current_chunk: Option<(Chunk<'a>, u32)>,
1449}
1450
1451impl<'a, I> TabStopCursor<'a, I>
1452where
1453 I: Iterator<Item = Chunk<'a>>,
1454{
1455 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1456 Self {
1457 chunks: chunks.into_iter(),
1458 byte_offset: 0,
1459 char_offset: 0,
1460 current_chunk: None,
1461 }
1462 }
1463
1464 fn bytes_until_next_char(&self) -> Option<usize> {
1465 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1466 let mut idx = *idx;
1467 let mut diff = 0;
1468 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1469 idx -= 1;
1470 diff += 1;
1471 }
1472
1473 if chunk.chars & (1 << idx) != 0 {
1474 Some(
1475 (chunk.text[idx as usize..].chars().next()?)
1476 .len_utf8()
1477 .saturating_sub(diff),
1478 )
1479 } else {
1480 None
1481 }
1482 })
1483 }
1484
1485 fn is_char_boundary(&self) -> bool {
1486 self.current_chunk
1487 .as_ref()
1488 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1489 }
1490
1491 /// distance: length to move forward while searching for the next tab stop
1492 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1493 if distance == 0 {
1494 return None;
1495 }
1496
1497 let mut distance_traversed = 0;
1498
1499 while let Some((mut chunk, chunk_position)) = self
1500 .current_chunk
1501 .take()
1502 .or_else(|| self.chunks.next().zip(Some(0)))
1503 {
1504 if chunk.tabs == 0 {
1505 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1506 if chunk_distance + distance_traversed >= distance {
1507 let overshoot = distance_traversed.abs_diff(distance);
1508
1509 self.byte_offset += overshoot;
1510 self.char_offset += get_char_offset(
1511 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1512 chunk.chars,
1513 );
1514
1515 if chunk_position + overshoot < 128 {
1516 self.current_chunk = Some((chunk, chunk_position + overshoot));
1517 }
1518
1519 return None;
1520 }
1521
1522 self.byte_offset += chunk_distance;
1523 self.char_offset += get_char_offset(
1524 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1525 chunk.chars,
1526 );
1527 distance_traversed += chunk_distance;
1528 continue;
1529 }
1530 let tab_position = chunk.tabs.trailing_zeros() + 1;
1531
1532 if distance_traversed + tab_position - chunk_position > distance {
1533 let cursor_position = distance_traversed.abs_diff(distance);
1534
1535 self.char_offset += get_char_offset(
1536 chunk_position..(chunk_position + cursor_position - 1),
1537 chunk.chars,
1538 );
1539 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1540 self.byte_offset += cursor_position;
1541
1542 return None;
1543 }
1544
1545 self.byte_offset += tab_position - chunk_position;
1546 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1547
1548 let tabstop = TabStop {
1549 char_offset: self.char_offset,
1550 byte_offset: self.byte_offset,
1551 };
1552
1553 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1554
1555 if tab_position as usize != chunk.text.len() {
1556 self.current_chunk = Some((chunk, tab_position));
1557 }
1558
1559 return Some(tabstop);
1560 }
1561
1562 None
1563 }
1564
1565 fn byte_offset(&self) -> u32 {
1566 self.byte_offset
1567 }
1568
1569 fn char_offset(&self) -> u32 {
1570 self.char_offset
1571 }
1572}
1573
1574#[inline(always)]
1575fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1576 if range.start == range.end {
1577 return if (1u128 << range.start) & bit_map == 0 {
1578 0
1579 } else {
1580 1
1581 };
1582 }
1583 let end_shift: u128 = 127u128 - range.end as u128;
1584 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1585 bit_mask = (bit_mask << end_shift) >> end_shift;
1586 let bit_map = bit_map & bit_mask;
1587
1588 bit_map.count_ones()
1589}
1590
1591#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1592struct TabStop {
1593 char_offset: u32,
1594 byte_offset: u32,
1595}