1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 #[ztracing::instrument(skip_all)]
24 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
25 let snapshot = TabSnapshot {
26 fold_snapshot,
27 tab_size: tab_size.min(MAX_TABS),
28 max_expansion_column: MAX_EXPANSION_COLUMN,
29 version: 0,
30 };
31 (Self(snapshot.clone()), snapshot)
32 }
33
34 #[cfg(test)]
35 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
36 self.0.max_expansion_column = column;
37 self.0.clone()
38 }
39
40 #[ztracing::instrument(skip_all)]
41 pub fn sync(
42 &mut self,
43 fold_snapshot: FoldSnapshot,
44 mut fold_edits: Vec<FoldEdit>,
45 tab_size: NonZeroU32,
46 ) -> (TabSnapshot, Vec<TabEdit>) {
47 let old_snapshot = &mut self.0;
48 let mut new_snapshot = TabSnapshot {
49 fold_snapshot,
50 tab_size: tab_size.min(MAX_TABS),
51 max_expansion_column: old_snapshot.max_expansion_column,
52 version: old_snapshot.version,
53 };
54
55 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
56 new_snapshot.version += 1;
57 }
58
59 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
60 // Expand each edit to include the next tab on the same line as the edit,
61 // and any subsequent tabs on that line that moved across the tab expansion
62 // boundary.
63 for fold_edit in &mut fold_edits {
64 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
65 let old_end_row_successor_offset = cmp::min(
66 FoldPoint::new(old_end.row() + 1, 0),
67 old_snapshot.fold_snapshot.max_point(),
68 )
69 .to_offset(&old_snapshot.fold_snapshot);
70 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
71
72 let mut offset_from_edit = 0;
73 let mut first_tab_offset = None;
74 let mut last_tab_with_changed_expansion_offset = None;
75 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
76 fold_edit.old.end..old_end_row_successor_offset,
77 false,
78 Highlights::default(),
79 ) {
80 let mut remaining_tabs = chunk.tabs;
81 while remaining_tabs != 0 {
82 let ix = remaining_tabs.trailing_zeros();
83 let offset_from_edit = offset_from_edit + ix;
84 if first_tab_offset.is_none() {
85 first_tab_offset = Some(offset_from_edit);
86 }
87
88 let old_column = old_end.column() + offset_from_edit;
89 let new_column = new_end.column() + offset_from_edit;
90 let was_expanded = old_column < old_snapshot.max_expansion_column;
91 let is_expanded = new_column < new_snapshot.max_expansion_column;
92 if was_expanded != is_expanded {
93 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
94 } else if !was_expanded && !is_expanded {
95 break 'outer;
96 }
97
98 remaining_tabs &= remaining_tabs - 1;
99 }
100
101 offset_from_edit += chunk.text.len() as u32;
102 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
103 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
104 {
105 break;
106 }
107 }
108
109 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
110 fold_edit.old.end.0 += offset as usize + 1;
111 fold_edit.new.end.0 += offset as usize + 1;
112 }
113 }
114
115 let _old_alloc_ptr = fold_edits.as_ptr();
116 // Combine any edits that overlap due to the expansion.
117 let mut fold_edits = fold_edits.into_iter();
118 if let Some(mut first_edit) = fold_edits.next() {
119 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
120 #[allow(clippy::filter_map_identity)]
121 let mut v: Vec<_> = fold_edits
122 .scan(&mut first_edit, |state, edit| {
123 if state.old.end >= edit.old.start {
124 state.old.end = edit.old.end;
125 state.new.end = edit.new.end;
126 Some(None) // Skip this edit, it's merged
127 } else {
128 let new_state = edit;
129 let result = Some(Some(state.clone())); // Yield the previous edit
130 **state = new_state;
131 result
132 }
133 })
134 .filter_map(|x| x)
135 .collect();
136 v.push(first_edit);
137 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
138 v.into_iter()
139 .map(|fold_edit| {
140 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
141 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
142 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
143 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
144 TabEdit {
145 old: old_snapshot.fold_point_to_tab_point(old_start)
146 ..old_snapshot.fold_point_to_tab_point(old_end),
147 new: new_snapshot.fold_point_to_tab_point(new_start)
148 ..new_snapshot.fold_point_to_tab_point(new_end),
149 }
150 })
151 .collect()
152 } else {
153 vec![]
154 }
155 } else {
156 new_snapshot.version += 1;
157 vec![TabEdit {
158 old: TabPoint::zero()..old_snapshot.max_point(),
159 new: TabPoint::zero()..new_snapshot.max_point(),
160 }]
161 };
162 *old_snapshot = new_snapshot;
163 (old_snapshot.clone(), tab_edits)
164 }
165}
166
167#[derive(Clone)]
168pub struct TabSnapshot {
169 pub fold_snapshot: FoldSnapshot,
170 pub tab_size: NonZeroU32,
171 pub max_expansion_column: u32,
172 pub version: usize,
173}
174
175impl std::ops::Deref for TabSnapshot {
176 type Target = FoldSnapshot;
177
178 fn deref(&self) -> &Self::Target {
179 &self.fold_snapshot
180 }
181}
182
183impl TabSnapshot {
184 #[ztracing::instrument(skip_all)]
185 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
186 &self.fold_snapshot.inlay_snapshot.buffer
187 }
188
189 #[ztracing::instrument(skip_all)]
190 pub fn line_len(&self, row: u32) -> u32 {
191 let max_point = self.max_point();
192 if row < max_point.row() {
193 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
194 .0
195 .column
196 } else {
197 max_point.column()
198 }
199 }
200
201 #[ztracing::instrument(skip_all)]
202 pub fn text_summary(&self) -> TextSummary {
203 self.text_summary_for_range(TabPoint::zero()..self.max_point())
204 }
205
206 #[ztracing::instrument(skip_all, fields(rows))]
207 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
208 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
209 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
210 let input_summary = self
211 .fold_snapshot
212 .text_summary_for_range(input_start..input_end);
213
214 let line_end = if range.start.row() == range.end.row() {
215 range.end
216 } else {
217 self.max_point()
218 };
219 let first_line_chars = self
220 .chunks(range.start..line_end, false, Highlights::default())
221 .flat_map(|chunk| chunk.text.chars())
222 .take_while(|&c| c != '\n')
223 .count() as u32;
224
225 let last_line_chars = if range.start.row() == range.end.row() {
226 first_line_chars
227 } else {
228 self.chunks(
229 TabPoint::new(range.end.row(), 0)..range.end,
230 false,
231 Highlights::default(),
232 )
233 .flat_map(|chunk| chunk.text.chars())
234 .count() as u32
235 };
236
237 TextSummary {
238 lines: range.end.0 - range.start.0,
239 first_line_chars,
240 last_line_chars,
241 longest_row: input_summary.longest_row,
242 longest_row_chars: input_summary.longest_row_chars,
243 }
244 }
245
246 #[ztracing::instrument(skip_all)]
247 pub(crate) fn chunks<'a>(
248 &'a self,
249 range: Range<TabPoint>,
250 language_aware: bool,
251 highlights: Highlights<'a>,
252 ) -> TabChunks<'a> {
253 let (input_start, expanded_char_column, to_next_stop) =
254 self.tab_point_to_fold_point(range.start, Bias::Left);
255 let input_column = input_start.column();
256 let input_start = input_start.to_offset(&self.fold_snapshot);
257 let input_end = self
258 .tab_point_to_fold_point(range.end, Bias::Right)
259 .0
260 .to_offset(&self.fold_snapshot);
261 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
262 range.end.column() - range.start.column()
263 } else {
264 to_next_stop
265 };
266
267 TabChunks {
268 snapshot: self,
269 fold_chunks: self.fold_snapshot.chunks(
270 input_start..input_end,
271 language_aware,
272 highlights,
273 ),
274 input_column,
275 column: expanded_char_column,
276 max_expansion_column: self.max_expansion_column,
277 output_position: range.start.0,
278 max_output_position: range.end.0,
279 tab_size: self.tab_size,
280 chunk: Chunk {
281 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
282 is_tab: true,
283 chars: 1u128.unbounded_shl(to_next_stop) - 1,
284 ..Default::default()
285 },
286 inside_leading_tab: to_next_stop > 0,
287 }
288 }
289
290 #[ztracing::instrument(skip_all)]
291 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
292 self.fold_snapshot.row_infos(row)
293 }
294
295 #[cfg(test)]
296 #[ztracing::instrument(skip_all)]
297 pub fn text(&self) -> String {
298 self.chunks(
299 TabPoint::zero()..self.max_point(),
300 false,
301 Highlights::default(),
302 )
303 .map(|chunk| chunk.text)
304 .collect()
305 }
306
307 #[ztracing::instrument(skip_all)]
308 pub fn max_point(&self) -> TabPoint {
309 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
310 }
311
312 #[ztracing::instrument(skip_all)]
313 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
314 self.fold_point_to_tab_point(
315 self.fold_snapshot
316 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
317 )
318 }
319
320 #[ztracing::instrument(skip_all)]
321 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
322 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
323 let tab_cursor = TabStopCursor::new(chunks);
324 let expanded = self.expand_tabs(tab_cursor, input.column());
325 TabPoint::new(input.row(), expanded)
326 }
327
328 #[ztracing::instrument(skip_all)]
329 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
330 TabPointCursor { this: self }
331 }
332
333 #[ztracing::instrument(skip_all)]
334 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
335 let chunks = self
336 .fold_snapshot
337 .chunks_at(FoldPoint::new(output.row(), 0));
338
339 let tab_cursor = TabStopCursor::new(chunks);
340 let expanded = output.column();
341 let (collapsed, expanded_char_column, to_next_stop) =
342 self.collapse_tabs(tab_cursor, expanded, bias);
343
344 (
345 FoldPoint::new(output.row(), collapsed),
346 expanded_char_column,
347 to_next_stop,
348 )
349 }
350
351 #[ztracing::instrument(skip_all)]
352 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
353 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
354 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
355 self.fold_point_to_tab_point(fold_point)
356 }
357
358 #[ztracing::instrument(skip_all)]
359 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
360 let fold_point = self.tab_point_to_fold_point(point, bias).0;
361 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
362 self.fold_snapshot
363 .inlay_snapshot
364 .to_buffer_point(inlay_point)
365 }
366
367 #[ztracing::instrument(skip_all)]
368 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
369 where
370 I: Iterator<Item = Chunk<'a>>,
371 {
372 let tab_size = self.tab_size.get();
373
374 let end_column = column.min(self.max_expansion_column);
375 let mut seek_target = end_column;
376 let mut tab_count = 0;
377 let mut expanded_tab_len = 0;
378
379 while let Some(tab_stop) = cursor.seek(seek_target) {
380 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
381 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
382 tab_count += 1;
383 expanded_tab_len += tab_len;
384
385 seek_target = end_column - cursor.byte_offset;
386 }
387
388 let left_over_char_bytes = if !cursor.is_char_boundary() {
389 cursor.bytes_until_next_char().unwrap_or(0) as u32
390 } else {
391 0
392 };
393
394 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
395 let expanded_bytes =
396 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
397
398 expanded_bytes + column.saturating_sub(collapsed_bytes)
399 }
400
401 #[ztracing::instrument(skip_all)]
402 fn collapse_tabs<'a, I>(
403 &self,
404 mut cursor: TabStopCursor<'a, I>,
405 column: u32,
406 bias: Bias,
407 ) -> (u32, u32, u32)
408 where
409 I: Iterator<Item = Chunk<'a>>,
410 {
411 let tab_size = self.tab_size.get();
412 let mut collapsed_column = column;
413 let mut seek_target = column.min(self.max_expansion_column);
414 let mut tab_count = 0;
415 let mut expanded_tab_len = 0;
416
417 while let Some(tab_stop) = cursor.seek(seek_target) {
418 // Calculate how much we want to expand this tab stop (into spaces)
419 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
420 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
421 // Increment tab count
422 tab_count += 1;
423 // The count of how many spaces we've added to this line in place of tab bytes
424 expanded_tab_len += tab_len;
425
426 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
427 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
428
429 // Did we expand past the search target?
430 if expanded_bytes > column {
431 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
432 // We expanded past the search target, so need to account for the offshoot
433 expanded_chars -= expanded_bytes - column;
434 return match bias {
435 Bias::Left => (
436 cursor.byte_offset() - 1,
437 expanded_chars,
438 expanded_bytes - column,
439 ),
440 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
441 };
442 } else {
443 // otherwise we only want to move the cursor collapse column forward
444 collapsed_column = collapsed_column - tab_len + 1;
445 seek_target = (collapsed_column - cursor.byte_offset)
446 .min(self.max_expansion_column - cursor.byte_offset);
447 }
448 }
449
450 let collapsed_bytes = cursor.byte_offset();
451 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
452 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
453 (
454 collapsed_bytes + column.saturating_sub(expanded_bytes),
455 expanded_chars,
456 0,
457 )
458 }
459}
460
461// todo(lw): Implement TabPointCursor properly
462pub struct TabPointCursor<'this> {
463 this: &'this TabSnapshot,
464}
465
466impl TabPointCursor<'_> {
467 #[ztracing::instrument(skip_all)]
468 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
469 self.this.fold_point_to_tab_point(point)
470 }
471}
472
473#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
474pub struct TabPoint(pub Point);
475
476impl TabPoint {
477 pub fn new(row: u32, column: u32) -> Self {
478 Self(Point::new(row, column))
479 }
480
481 pub fn zero() -> Self {
482 Self::new(0, 0)
483 }
484
485 pub fn row(self) -> u32 {
486 self.0.row
487 }
488
489 pub fn column(self) -> u32 {
490 self.0.column
491 }
492}
493
494impl From<Point> for TabPoint {
495 fn from(point: Point) -> Self {
496 Self(point)
497 }
498}
499
500pub type TabEdit = text::Edit<TabPoint>;
501
502#[derive(Clone, Debug, Default, Eq, PartialEq)]
503pub struct TextSummary {
504 pub lines: Point,
505 pub first_line_chars: u32,
506 pub last_line_chars: u32,
507 pub longest_row: u32,
508 pub longest_row_chars: u32,
509}
510
511impl<'a> From<&'a str> for TextSummary {
512 #[ztracing::instrument(skip_all)]
513 fn from(text: &'a str) -> Self {
514 let sum = text::TextSummary::from(text);
515
516 TextSummary {
517 lines: sum.lines,
518 first_line_chars: sum.first_line_chars,
519 last_line_chars: sum.last_line_chars,
520 longest_row: sum.longest_row,
521 longest_row_chars: sum.longest_row_chars,
522 }
523 }
524}
525
526impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
527 #[ztracing::instrument(skip_all)]
528 fn add_assign(&mut self, other: &'a Self) {
529 let joined_chars = self.last_line_chars + other.first_line_chars;
530 if joined_chars > self.longest_row_chars {
531 self.longest_row = self.lines.row;
532 self.longest_row_chars = joined_chars;
533 }
534 if other.longest_row_chars > self.longest_row_chars {
535 self.longest_row = self.lines.row + other.longest_row;
536 self.longest_row_chars = other.longest_row_chars;
537 }
538
539 if self.lines.row == 0 {
540 self.first_line_chars += other.first_line_chars;
541 }
542
543 if other.lines.row == 0 {
544 self.last_line_chars += other.first_line_chars;
545 } else {
546 self.last_line_chars = other.last_line_chars;
547 }
548
549 self.lines += &other.lines;
550 }
551}
552
553pub struct TabChunks<'a> {
554 snapshot: &'a TabSnapshot,
555 max_expansion_column: u32,
556 max_output_position: Point,
557 tab_size: NonZeroU32,
558 // region: iteration state
559 fold_chunks: FoldChunks<'a>,
560 chunk: Chunk<'a>,
561 column: u32,
562 output_position: Point,
563 input_column: u32,
564 inside_leading_tab: bool,
565 // endregion: iteration state
566}
567
568impl TabChunks<'_> {
569 #[ztracing::instrument(skip_all)]
570 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
571 let (input_start, expanded_char_column, to_next_stop) = self
572 .snapshot
573 .tab_point_to_fold_point(range.start, Bias::Left);
574 let input_column = input_start.column();
575 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
576 let input_end = self
577 .snapshot
578 .tab_point_to_fold_point(range.end, Bias::Right)
579 .0
580 .to_offset(&self.snapshot.fold_snapshot);
581 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
582 range.end.column() - range.start.column()
583 } else {
584 to_next_stop
585 };
586
587 self.fold_chunks.seek(input_start..input_end);
588 self.input_column = input_column;
589 self.column = expanded_char_column;
590 self.output_position = range.start.0;
591 self.max_output_position = range.end.0;
592 self.chunk = Chunk {
593 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
594 is_tab: true,
595 chars: 1u128.unbounded_shl(to_next_stop) - 1,
596 ..Default::default()
597 };
598 self.inside_leading_tab = to_next_stop > 0;
599 }
600}
601
602impl<'a> Iterator for TabChunks<'a> {
603 type Item = Chunk<'a>;
604
605 #[ztracing::instrument(skip_all)]
606 fn next(&mut self) -> Option<Self::Item> {
607 while self.chunk.text.is_empty() {
608 let chunk = self.fold_chunks.next()?;
609 self.chunk = chunk;
610 if self.inside_leading_tab {
611 self.chunk.text = &self.chunk.text[1..];
612 self.chunk.tabs >>= 1;
613 self.chunk.chars >>= 1;
614 self.chunk.newlines >>= 1;
615 self.inside_leading_tab = false;
616 self.input_column += 1;
617 }
618 }
619
620 let first_tab_ix = if self.chunk.tabs != 0 {
621 self.chunk.tabs.trailing_zeros() as usize
622 } else {
623 self.chunk.text.len()
624 };
625
626 if first_tab_ix == 0 {
627 self.chunk.text = &self.chunk.text[1..];
628 self.chunk.tabs >>= 1;
629 self.chunk.chars >>= 1;
630 self.chunk.newlines >>= 1;
631
632 let tab_size = if self.input_column < self.max_expansion_column {
633 self.tab_size.get()
634 } else {
635 1
636 };
637 let mut len = tab_size - self.column % tab_size;
638 let next_output_position = cmp::min(
639 self.output_position + Point::new(0, len),
640 self.max_output_position,
641 );
642 len = next_output_position.column - self.output_position.column;
643 self.column += len;
644 self.input_column += 1;
645 self.output_position = next_output_position;
646
647 return Some(Chunk {
648 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
649 is_tab: true,
650 chars: 1u128.unbounded_shl(len) - 1,
651 tabs: 0,
652 newlines: 0,
653 ..self.chunk.clone()
654 });
655 }
656
657 let prefix_len = first_tab_ix;
658 let (prefix, suffix) = self.chunk.text.split_at(prefix_len);
659
660 let mask = 1u128.unbounded_shl(prefix_len as u32).wrapping_sub(1);
661 let prefix_chars = self.chunk.chars & mask;
662 let prefix_tabs = self.chunk.tabs & mask;
663 let prefix_newlines = self.chunk.newlines & mask;
664
665 self.chunk.text = suffix;
666 self.chunk.tabs = self.chunk.tabs.unbounded_shr(prefix_len as u32);
667 self.chunk.chars = self.chunk.chars.unbounded_shr(prefix_len as u32);
668 self.chunk.newlines = self.chunk.newlines.unbounded_shr(prefix_len as u32);
669
670 let newline_count = prefix_newlines.count_ones();
671 if newline_count > 0 {
672 let last_newline_bit = 128 - prefix_newlines.leading_zeros();
673 let chars_after_last_newline =
674 prefix_chars.unbounded_shr(last_newline_bit).count_ones();
675 let bytes_after_last_newline = prefix_len as u32 - last_newline_bit;
676
677 self.column = chars_after_last_newline;
678 self.input_column = bytes_after_last_newline;
679 self.output_position = Point::new(
680 self.output_position.row + newline_count,
681 bytes_after_last_newline,
682 );
683 } else {
684 let char_count = prefix_chars.count_ones();
685 self.column += char_count;
686 if !self.inside_leading_tab {
687 self.input_column += prefix_len as u32;
688 }
689 self.output_position.column += prefix_len as u32;
690 }
691
692 Some(Chunk {
693 text: prefix,
694 chars: prefix_chars,
695 tabs: prefix_tabs,
696 newlines: prefix_newlines,
697 ..self.chunk.clone()
698 })
699 }
700}
701
702#[cfg(test)]
703mod tests {
704 use std::mem;
705
706 use super::*;
707 use crate::{
708 MultiBuffer,
709 display_map::{
710 fold_map::{FoldMap, FoldOffset, FoldPlaceholder},
711 inlay_map::InlayMap,
712 },
713 };
714 use multi_buffer::MultiBufferOffset;
715 use rand::{Rng, prelude::StdRng};
716 use util;
717
718 impl TabSnapshot {
719 fn expected_collapse_tabs(
720 &self,
721 chars: impl Iterator<Item = char>,
722 column: u32,
723 bias: Bias,
724 ) -> (u32, u32, u32) {
725 let tab_size = self.tab_size.get();
726
727 let mut expanded_bytes = 0;
728 let mut expanded_chars = 0;
729 let mut collapsed_bytes = 0;
730 for c in chars {
731 if expanded_bytes >= column {
732 break;
733 }
734 if collapsed_bytes >= self.max_expansion_column {
735 break;
736 }
737
738 if c == '\t' {
739 let tab_len = tab_size - (expanded_chars % tab_size);
740 expanded_chars += tab_len;
741 expanded_bytes += tab_len;
742 if expanded_bytes > column {
743 expanded_chars -= expanded_bytes - column;
744 return match bias {
745 Bias::Left => {
746 (collapsed_bytes, expanded_chars, expanded_bytes - column)
747 }
748 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
749 };
750 }
751 } else {
752 expanded_chars += 1;
753 expanded_bytes += c.len_utf8() as u32;
754 }
755
756 if expanded_bytes > column && matches!(bias, Bias::Left) {
757 expanded_chars -= 1;
758 break;
759 }
760
761 collapsed_bytes += c.len_utf8() as u32;
762 }
763
764 (
765 collapsed_bytes + column.saturating_sub(expanded_bytes),
766 expanded_chars,
767 0,
768 )
769 }
770
771 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
772 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
773 let expanded = self.expected_expand_tabs(chars, input.column());
774 TabPoint::new(input.row(), expanded)
775 }
776
777 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
778 let tab_size = self.tab_size.get();
779
780 let mut expanded_chars = 0;
781 let mut expanded_bytes = 0;
782 let mut collapsed_bytes = 0;
783 let end_column = column.min(self.max_expansion_column);
784 for c in chars {
785 if collapsed_bytes >= end_column {
786 break;
787 }
788 if c == '\t' {
789 let tab_len = tab_size - expanded_chars % tab_size;
790 expanded_bytes += tab_len;
791 expanded_chars += tab_len;
792 } else {
793 expanded_bytes += c.len_utf8() as u32;
794 expanded_chars += 1;
795 }
796 collapsed_bytes += c.len_utf8() as u32;
797 }
798
799 expanded_bytes + column.saturating_sub(collapsed_bytes)
800 }
801
802 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
803 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
804 let expanded = output.column();
805 let (collapsed, expanded_char_column, to_next_stop) =
806 self.expected_collapse_tabs(chars, expanded, bias);
807 (
808 FoldPoint::new(output.row(), collapsed),
809 expanded_char_column,
810 to_next_stop,
811 )
812 }
813 }
814
815 #[gpui::test]
816 fn test_expand_tabs(cx: &mut gpui::App) {
817 let test_values = [
818 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
819 (" \twςe", 4),
820 ("fε", 1),
821 ("i❎\t", 3),
822 ];
823 let buffer = MultiBuffer::build_simple("", cx);
824 let buffer_snapshot = buffer.read(cx).snapshot(cx);
825 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
826 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
827 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
828
829 for (text, column) in test_values {
830 let mut tabs = 0u128;
831 let mut chars = 0u128;
832 for (idx, c) in text.char_indices() {
833 if c == '\t' {
834 tabs |= 1 << idx;
835 }
836 chars |= 1 << idx;
837 }
838
839 let chunks = [Chunk {
840 text,
841 tabs,
842 chars,
843 ..Default::default()
844 }];
845
846 let cursor = TabStopCursor::new(chunks);
847
848 assert_eq!(
849 tab_snapshot.expected_expand_tabs(text.chars(), column),
850 tab_snapshot.expand_tabs(cursor, column)
851 );
852 }
853 }
854
855 #[gpui::test]
856 fn test_collapse_tabs(cx: &mut gpui::App) {
857 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
858
859 let buffer = MultiBuffer::build_simple(input, cx);
860 let buffer_snapshot = buffer.read(cx).snapshot(cx);
861 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
862 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
863 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
864
865 for (ix, _) in input.char_indices() {
866 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
867
868 assert_eq!(
869 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
870 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
871 "Failed with tab_point at column {ix}"
872 );
873 assert_eq!(
874 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
875 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
876 "Failed with tab_point at column {ix}"
877 );
878
879 assert_eq!(
880 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
881 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
882 "Failed with tab_point at column {ix}"
883 );
884 assert_eq!(
885 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
886 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
887 "Failed with tab_point at column {ix}"
888 );
889 }
890 }
891
892 #[gpui::test]
893 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
894 // This test reproduces a specific panic where to_fold_point returns incorrect results
895 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
896
897 let text = "γ\tw⭐\n🍐🍗 \t";
898 let buffer = MultiBuffer::build_simple(text, cx);
899 let buffer_snapshot = buffer.read(cx).snapshot(cx);
900 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
901 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
902 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
903
904 // This should panic with the expected vs actual mismatch
905 let tab_point = TabPoint::new(0, 9);
906 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
907 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
908
909 assert_eq!(result, expected);
910 }
911
912 #[gpui::test(iterations = 100)]
913 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
914 // Generate random input string with up to 200 characters including tabs
915 // to stay within the MAX_EXPANSION_COLUMN limit of 256
916 let len = rng.random_range(0..=2048);
917 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
918 let mut input = String::with_capacity(len);
919
920 for _ in 0..len {
921 if rng.random_bool(0.1) {
922 // 10% chance of inserting a tab
923 input.push('\t');
924 } else {
925 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
926 let ch = loop {
927 let ascii_code = rng.random_range(32..=126); // printable ASCII range
928 let ch = ascii_code as u8 as char;
929 if ch != '\t' {
930 break ch;
931 }
932 };
933 input.push(ch);
934 }
935 }
936
937 let buffer = MultiBuffer::build_simple(&input, cx);
938 let buffer_snapshot = buffer.read(cx).snapshot(cx);
939 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
940 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
941 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
942 tab_snapshot.max_expansion_column = rng.random_range(0..323);
943 tab_snapshot.tab_size = tab_size;
944
945 for (ix, _) in input.char_indices() {
946 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
947
948 assert_eq!(
949 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
950 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
951 "Failed with input: {}, with idx: {ix}",
952 input
953 );
954 assert_eq!(
955 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
956 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
957 "Failed with input: {}, with idx: {ix}",
958 input
959 );
960
961 assert_eq!(
962 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
963 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
964 "Failed with input: {}, with idx: {ix}",
965 input
966 );
967 assert_eq!(
968 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
969 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
970 "Failed with input: {}, with idx: {ix}",
971 input
972 );
973 }
974 }
975
976 #[gpui::test]
977 fn test_long_lines(cx: &mut gpui::App) {
978 let max_expansion_column = 12;
979 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
980 let output = "A BC DEF G HI J K L M";
981
982 let buffer = MultiBuffer::build_simple(input, cx);
983 let buffer_snapshot = buffer.read(cx).snapshot(cx);
984 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
985 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
986 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
987
988 tab_snapshot.max_expansion_column = max_expansion_column;
989 assert_eq!(tab_snapshot.text(), output);
990
991 for (ix, c) in input.char_indices() {
992 assert_eq!(
993 tab_snapshot
994 .chunks(
995 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
996 false,
997 Highlights::default(),
998 )
999 .map(|c| c.text)
1000 .collect::<String>(),
1001 &output[ix..],
1002 "text from index {ix}"
1003 );
1004
1005 if c != '\t' {
1006 let input_point = Point::new(0, ix as u32);
1007 let output_point = Point::new(0, output.find(c).unwrap() as u32);
1008 assert_eq!(
1009 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
1010 TabPoint(output_point),
1011 "to_tab_point({input_point:?})"
1012 );
1013 assert_eq!(
1014 tab_snapshot
1015 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
1016 .0,
1017 FoldPoint(input_point),
1018 "to_fold_point({output_point:?})"
1019 );
1020 }
1021 }
1022 }
1023
1024 #[gpui::test]
1025 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
1026 let max_expansion_column = 8;
1027 let input = "abcdefg⋯hij";
1028
1029 let buffer = MultiBuffer::build_simple(input, cx);
1030 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1031 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1032 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1033 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1034
1035 tab_snapshot.max_expansion_column = max_expansion_column;
1036 assert_eq!(tab_snapshot.text(), input);
1037 }
1038
1039 #[gpui::test]
1040 fn test_marking_tabs(cx: &mut gpui::App) {
1041 let input = "\t \thello";
1042
1043 let buffer = MultiBuffer::build_simple(input, cx);
1044 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1045 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1046 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1047 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1048
1049 assert_eq!(
1050 chunks(&tab_snapshot, TabPoint::zero()),
1051 vec![
1052 (" ".to_string(), true),
1053 (" ".to_string(), false),
1054 (" ".to_string(), true),
1055 ("hello".to_string(), false),
1056 ]
1057 );
1058 assert_eq!(
1059 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1060 vec![
1061 (" ".to_string(), true),
1062 (" ".to_string(), false),
1063 (" ".to_string(), true),
1064 ("hello".to_string(), false),
1065 ]
1066 );
1067
1068 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1069 let mut chunks = Vec::new();
1070 let mut was_tab = false;
1071 let mut text = String::new();
1072 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1073 {
1074 if chunk.is_tab != was_tab {
1075 if !text.is_empty() {
1076 chunks.push((mem::take(&mut text), was_tab));
1077 }
1078 was_tab = chunk.is_tab;
1079 }
1080 text.push_str(chunk.text);
1081 }
1082
1083 if !text.is_empty() {
1084 chunks.push((text, was_tab));
1085 }
1086 chunks
1087 }
1088 }
1089
1090 #[gpui::test]
1091 fn test_empty_chunk_after_leading_tab_trim(cx: &mut gpui::App) {
1092 // We fold "hello" (offsets 1..6) so the fold map creates a
1093 // transform boundary at offset 1, producing a 1-byte fold chunk
1094 // for the tab.
1095 let text = "\thello";
1096 let buffer = MultiBuffer::build_simple(text, cx);
1097 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1098 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1099 let mut fold_map = FoldMap::new(inlay_snapshot.clone()).0;
1100
1101 let (mut writer, _, _) = fold_map.write(inlay_snapshot.clone(), vec![]);
1102 writer.fold(vec![(
1103 MultiBufferOffset(1)..MultiBufferOffset(6),
1104 FoldPlaceholder::test(),
1105 )]);
1106 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1107
1108 let tab_size = NonZeroU32::new(4).unwrap();
1109 let (_, tab_snapshot) = TabMap::new(fold_snapshot, tab_size);
1110
1111 // The tab at column 0 expands to 4 spaces (columns 0‥4).
1112 // Seek starting at column 2 (middle of that tab) so that
1113 // `inside_leading_tab = true` and `to_next_stop = 2`.
1114 // Set the end just past the tab expansion so the iterator must
1115 // process the tab byte from the fold chunk.
1116 let max = tab_snapshot.max_point();
1117 let start = TabPoint::new(0, 2);
1118 let end = max;
1119
1120 // This should not panic.
1121 let result: String = tab_snapshot
1122 .chunks(start..end, false, Highlights::default())
1123 .map(|c| c.text)
1124 .collect();
1125 assert!(!result.is_empty());
1126 }
1127
1128 #[gpui::test(iterations = 100)]
1129 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1130 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1131 let len = rng.random_range(0..30);
1132 let buffer = if rng.random() {
1133 let text = util::RandomCharIter::new(&mut rng)
1134 .take(len)
1135 .collect::<String>();
1136 MultiBuffer::build_simple(&text, cx)
1137 } else {
1138 MultiBuffer::build_random(&mut rng, cx)
1139 };
1140 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1141 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1142
1143 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1144 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1145 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1146 fold_map.randomly_mutate(&mut rng);
1147 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1148 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1149 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1150 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1151
1152 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1153 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1154
1155 let text = text::Rope::from(tabs_snapshot.text().as_str());
1156 log::info!(
1157 "TabMap text (tab size: {}): {:?}",
1158 tab_size,
1159 tabs_snapshot.text(),
1160 );
1161
1162 for _ in 0..5 {
1163 let end_row = rng.random_range(0..=text.max_point().row);
1164 let end_column = rng.random_range(0..=text.line_len(end_row));
1165 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1166 let start_row = rng.random_range(0..=text.max_point().row);
1167 let start_column = rng.random_range(0..=text.line_len(start_row));
1168 let mut start =
1169 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1170 if start > end {
1171 mem::swap(&mut start, &mut end);
1172 }
1173
1174 let expected_text = text
1175 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1176 .collect::<String>();
1177 let expected_summary = TextSummary::from(expected_text.as_str());
1178 assert_eq!(
1179 tabs_snapshot
1180 .chunks(start..end, false, Highlights::default())
1181 .map(|c| c.text)
1182 .collect::<String>(),
1183 expected_text,
1184 "chunks({:?}..{:?})",
1185 start,
1186 end
1187 );
1188
1189 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1190 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1191 actual_summary.longest_row = expected_summary.longest_row;
1192 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1193 }
1194 assert_eq!(actual_summary, expected_summary);
1195 }
1196
1197 for row in 0..=text.max_point().row {
1198 assert_eq!(
1199 tabs_snapshot.line_len(row),
1200 text.line_len(row),
1201 "line_len({row})"
1202 );
1203 }
1204 }
1205
1206 #[gpui::test(iterations = 100)]
1207 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1208 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1209 let len = rng.random_range(0..=2000);
1210
1211 // Generate random text using RandomCharIter
1212 let text = util::RandomCharIter::new(&mut rng)
1213 .take(len)
1214 .collect::<String>();
1215
1216 // Create buffer and tab map
1217 let buffer = MultiBuffer::build_simple(&text, cx);
1218 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1219 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1220 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1221 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1222
1223 let mut next_inlay_id = 0;
1224 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1225 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1226 let max_fold_point = fold_snapshot.max_point();
1227 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1228
1229 // Test random fold points
1230 for _ in 0..50 {
1231 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1232 // Generate random fold point
1233 let row = rng.random_range(0..=max_fold_point.row());
1234 let max_column = if row < max_fold_point.row() {
1235 fold_snapshot.line_len(row)
1236 } else {
1237 max_fold_point.column()
1238 };
1239 let column = rng.random_range(0..=max_column + 10);
1240 let fold_point = FoldPoint::new(row, column);
1241
1242 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1243 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1244
1245 assert_eq!(
1246 actual, expected,
1247 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1248 fold_point, text
1249 );
1250 }
1251 }
1252
1253 #[gpui::test]
1254 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1255 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1256 let buffer = MultiBuffer::build_simple(text, cx);
1257 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1258 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1259 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1260 let chunks = fold_snapshot.chunks(
1261 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1262 false,
1263 Default::default(),
1264 );
1265 let mut cursor = TabStopCursor::new(chunks);
1266 assert!(cursor.seek(0).is_none());
1267 let mut tab_stops = Vec::new();
1268
1269 let mut all_tab_stops = Vec::new();
1270 let mut byte_offset = 0;
1271 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1272 byte_offset += ch.len_utf8() as u32;
1273
1274 if ch == '\t' {
1275 all_tab_stops.push(TabStop {
1276 byte_offset,
1277 char_offset: offset as u32 + 1,
1278 });
1279 }
1280 }
1281
1282 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1283 tab_stops.push(tab_stop);
1284 }
1285 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1286
1287 assert_eq!(cursor.byte_offset(), byte_offset);
1288 }
1289
1290 #[gpui::test]
1291 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1292 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1293
1294 let buffer = MultiBuffer::build_simple(input, cx);
1295 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1296 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1297 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1298
1299 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1300 let mut cursor = TabStopCursor::new(chunks);
1301
1302 let mut actual_tab_stops = Vec::new();
1303
1304 let mut expected_tab_stops = Vec::new();
1305 let mut byte_offset = 0;
1306 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1307 byte_offset += ch.len_utf8() as u32;
1308
1309 if ch == '\t' {
1310 expected_tab_stops.push(TabStop {
1311 byte_offset,
1312 char_offset: offset as u32 + 1,
1313 });
1314 }
1315 }
1316
1317 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1318 actual_tab_stops.push(tab_stop);
1319 }
1320 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1321
1322 assert_eq!(cursor.byte_offset(), byte_offset);
1323 }
1324
1325 #[gpui::test(iterations = 100)]
1326 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1327 // Generate random input string with up to 512 characters including tabs
1328 let len = rng.random_range(0..=2048);
1329 let mut input = String::with_capacity(len);
1330
1331 let mut skip_tabs = rng.random_bool(0.10);
1332 for idx in 0..len {
1333 if idx % 128 == 0 {
1334 skip_tabs = rng.random_bool(0.10);
1335 }
1336
1337 if rng.random_bool(0.15) && !skip_tabs {
1338 input.push('\t');
1339 } else {
1340 let ch = loop {
1341 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1342 let ch = ascii_code as u8 as char;
1343 if ch != '\t' {
1344 break ch;
1345 }
1346 };
1347 input.push(ch);
1348 }
1349 }
1350
1351 // Build the buffer and create cursor
1352 let buffer = MultiBuffer::build_simple(&input, cx);
1353 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1354 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1355 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1356
1357 // First, collect all expected tab positions
1358 let mut all_tab_stops = Vec::new();
1359 let mut byte_offset = 1;
1360 let mut char_offset = 1;
1361 for ch in buffer_snapshot.text().chars() {
1362 if ch == '\t' {
1363 all_tab_stops.push(TabStop {
1364 byte_offset,
1365 char_offset,
1366 });
1367 }
1368 byte_offset += ch.len_utf8() as u32;
1369 char_offset += 1;
1370 }
1371
1372 // Test with various distances
1373 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1374 // let distances = vec![150];
1375
1376 for distance in distances {
1377 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1378 let mut cursor = TabStopCursor::new(chunks);
1379
1380 let mut found_tab_stops = Vec::new();
1381 let mut position = distance;
1382 while let Some(tab_stop) = cursor.seek(position) {
1383 found_tab_stops.push(tab_stop);
1384 position = distance - tab_stop.byte_offset;
1385 }
1386
1387 let expected_found_tab_stops: Vec<_> = all_tab_stops
1388 .iter()
1389 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1390 .cloned()
1391 .collect();
1392
1393 pretty_assertions::assert_eq!(
1394 found_tab_stops,
1395 expected_found_tab_stops,
1396 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1397 distance,
1398 input
1399 );
1400
1401 let final_position = cursor.byte_offset();
1402 if !found_tab_stops.is_empty() {
1403 let last_tab_stop = found_tab_stops.last().unwrap();
1404 assert!(
1405 final_position >= last_tab_stop.byte_offset,
1406 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1407 final_position,
1408 last_tab_stop.byte_offset,
1409 input
1410 );
1411 }
1412 }
1413 }
1414
1415 #[gpui::test]
1416 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1417 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1418 let buffer = MultiBuffer::build_simple(text, cx);
1419 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1420 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1421 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1422 let chunks = fold_snapshot.chunks(
1423 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1424 false,
1425 Default::default(),
1426 );
1427 let mut cursor = TabStopCursor::new(chunks);
1428 assert!(cursor.seek(0).is_none());
1429
1430 let mut expected_tab_stops = Vec::new();
1431 let mut byte_offset = 0;
1432 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1433 byte_offset += ch.len_utf8() as u32;
1434
1435 if ch == '\t' {
1436 expected_tab_stops.push(TabStop {
1437 byte_offset,
1438 char_offset: i as u32 + 1,
1439 });
1440 }
1441 }
1442
1443 let mut actual_tab_stops = Vec::new();
1444 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1445 actual_tab_stops.push(tab_stop);
1446 }
1447
1448 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1449
1450 assert_eq!(cursor.byte_offset(), byte_offset);
1451 }
1452
1453 #[gpui::test(iterations = 100)]
1454 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1455 // Generate random input string with up to 512 characters including tabs
1456 let len = rng.random_range(0..=2048);
1457 let input = util::RandomCharIter::new(&mut rng)
1458 .take(len)
1459 .collect::<String>();
1460
1461 // Build the buffer and create cursor
1462 let buffer = MultiBuffer::build_simple(&input, cx);
1463 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1464 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1465 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1466
1467 // First, collect all expected tab positions
1468 let mut all_tab_stops = Vec::new();
1469 let mut byte_offset = 0;
1470 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1471 byte_offset += ch.len_utf8() as u32;
1472 if ch == '\t' {
1473 all_tab_stops.push(TabStop {
1474 byte_offset,
1475 char_offset: i as u32 + 1,
1476 });
1477 }
1478 }
1479
1480 // Test with various distances
1481 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1482 let distances = vec![150];
1483
1484 for distance in distances {
1485 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1486 let mut cursor = TabStopCursor::new(chunks);
1487
1488 let mut found_tab_stops = Vec::new();
1489 let mut position = distance;
1490 while let Some(tab_stop) = cursor.seek(position) {
1491 found_tab_stops.push(tab_stop);
1492 position = distance - tab_stop.byte_offset;
1493 }
1494
1495 let expected_found_tab_stops: Vec<_> = all_tab_stops
1496 .iter()
1497 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1498 .cloned()
1499 .collect();
1500
1501 pretty_assertions::assert_eq!(
1502 found_tab_stops,
1503 expected_found_tab_stops,
1504 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1505 distance,
1506 input
1507 );
1508
1509 let final_position = cursor.byte_offset();
1510 if !found_tab_stops.is_empty() {
1511 let last_tab_stop = found_tab_stops.last().unwrap();
1512 assert!(
1513 final_position >= last_tab_stop.byte_offset,
1514 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1515 final_position,
1516 last_tab_stop.byte_offset,
1517 input
1518 );
1519 }
1520 }
1521 }
1522}
1523
1524struct TabStopCursor<'a, I>
1525where
1526 I: Iterator<Item = Chunk<'a>>,
1527{
1528 chunks: I,
1529 byte_offset: u32,
1530 char_offset: u32,
1531 /// Chunk
1532 /// last tab position iterated through
1533 current_chunk: Option<(Chunk<'a>, u32)>,
1534}
1535
1536impl<'a, I> TabStopCursor<'a, I>
1537where
1538 I: Iterator<Item = Chunk<'a>>,
1539{
1540 #[ztracing::instrument(skip_all)]
1541 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1542 Self {
1543 chunks: chunks.into_iter(),
1544 byte_offset: 0,
1545 char_offset: 0,
1546 current_chunk: None,
1547 }
1548 }
1549
1550 #[ztracing::instrument(skip_all)]
1551 fn bytes_until_next_char(&self) -> Option<usize> {
1552 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1553 let mut idx = *idx;
1554 let mut diff = 0;
1555 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1556 idx -= 1;
1557 diff += 1;
1558 }
1559
1560 if chunk.chars & (1 << idx) != 0 {
1561 Some(
1562 (chunk.text[idx as usize..].chars().next()?)
1563 .len_utf8()
1564 .saturating_sub(diff),
1565 )
1566 } else {
1567 None
1568 }
1569 })
1570 }
1571
1572 #[ztracing::instrument(skip_all)]
1573 fn is_char_boundary(&self) -> bool {
1574 self.current_chunk
1575 .as_ref()
1576 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1577 }
1578
1579 /// distance: length to move forward while searching for the next tab stop
1580 #[ztracing::instrument(skip_all)]
1581 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1582 if distance == 0 {
1583 return None;
1584 }
1585
1586 let mut distance_traversed = 0;
1587
1588 while let Some((mut chunk, chunk_position)) = self
1589 .current_chunk
1590 .take()
1591 .or_else(|| self.chunks.next().zip(Some(0)))
1592 {
1593 if chunk.tabs == 0 {
1594 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1595 if chunk_distance + distance_traversed >= distance {
1596 let overshoot = distance_traversed.abs_diff(distance);
1597
1598 self.byte_offset += overshoot;
1599 self.char_offset += get_char_offset(
1600 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1601 chunk.chars,
1602 );
1603
1604 if chunk_position + overshoot < 128 {
1605 self.current_chunk = Some((chunk, chunk_position + overshoot));
1606 }
1607
1608 return None;
1609 }
1610
1611 self.byte_offset += chunk_distance;
1612 self.char_offset += get_char_offset(
1613 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1614 chunk.chars,
1615 );
1616 distance_traversed += chunk_distance;
1617 continue;
1618 }
1619 let tab_position = chunk.tabs.trailing_zeros() + 1;
1620
1621 if distance_traversed + tab_position - chunk_position > distance {
1622 let cursor_position = distance_traversed.abs_diff(distance);
1623
1624 self.char_offset += get_char_offset(
1625 chunk_position..(chunk_position + cursor_position - 1),
1626 chunk.chars,
1627 );
1628 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1629 self.byte_offset += cursor_position;
1630
1631 return None;
1632 }
1633
1634 self.byte_offset += tab_position - chunk_position;
1635 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1636
1637 let tabstop = TabStop {
1638 char_offset: self.char_offset,
1639 byte_offset: self.byte_offset,
1640 };
1641
1642 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1643
1644 if tab_position as usize != chunk.text.len() {
1645 self.current_chunk = Some((chunk, tab_position));
1646 }
1647
1648 return Some(tabstop);
1649 }
1650
1651 None
1652 }
1653
1654 fn byte_offset(&self) -> u32 {
1655 self.byte_offset
1656 }
1657
1658 fn char_offset(&self) -> u32 {
1659 self.char_offset
1660 }
1661}
1662
1663#[inline(always)]
1664fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1665 if range.start == range.end {
1666 return if (1u128 << range.start) & bit_map == 0 {
1667 0
1668 } else {
1669 1
1670 };
1671 }
1672 let end_shift: u128 = 127u128 - range.end as u128;
1673 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1674 bit_mask = (bit_mask << end_shift) >> end_shift;
1675 let bit_map = bit_map & bit_mask;
1676
1677 bit_map.count_ones()
1678}
1679
1680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1681struct TabStop {
1682 char_offset: u32,
1683 byte_offset: u32,
1684}