1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::{LanguageAwareStyling, Point};
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 #[ztracing::instrument(skip_all)]
24 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
25 let snapshot = TabSnapshot {
26 fold_snapshot,
27 tab_size: tab_size.min(MAX_TABS),
28 max_expansion_column: MAX_EXPANSION_COLUMN,
29 version: 0,
30 };
31 (Self(snapshot.clone()), snapshot)
32 }
33
34 #[cfg(test)]
35 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
36 self.0.max_expansion_column = column;
37 self.0.clone()
38 }
39
40 #[ztracing::instrument(skip_all)]
41 pub fn sync(
42 &mut self,
43 fold_snapshot: FoldSnapshot,
44 mut fold_edits: Vec<FoldEdit>,
45 tab_size: NonZeroU32,
46 ) -> (TabSnapshot, Vec<TabEdit>) {
47 let tab_size = tab_size.min(MAX_TABS);
48
49 if self.0.tab_size != tab_size {
50 let old_max_point = self.0.max_point();
51 self.0.version += 1;
52 self.0.fold_snapshot = fold_snapshot;
53 self.0.tab_size = tab_size;
54 return (
55 self.0.clone(),
56 vec![TabEdit {
57 old: TabPoint::zero()..old_max_point,
58 new: TabPoint::zero()..self.0.max_point(),
59 }],
60 );
61 }
62
63 let old_snapshot = &mut self.0;
64 let mut new_version = old_snapshot.version;
65 if old_snapshot.fold_snapshot.version != fold_snapshot.version {
66 new_version += 1;
67 }
68
69 if fold_edits.is_empty() {
70 old_snapshot.version = new_version;
71 old_snapshot.fold_snapshot = fold_snapshot;
72 old_snapshot.tab_size = tab_size;
73 return (old_snapshot.clone(), vec![]);
74 }
75 // Expand each edit to include the next tab on the same line as the edit,
76 // and any subsequent tabs on that line that moved across the tab expansion
77 // boundary.
78 //
79 // This is necessary because a tab's display width depends on its column
80 // position: it expands to fill up to the next tab stop. When an edit
81 // shifts text on a line, any tab character after the edit may now render
82 // at a different width even though the tab byte itself wasn't touched.
83 // Additionally, tabs beyond `max_expansion_column` are rendered as a
84 // single space instead of expanding to the next tab stop. An edit that
85 // shifts a tab across that boundary changes its display width, so the
86 // edit must cover it. We scan forward from the edit end to the end of
87 // the line, extending the edit to include the first subsequent tab (whose
88 // rendered width may have changed) and the last tab that crossed the
89 // expansion boundary (transitioning between expanded and non-expanded).
90 for fold_edit in &mut fold_edits {
91 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
92 let old_end_row_successor_offset = cmp::min(
93 FoldPoint::new(old_end.row() + 1, 0),
94 old_snapshot.fold_snapshot.max_point(),
95 )
96 .to_offset(&old_snapshot.fold_snapshot);
97 let new_end = fold_edit.new.end.to_point(&fold_snapshot);
98
99 let mut offset_from_edit = 0;
100 let mut first_tab_offset = None;
101 let mut last_tab_with_changed_expansion_offset = None;
102 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
103 fold_edit.old.end..old_end_row_successor_offset,
104 LanguageAwareStyling {
105 tree_sitter: false,
106 diagnostics: false,
107 },
108 Highlights::default(),
109 ) {
110 let mut remaining_tabs = chunk.tabs;
111 while remaining_tabs != 0 {
112 let ix = remaining_tabs.trailing_zeros();
113 let offset_from_edit = offset_from_edit + ix;
114 if first_tab_offset.is_none() {
115 first_tab_offset = Some(offset_from_edit);
116 }
117
118 let old_column = old_end.column() + offset_from_edit;
119 let new_column = new_end.column() + offset_from_edit;
120 let was_expanded = old_column < old_snapshot.max_expansion_column;
121 let is_expanded = new_column < old_snapshot.max_expansion_column;
122 if was_expanded != is_expanded {
123 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
124 } else if !was_expanded && !is_expanded {
125 break 'outer;
126 }
127
128 remaining_tabs &= remaining_tabs - 1;
129 }
130
131 offset_from_edit += chunk.text.len() as u32;
132 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
133 && new_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
134 {
135 break;
136 }
137 }
138
139 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
140 fold_edit.old.end.0 += offset as usize + 1;
141 fold_edit.new.end.0 += offset as usize + 1;
142 }
143 }
144
145 let new_snapshot = TabSnapshot {
146 fold_snapshot,
147 tab_size,
148 max_expansion_column: old_snapshot.max_expansion_column,
149 version: new_version,
150 };
151
152 let _old_alloc_ptr = fold_edits.as_ptr();
153 // Combine any edits that overlap due to the expansion.
154 let mut fold_edits = fold_edits.into_iter();
155 let mut first_edit = fold_edits.next().unwrap();
156 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
157 #[allow(clippy::filter_map_identity)]
158 let mut v: Vec<_> = fold_edits
159 .scan(&mut first_edit, |state, edit| {
160 if state.old.end >= edit.old.start {
161 state.old.end = edit.old.end;
162 state.new.end = edit.new.end;
163 Some(None) // Skip this edit, it's merged
164 } else {
165 let new_state = edit;
166 let result = Some(Some(state.clone())); // Yield the previous edit
167 **state = new_state;
168 result
169 }
170 })
171 .filter_map(|x| x)
172 .collect();
173 v.push(first_edit);
174 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
175 let tab_edits = v
176 .into_iter()
177 .map(|fold_edit| {
178 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
179 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
180 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
181 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
182 TabEdit {
183 old: old_snapshot.fold_point_to_tab_point(old_start)
184 ..old_snapshot.fold_point_to_tab_point(old_end),
185 new: new_snapshot.fold_point_to_tab_point(new_start)
186 ..new_snapshot.fold_point_to_tab_point(new_end),
187 }
188 })
189 .collect();
190 *old_snapshot = new_snapshot;
191 (old_snapshot.clone(), tab_edits)
192 }
193}
194
195#[derive(Clone)]
196pub struct TabSnapshot {
197 pub fold_snapshot: FoldSnapshot,
198 pub tab_size: NonZeroU32,
199 /// The maximum column up to which a tab can expand.
200 /// Any tab after this column will not expand.
201 pub max_expansion_column: u32,
202 pub version: usize,
203}
204
205impl std::ops::Deref for TabSnapshot {
206 type Target = FoldSnapshot;
207
208 fn deref(&self) -> &Self::Target {
209 &self.fold_snapshot
210 }
211}
212
213impl TabSnapshot {
214 #[ztracing::instrument(skip_all)]
215 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
216 &self.fold_snapshot.inlay_snapshot.buffer
217 }
218
219 #[ztracing::instrument(skip_all)]
220 pub fn line_len(&self, row: u32) -> u32 {
221 let max_point = self.max_point();
222 if row < max_point.row() {
223 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
224 .0
225 .column
226 } else {
227 max_point.column()
228 }
229 }
230
231 #[ztracing::instrument(skip_all)]
232 pub fn text_summary(&self) -> TextSummary {
233 self.text_summary_for_range(TabPoint::zero()..self.max_point())
234 }
235
236 #[ztracing::instrument(skip_all, fields(rows))]
237 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
238 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
239 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
240 let input_summary = self
241 .fold_snapshot
242 .text_summary_for_range(input_start..input_end);
243
244 let line_end = if range.start.row() == range.end.row() {
245 range.end
246 } else {
247 self.max_point()
248 };
249 let first_line_chars = self
250 .chunks(
251 range.start..line_end,
252 LanguageAwareStyling {
253 tree_sitter: false,
254 diagnostics: false,
255 },
256 Highlights::default(),
257 )
258 .flat_map(|chunk| chunk.text.chars())
259 .take_while(|&c| c != '\n')
260 .count() as u32;
261
262 let last_line_chars = if range.start.row() == range.end.row() {
263 first_line_chars
264 } else {
265 self.chunks(
266 TabPoint::new(range.end.row(), 0)..range.end,
267 LanguageAwareStyling {
268 tree_sitter: false,
269 diagnostics: false,
270 },
271 Highlights::default(),
272 )
273 .flat_map(|chunk| chunk.text.chars())
274 .count() as u32
275 };
276
277 TextSummary {
278 lines: range.end.0 - range.start.0,
279 first_line_chars,
280 last_line_chars,
281 longest_row: input_summary.longest_row,
282 longest_row_chars: input_summary.longest_row_chars,
283 }
284 }
285
286 #[ztracing::instrument(skip_all)]
287 pub(crate) fn chunks<'a>(
288 &'a self,
289 range: Range<TabPoint>,
290 language_aware: LanguageAwareStyling,
291 highlights: Highlights<'a>,
292 ) -> TabChunks<'a> {
293 let (input_start, expanded_char_column, to_next_stop) =
294 self.tab_point_to_fold_point(range.start, Bias::Left);
295 let input_column = input_start.column();
296 let input_start = input_start.to_offset(&self.fold_snapshot);
297 let input_end = self
298 .tab_point_to_fold_point(range.end, Bias::Right)
299 .0
300 .to_offset(&self.fold_snapshot);
301 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
302 range.end.column() - range.start.column()
303 } else {
304 to_next_stop
305 };
306
307 TabChunks {
308 snapshot: self,
309 fold_chunks: self.fold_snapshot.chunks(
310 input_start..input_end,
311 language_aware,
312 highlights,
313 ),
314 input_column,
315 column: expanded_char_column,
316 max_expansion_column: self.max_expansion_column,
317 output_position: range.start.0,
318 max_output_position: range.end.0,
319 tab_size: self.tab_size,
320 chunk: Chunk {
321 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
322 is_tab: true,
323 chars: 1u128.unbounded_shl(to_next_stop) - 1,
324 ..Default::default()
325 },
326 inside_leading_tab: to_next_stop > 0,
327 }
328 }
329
330 #[ztracing::instrument(skip_all)]
331 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
332 self.fold_snapshot.row_infos(row)
333 }
334
335 #[cfg(test)]
336 #[ztracing::instrument(skip_all)]
337 pub fn text(&self) -> String {
338 self.chunks(
339 TabPoint::zero()..self.max_point(),
340 LanguageAwareStyling {
341 tree_sitter: false,
342 diagnostics: false,
343 },
344 Highlights::default(),
345 )
346 .map(|chunk| chunk.text)
347 .collect()
348 }
349
350 #[ztracing::instrument(skip_all)]
351 pub fn max_point(&self) -> TabPoint {
352 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
353 }
354
355 #[ztracing::instrument(skip_all)]
356 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
357 self.fold_point_to_tab_point(
358 self.fold_snapshot
359 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
360 )
361 }
362
363 #[ztracing::instrument(skip_all)]
364 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
365 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
366 let tab_cursor = TabStopCursor::new(chunks);
367 let expanded = self.expand_tabs(tab_cursor, input.column());
368 TabPoint::new(input.row(), expanded)
369 }
370
371 #[ztracing::instrument(skip_all)]
372 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
373 TabPointCursor { this: self }
374 }
375
376 #[ztracing::instrument(skip_all)]
377 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
378 let chunks = self
379 .fold_snapshot
380 .chunks_at(FoldPoint::new(output.row(), 0));
381
382 let tab_cursor = TabStopCursor::new(chunks);
383 let expanded = output.column();
384 let (collapsed, expanded_char_column, to_next_stop) =
385 self.collapse_tabs(tab_cursor, expanded, bias);
386
387 (
388 FoldPoint::new(output.row(), collapsed),
389 expanded_char_column,
390 to_next_stop,
391 )
392 }
393
394 #[ztracing::instrument(skip_all)]
395 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
396 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
397 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
398 self.fold_point_to_tab_point(fold_point)
399 }
400
401 #[ztracing::instrument(skip_all)]
402 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
403 let fold_point = self.tab_point_to_fold_point(point, bias).0;
404 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
405 self.fold_snapshot
406 .inlay_snapshot
407 .to_buffer_point(inlay_point)
408 }
409
410 #[ztracing::instrument(skip_all)]
411 fn expand_tabs<'a>(&self, mut cursor: TabStopCursor<'a>, column: u32) -> u32 {
412 // we only ever act on a single row at a time
413 // so the main difference is that other layers build a transform sumtree, and can then just run through that
414 // we cant quite do this here, as we need to work with the previous layer chunk to understand the tabs of the corresponding row
415 // we can still do forward searches for this though, we search for a row, then traverse the column up to where we need to be
416 let tab_size = self.tab_size.get();
417
418 let end_column = column.min(self.max_expansion_column);
419 let mut seek_target = end_column;
420 let mut tab_count = 0;
421 let mut expanded_tab_len = 0;
422
423 while let Some(tab_stop) = cursor.seek_forward(seek_target) {
424 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
425 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
426 tab_count += 1;
427 expanded_tab_len += tab_len;
428
429 seek_target = end_column - cursor.byte_offset;
430 }
431
432 let left_over_char_bytes = if !cursor.is_char_boundary() {
433 cursor.bytes_until_next_char().unwrap_or(0) as u32
434 } else {
435 0
436 };
437
438 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
439 let expanded_bytes =
440 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
441
442 expanded_bytes + column.saturating_sub(collapsed_bytes)
443 }
444
445 #[ztracing::instrument(skip_all)]
446 fn collapse_tabs<'a>(
447 &self,
448 mut cursor: TabStopCursor<'a>,
449 column: u32,
450 bias: Bias,
451 ) -> (u32, u32, u32) {
452 let tab_size = self.tab_size.get();
453 let mut collapsed_column = column;
454 let mut seek_target = column.min(self.max_expansion_column);
455 let mut tab_count = 0;
456 let mut expanded_tab_len = 0;
457
458 while let Some(tab_stop) = cursor.seek_forward(seek_target) {
459 // Calculate how much we want to expand this tab stop (into spaces)
460 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
461 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
462 // Increment tab count
463 tab_count += 1;
464 // The count of how many spaces we've added to this line in place of tab bytes
465 expanded_tab_len += tab_len;
466
467 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
468 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
469
470 // Did we expand past the search target?
471 if expanded_bytes > column {
472 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
473 // We expanded past the search target, so need to account for the offshoot
474 expanded_chars -= expanded_bytes - column;
475 return match bias {
476 Bias::Left => (
477 cursor.byte_offset() - 1,
478 expanded_chars,
479 expanded_bytes - column,
480 ),
481 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
482 };
483 } else {
484 // otherwise we only want to move the cursor collapse column forward
485 collapsed_column = collapsed_column - tab_len + 1;
486 seek_target = (collapsed_column - cursor.byte_offset)
487 .min(self.max_expansion_column - cursor.byte_offset);
488 }
489 }
490
491 let collapsed_bytes = cursor.byte_offset();
492 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
493 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
494 (
495 collapsed_bytes + column.saturating_sub(expanded_bytes),
496 expanded_chars,
497 0,
498 )
499 }
500}
501
502// todo(lw): Implement TabPointCursor properly
503pub struct TabPointCursor<'this> {
504 this: &'this TabSnapshot,
505}
506
507impl TabPointCursor<'_> {
508 #[ztracing::instrument(skip_all)]
509 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
510 self.this.fold_point_to_tab_point(point)
511 }
512}
513
514#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
515pub struct TabPoint(pub Point);
516
517impl TabPoint {
518 pub fn new(row: u32, column: u32) -> Self {
519 Self(Point::new(row, column))
520 }
521
522 pub fn zero() -> Self {
523 Self::new(0, 0)
524 }
525
526 pub fn row(self) -> u32 {
527 self.0.row
528 }
529
530 pub fn column(self) -> u32 {
531 self.0.column
532 }
533}
534
535impl From<Point> for TabPoint {
536 fn from(point: Point) -> Self {
537 Self(point)
538 }
539}
540
541pub type TabEdit = text::Edit<TabPoint>;
542
543#[derive(Clone, Debug, Default, Eq, PartialEq)]
544pub struct TextSummary {
545 pub lines: Point,
546 pub first_line_chars: u32,
547 pub last_line_chars: u32,
548 pub longest_row: u32,
549 pub longest_row_chars: u32,
550}
551
552impl<'a> From<&'a str> for TextSummary {
553 #[ztracing::instrument(skip_all)]
554 fn from(text: &'a str) -> Self {
555 let sum = text::TextSummary::from(text);
556
557 TextSummary {
558 lines: sum.lines,
559 first_line_chars: sum.first_line_chars,
560 last_line_chars: sum.last_line_chars,
561 longest_row: sum.longest_row,
562 longest_row_chars: sum.longest_row_chars,
563 }
564 }
565}
566
567impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
568 #[ztracing::instrument(skip_all)]
569 fn add_assign(&mut self, other: &'a Self) {
570 let joined_chars = self.last_line_chars + other.first_line_chars;
571 if joined_chars > self.longest_row_chars {
572 self.longest_row = self.lines.row;
573 self.longest_row_chars = joined_chars;
574 }
575 if other.longest_row_chars > self.longest_row_chars {
576 self.longest_row = self.lines.row + other.longest_row;
577 self.longest_row_chars = other.longest_row_chars;
578 }
579
580 if self.lines.row == 0 {
581 self.first_line_chars += other.first_line_chars;
582 }
583
584 if other.lines.row == 0 {
585 self.last_line_chars += other.first_line_chars;
586 } else {
587 self.last_line_chars = other.last_line_chars;
588 }
589
590 self.lines += &other.lines;
591 }
592}
593
594pub struct TabChunks<'a> {
595 snapshot: &'a TabSnapshot,
596 max_expansion_column: u32,
597 max_output_position: Point,
598 tab_size: NonZeroU32,
599 // region: iteration state
600 fold_chunks: FoldChunks<'a>,
601 chunk: Chunk<'a>,
602 column: u32,
603 output_position: Point,
604 input_column: u32,
605 inside_leading_tab: bool,
606 // endregion: iteration state
607}
608
609impl TabChunks<'_> {
610 #[ztracing::instrument(skip_all)]
611 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
612 let (input_start, expanded_char_column, to_next_stop) = self
613 .snapshot
614 .tab_point_to_fold_point(range.start, Bias::Left);
615 let input_column = input_start.column();
616 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
617 let input_end = self
618 .snapshot
619 .tab_point_to_fold_point(range.end, Bias::Right)
620 .0
621 .to_offset(&self.snapshot.fold_snapshot);
622 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
623 range.end.column() - range.start.column()
624 } else {
625 to_next_stop
626 };
627
628 self.fold_chunks.seek(input_start..input_end);
629 self.input_column = input_column;
630 self.column = expanded_char_column;
631 self.output_position = range.start.0;
632 self.max_output_position = range.end.0;
633 self.chunk = Chunk {
634 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
635 is_tab: true,
636 chars: 1u128.unbounded_shl(to_next_stop) - 1,
637 ..Default::default()
638 };
639 self.inside_leading_tab = to_next_stop > 0;
640 }
641}
642
643impl<'a> Iterator for TabChunks<'a> {
644 type Item = Chunk<'a>;
645
646 #[ztracing::instrument(skip_all)]
647 fn next(&mut self) -> Option<Self::Item> {
648 while self.chunk.text.is_empty() {
649 let chunk = self.fold_chunks.next()?;
650 self.chunk = chunk;
651 if self.inside_leading_tab {
652 self.chunk.text = &self.chunk.text[1..];
653 self.chunk.tabs >>= 1;
654 self.chunk.chars >>= 1;
655 self.chunk.newlines >>= 1;
656 self.inside_leading_tab = false;
657 self.input_column += 1;
658 }
659 }
660
661 if self.chunk.tabs & 1 != 0 {
662 self.chunk.text = &self.chunk.text[1..];
663 self.chunk.tabs >>= 1;
664 self.chunk.chars >>= 1;
665 self.chunk.newlines >>= 1;
666
667 let tab_size = if self.input_column < self.max_expansion_column {
668 self.tab_size.get()
669 } else {
670 1
671 };
672 let mut len = tab_size - self.column % tab_size;
673 let next_output_position = cmp::min(
674 self.output_position + Point::new(0, len),
675 self.max_output_position,
676 );
677 len = next_output_position.column - self.output_position.column;
678 self.column += len;
679 self.input_column += 1;
680 self.output_position = next_output_position;
681
682 return Some(Chunk {
683 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
684 is_tab: true,
685 chars: 1u128.unbounded_shl(len) - 1,
686 tabs: 0,
687 newlines: 0,
688 ..self.chunk.clone()
689 });
690 }
691
692 // Fast path: no tabs in the remaining chunk, return it directly
693 if self.chunk.tabs == 0 {
694 let chunk = self.chunk.clone();
695 self.chunk.text = "";
696 self.chunk.tabs = 0;
697 self.chunk.chars = 0;
698 self.chunk.newlines = 0;
699 let chunk_len = chunk.text.len() as u32;
700
701 let newline_count = chunk.newlines.count_ones();
702 if newline_count > 0 {
703 let last_newline_bit = 128 - chunk.newlines.leading_zeros();
704 let chars_after_last_newline =
705 chunk.chars.unbounded_shr(last_newline_bit).count_ones();
706 let bytes_after_last_newline = chunk_len - last_newline_bit;
707
708 self.column = chars_after_last_newline;
709 self.input_column = bytes_after_last_newline;
710 self.output_position = Point::new(
711 self.output_position.row + newline_count,
712 bytes_after_last_newline,
713 );
714 } else {
715 let char_count = chunk.chars.count_ones();
716 self.column += char_count;
717 if !self.inside_leading_tab {
718 self.input_column += chunk_len;
719 }
720 self.output_position.column += chunk_len;
721 }
722
723 return Some(chunk);
724 }
725
726 // Split at the next tab position
727 let prefix_len = self.chunk.tabs.trailing_zeros() as usize;
728 let (prefix, suffix) = self.chunk.text.split_at(prefix_len);
729
730 let mask = 1u128.unbounded_shl(prefix_len as u32).wrapping_sub(1);
731 let prefix_chars = self.chunk.chars & mask;
732 let prefix_newlines = self.chunk.newlines & mask;
733
734 self.chunk.text = suffix;
735 self.chunk.tabs = self.chunk.tabs.unbounded_shr(prefix_len as u32);
736 self.chunk.chars = self.chunk.chars.unbounded_shr(prefix_len as u32);
737 self.chunk.newlines = self.chunk.newlines.unbounded_shr(prefix_len as u32);
738
739 let newline_count = prefix_newlines.count_ones();
740 if newline_count > 0 {
741 let last_newline_bit = 128 - prefix_newlines.leading_zeros();
742 let chars_after_last_newline =
743 prefix_chars.unbounded_shr(last_newline_bit).count_ones();
744 let bytes_after_last_newline = prefix_len as u32 - last_newline_bit;
745
746 self.column = chars_after_last_newline;
747 self.input_column = bytes_after_last_newline;
748 self.output_position = Point::new(
749 self.output_position.row + newline_count,
750 bytes_after_last_newline,
751 );
752 } else {
753 let char_count = prefix_chars.count_ones();
754 self.column += char_count;
755 if !self.inside_leading_tab {
756 self.input_column += prefix_len as u32;
757 }
758 self.output_position.column += prefix_len as u32;
759 }
760
761 Some(Chunk {
762 text: prefix,
763 chars: prefix_chars,
764 tabs: 0,
765 newlines: prefix_newlines,
766 ..self.chunk.clone()
767 })
768 }
769}
770
771struct TabStopCursor<'a> {
772 chunks: FoldChunks<'a>,
773 byte_offset: u32,
774 char_offset: u32,
775 /// Chunk
776 /// last tab position iterated through
777 current_chunk: Option<(TabStopChunk<'a>, u32)>,
778}
779
780struct TabStopChunk<'a> {
781 chars: u128,
782 text: &'a str,
783 tabs: u128,
784}
785
786impl<'a> TabStopCursor<'a> {
787 fn new(chunks: FoldChunks<'a>) -> Self {
788 Self {
789 chunks,
790 byte_offset: 0,
791 char_offset: 0,
792 current_chunk: None,
793 }
794 }
795
796 fn bytes_until_next_char(&self) -> Option<usize> {
797 self.current_chunk.as_ref().map(|&(ref chunk, idx)| {
798 let higher_chars = chunk.chars.unbounded_shr(idx + 1);
799
800 if higher_chars != 0 {
801 higher_chars.trailing_zeros() as usize + 1
802 } else {
803 chunk.text.len() - idx as usize
804 }
805 })
806 }
807
808 fn is_char_boundary(&self) -> bool {
809 self.current_chunk
810 .as_ref()
811 .is_some_and(|&(ref chunk, idx)| {
812 (1u128.unbounded_shl(idx) & chunk.chars) != 0 || idx as usize == chunk.text.len()
813 })
814 }
815
816 /// distance: length to move forward while searching for the next tab stop
817 #[ztracing::instrument(skip_all)]
818 fn seek_forward(&mut self, distance: u32) -> Option<TabStop> {
819 if distance == 0 {
820 return None;
821 }
822
823 let mut distance_remaining = distance;
824
825 while let Some((mut chunk, chunk_position)) = self.current_chunk.take().or_else(|| {
826 self.chunks.next().map(|chunk| {
827 (
828 TabStopChunk {
829 chars: chunk.chars,
830 text: chunk.text,
831 tabs: chunk.tabs,
832 },
833 0,
834 )
835 })
836 }) {
837 let chunk_len = chunk.text.len() as u32;
838
839 if chunk.tabs == 0 {
840 let chunk_remaining = chunk_len - chunk_position;
841 if chunk_remaining >= distance_remaining {
842 let end = chunk_position + distance_remaining;
843 self.byte_offset += distance_remaining;
844 self.char_offset +=
845 count_chars_in_byte_range(chunk_position..(end - 1), chunk.chars);
846 if end < 128 {
847 self.current_chunk = Some((chunk, end));
848 }
849 return None;
850 }
851
852 self.byte_offset += chunk_remaining;
853 self.char_offset +=
854 count_chars_in_byte_range(chunk_position..(chunk_len - 1), chunk.chars);
855 distance_remaining -= chunk_remaining;
856 continue;
857 }
858
859 let tab_end = chunk.tabs.trailing_zeros() + 1;
860 let bytes_to_tab = tab_end - chunk_position;
861
862 if bytes_to_tab > distance_remaining {
863 let end = chunk_position + distance_remaining;
864 self.byte_offset += distance_remaining;
865 self.char_offset +=
866 count_chars_in_byte_range(chunk_position..(end - 1), chunk.chars);
867 self.current_chunk = Some((chunk, end));
868 return None;
869 }
870
871 self.byte_offset += bytes_to_tab;
872 self.char_offset +=
873 count_chars_in_byte_range(chunk_position..(tab_end - 1), chunk.chars);
874
875 let tabstop = TabStop {
876 char_offset: self.char_offset,
877 byte_offset: self.byte_offset,
878 };
879
880 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
881
882 if tab_end != chunk_len {
883 self.current_chunk = Some((chunk, tab_end));
884 }
885
886 return Some(tabstop);
887 }
888
889 None
890 }
891
892 fn byte_offset(&self) -> u32 {
893 self.byte_offset
894 }
895
896 fn char_offset(&self) -> u32 {
897 self.char_offset
898 }
899}
900
901#[inline(always)]
902fn count_chars_in_byte_range(range: Range<u32>, bitmap: u128) -> u32 {
903 let low_mask = u128::MAX << range.start;
904 let high_mask = u128::MAX >> (127 - range.end);
905 (bitmap & low_mask & high_mask).count_ones()
906}
907
908#[derive(Clone, Copy, Debug, PartialEq, Eq)]
909struct TabStop {
910 char_offset: u32,
911 byte_offset: u32,
912}
913
914#[cfg(test)]
915mod tests {
916 use std::mem;
917
918 use super::*;
919 use crate::{
920 MultiBuffer,
921 display_map::{
922 fold_map::{FoldMap, FoldOffset, FoldPlaceholder},
923 inlay_map::InlayMap,
924 },
925 };
926 use multi_buffer::MultiBufferOffset;
927 use rand::{Rng, prelude::StdRng};
928 use util;
929
930 impl TabSnapshot {
931 fn expected_collapse_tabs(
932 &self,
933 chars: impl Iterator<Item = char>,
934 column: u32,
935 bias: Bias,
936 ) -> (u32, u32, u32) {
937 let tab_size = self.tab_size.get();
938
939 let mut expanded_bytes = 0;
940 let mut expanded_chars = 0;
941 let mut collapsed_bytes = 0;
942 for c in chars {
943 if expanded_bytes >= column {
944 break;
945 }
946 if collapsed_bytes >= self.max_expansion_column {
947 break;
948 }
949
950 if c == '\t' {
951 let tab_len = tab_size - (expanded_chars % tab_size);
952 expanded_chars += tab_len;
953 expanded_bytes += tab_len;
954 if expanded_bytes > column {
955 expanded_chars -= expanded_bytes - column;
956 return match bias {
957 Bias::Left => {
958 (collapsed_bytes, expanded_chars, expanded_bytes - column)
959 }
960 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
961 };
962 }
963 } else {
964 expanded_chars += 1;
965 expanded_bytes += c.len_utf8() as u32;
966 }
967
968 if expanded_bytes > column && matches!(bias, Bias::Left) {
969 expanded_chars -= 1;
970 break;
971 }
972
973 collapsed_bytes += c.len_utf8() as u32;
974 }
975
976 (
977 collapsed_bytes + column.saturating_sub(expanded_bytes),
978 expanded_chars,
979 0,
980 )
981 }
982
983 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
984 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
985 let expanded = self.expected_expand_tabs(chars, input.column());
986 TabPoint::new(input.row(), expanded)
987 }
988
989 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
990 let tab_size = self.tab_size.get();
991
992 let mut expanded_chars = 0;
993 let mut expanded_bytes = 0;
994 let mut collapsed_bytes = 0;
995 let end_column = column.min(self.max_expansion_column);
996 for c in chars {
997 if collapsed_bytes >= end_column {
998 break;
999 }
1000 if c == '\t' {
1001 let tab_len = tab_size - expanded_chars % tab_size;
1002 expanded_bytes += tab_len;
1003 expanded_chars += tab_len;
1004 } else {
1005 expanded_bytes += c.len_utf8() as u32;
1006 expanded_chars += 1;
1007 }
1008 collapsed_bytes += c.len_utf8() as u32;
1009 }
1010
1011 expanded_bytes + column.saturating_sub(collapsed_bytes)
1012 }
1013
1014 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
1015 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
1016 let expanded = output.column();
1017 let (collapsed, expanded_char_column, to_next_stop) =
1018 self.expected_collapse_tabs(chars, expanded, bias);
1019 (
1020 FoldPoint::new(output.row(), collapsed),
1021 expanded_char_column,
1022 to_next_stop,
1023 )
1024 }
1025 }
1026
1027 #[gpui::test]
1028 fn test_expand_tabs(cx: &mut gpui::App) {
1029 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
1030
1031 let buffer = MultiBuffer::build_simple(input, cx);
1032 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1033 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1034 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1035 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1036
1037 for (ix, _) in input.char_indices() {
1038 let fold_point = FoldPoint::new(0, ix as u32);
1039
1040 assert_eq!(
1041 tab_snapshot.expected_to_tab_point(fold_point),
1042 tab_snapshot.fold_point_to_tab_point(fold_point),
1043 "Failed with fold_point at column {ix}"
1044 );
1045 }
1046 }
1047
1048 #[gpui::test]
1049 fn test_collapse_tabs(cx: &mut gpui::App) {
1050 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
1051
1052 let buffer = MultiBuffer::build_simple(input, cx);
1053 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1054 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1055 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1056 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1057
1058 for (ix, _) in input.char_indices() {
1059 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
1060
1061 assert_eq!(
1062 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
1063 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
1064 "Failed with tab_point at column {ix}"
1065 );
1066 assert_eq!(
1067 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
1068 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
1069 "Failed with tab_point at column {ix}"
1070 );
1071
1072 assert_eq!(
1073 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
1074 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
1075 "Failed with tab_point at column {ix}"
1076 );
1077 assert_eq!(
1078 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
1079 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
1080 "Failed with tab_point at column {ix}"
1081 );
1082 }
1083 }
1084
1085 #[gpui::test]
1086 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
1087 // This test reproduces a specific panic where to_fold_point returns incorrect results
1088 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
1089
1090 let text = "γ\tw⭐\n🍐🍗 \t";
1091 let buffer = MultiBuffer::build_simple(text, cx);
1092 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1093 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1094 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1095 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1096
1097 // This should panic with the expected vs actual mismatch
1098 let tab_point = TabPoint::new(0, 9);
1099 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
1100 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
1101
1102 assert_eq!(result, expected);
1103 }
1104
1105 #[gpui::test(iterations = 100)]
1106 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
1107 // Generate random input string with up to 200 characters including tabs
1108 // to stay within the MAX_EXPANSION_COLUMN limit of 256
1109 let len = rng.random_range(0..=2048);
1110 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1111 let mut input = String::with_capacity(len);
1112
1113 for _ in 0..len {
1114 if rng.random_bool(0.1) {
1115 // 10% chance of inserting a tab
1116 input.push('\t');
1117 } else {
1118 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
1119 let ch = loop {
1120 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1121 let ch = ascii_code as u8 as char;
1122 if ch != '\t' {
1123 break ch;
1124 }
1125 };
1126 input.push(ch);
1127 }
1128 }
1129
1130 let buffer = MultiBuffer::build_simple(&input, cx);
1131 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1132 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1133 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1134 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1135 tab_snapshot.max_expansion_column = rng.random_range(0..323);
1136 tab_snapshot.tab_size = tab_size;
1137
1138 for (ix, _) in input.char_indices() {
1139 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
1140
1141 assert_eq!(
1142 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
1143 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
1144 "Failed with input: {}, with idx: {ix}",
1145 input
1146 );
1147 assert_eq!(
1148 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
1149 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
1150 "Failed with input: {}, with idx: {ix}",
1151 input
1152 );
1153
1154 assert_eq!(
1155 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
1156 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
1157 "Failed with input: {}, with idx: {ix}",
1158 input
1159 );
1160 assert_eq!(
1161 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
1162 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
1163 "Failed with input: {}, with idx: {ix}",
1164 input
1165 );
1166 }
1167 }
1168
1169 #[gpui::test]
1170 fn test_long_lines(cx: &mut gpui::App) {
1171 let max_expansion_column = 12;
1172 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
1173 let output = "A BC DEF G HI J K L M";
1174
1175 let buffer = MultiBuffer::build_simple(input, cx);
1176 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1177 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1178 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1179 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1180
1181 tab_snapshot.max_expansion_column = max_expansion_column;
1182 assert_eq!(tab_snapshot.text(), output);
1183
1184 for (ix, c) in input.char_indices() {
1185 assert_eq!(
1186 tab_snapshot
1187 .chunks(
1188 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
1189 LanguageAwareStyling {
1190 tree_sitter: false,
1191 diagnostics: false,
1192 },
1193 Highlights::default(),
1194 )
1195 .map(|c| c.text)
1196 .collect::<String>(),
1197 &output[ix..],
1198 "text from index {ix}"
1199 );
1200
1201 if c != '\t' {
1202 let input_point = Point::new(0, ix as u32);
1203 let output_point = Point::new(0, output.find(c).unwrap() as u32);
1204 assert_eq!(
1205 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
1206 TabPoint(output_point),
1207 "to_tab_point({input_point:?})"
1208 );
1209 assert_eq!(
1210 tab_snapshot
1211 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
1212 .0,
1213 FoldPoint(input_point),
1214 "to_fold_point({output_point:?})"
1215 );
1216 }
1217 }
1218 }
1219
1220 #[gpui::test]
1221 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
1222 let max_expansion_column = 8;
1223 let input = "abcdefg⋯hij";
1224
1225 let buffer = MultiBuffer::build_simple(input, cx);
1226 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1227 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1228 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1229 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1230
1231 tab_snapshot.max_expansion_column = max_expansion_column;
1232 assert_eq!(tab_snapshot.text(), input);
1233 }
1234
1235 #[gpui::test]
1236 fn test_marking_tabs(cx: &mut gpui::App) {
1237 let input = "\t \thello";
1238
1239 let buffer = MultiBuffer::build_simple(input, cx);
1240 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1241 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1242 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1243 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1244
1245 assert_eq!(
1246 chunks(&tab_snapshot, TabPoint::zero()),
1247 vec![
1248 (" ".to_string(), true),
1249 (" ".to_string(), false),
1250 (" ".to_string(), true),
1251 ("hello".to_string(), false),
1252 ]
1253 );
1254 assert_eq!(
1255 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1256 vec![
1257 (" ".to_string(), true),
1258 (" ".to_string(), false),
1259 (" ".to_string(), true),
1260 ("hello".to_string(), false),
1261 ]
1262 );
1263
1264 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1265 let mut chunks = Vec::new();
1266 let mut was_tab = false;
1267 let mut text = String::new();
1268 for chunk in snapshot.chunks(
1269 start..snapshot.max_point(),
1270 LanguageAwareStyling {
1271 tree_sitter: false,
1272 diagnostics: false,
1273 },
1274 Highlights::default(),
1275 ) {
1276 if chunk.is_tab != was_tab {
1277 if !text.is_empty() {
1278 chunks.push((mem::take(&mut text), was_tab));
1279 }
1280 was_tab = chunk.is_tab;
1281 }
1282 text.push_str(chunk.text);
1283 }
1284
1285 if !text.is_empty() {
1286 chunks.push((text, was_tab));
1287 }
1288 chunks
1289 }
1290 }
1291
1292 #[gpui::test]
1293 fn test_empty_chunk_after_leading_tab_trim(cx: &mut gpui::App) {
1294 // We fold "hello" (offsets 1..6) so the fold map creates a
1295 // transform boundary at offset 1, producing a 1-byte fold chunk
1296 // for the tab.
1297 let text = "\thello";
1298 let buffer = MultiBuffer::build_simple(text, cx);
1299 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1300 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1301 let mut fold_map = FoldMap::new(inlay_snapshot.clone()).0;
1302
1303 let (mut writer, _, _) = fold_map.write(inlay_snapshot.clone(), vec![]);
1304 writer.fold(vec![(
1305 MultiBufferOffset(1)..MultiBufferOffset(6),
1306 FoldPlaceholder::test(),
1307 )]);
1308 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1309
1310 let tab_size = NonZeroU32::new(4).unwrap();
1311 let (_, tab_snapshot) = TabMap::new(fold_snapshot, tab_size);
1312
1313 // The tab at column 0 expands to 4 spaces (columns 0‥4).
1314 // Seek starting at column 2 (middle of that tab) so that
1315 // `inside_leading_tab = true` and `to_next_stop = 2`.
1316 // Set the end just past the tab expansion so the iterator must
1317 // process the tab byte from the fold chunk.
1318 let max = tab_snapshot.max_point();
1319 let start = TabPoint::new(0, 2);
1320 let end = max;
1321
1322 // This should not panic.
1323 let result: String = tab_snapshot
1324 .chunks(
1325 start..end,
1326 LanguageAwareStyling {
1327 tree_sitter: false,
1328 diagnostics: false,
1329 },
1330 Highlights::default(),
1331 )
1332 .map(|c| c.text)
1333 .collect();
1334 assert!(!result.is_empty());
1335 }
1336
1337 #[gpui::test(iterations = 100)]
1338 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1339 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1340 let len = rng.random_range(0..30);
1341 let buffer = if rng.random() {
1342 let text = util::RandomCharIter::new(&mut rng)
1343 .take(len)
1344 .collect::<String>();
1345 MultiBuffer::build_simple(&text, cx)
1346 } else {
1347 MultiBuffer::build_random(&mut rng, cx)
1348 };
1349 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1350 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1351
1352 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1353 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1354 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1355 fold_map.randomly_mutate(&mut rng);
1356 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1357 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1358 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1359 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1360
1361 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1362 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1363
1364 let text = text::Rope::from(tabs_snapshot.text().as_str());
1365 log::info!(
1366 "TabMap text (tab size: {}): {:?}",
1367 tab_size,
1368 tabs_snapshot.text(),
1369 );
1370
1371 for _ in 0..5 {
1372 let end_row = rng.random_range(0..=text.max_point().row);
1373 let end_column = rng.random_range(0..=text.line_len(end_row));
1374 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1375 let start_row = rng.random_range(0..=text.max_point().row);
1376 let start_column = rng.random_range(0..=text.line_len(start_row));
1377 let mut start =
1378 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1379 if start > end {
1380 mem::swap(&mut start, &mut end);
1381 }
1382
1383 let expected_text = text
1384 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1385 .collect::<String>();
1386 let expected_summary = TextSummary::from(expected_text.as_str());
1387 assert_eq!(
1388 tabs_snapshot
1389 .chunks(
1390 start..end,
1391 LanguageAwareStyling {
1392 tree_sitter: false,
1393 diagnostics: false,
1394 },
1395 Highlights::default()
1396 )
1397 .map(|c| c.text)
1398 .collect::<String>(),
1399 expected_text,
1400 "chunks({:?}..{:?})",
1401 start,
1402 end
1403 );
1404
1405 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1406 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1407 actual_summary.longest_row = expected_summary.longest_row;
1408 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1409 }
1410 assert_eq!(actual_summary, expected_summary);
1411 }
1412
1413 for row in 0..=text.max_point().row {
1414 assert_eq!(
1415 tabs_snapshot.line_len(row),
1416 text.line_len(row),
1417 "line_len({row})"
1418 );
1419 }
1420 }
1421
1422 #[gpui::test(iterations = 100)]
1423 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1424 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1425 let len = rng.random_range(0..=2000);
1426
1427 // Generate random text using RandomCharIter
1428 let text = util::RandomCharIter::new(&mut rng)
1429 .take(len)
1430 .collect::<String>();
1431
1432 // Create buffer and tab map
1433 let buffer = MultiBuffer::build_simple(&text, cx);
1434 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1435 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1436 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1437 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1438
1439 let mut next_inlay_id = 0;
1440 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1441 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1442 let max_fold_point = fold_snapshot.max_point();
1443 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1444
1445 // Test random fold points
1446 for _ in 0..50 {
1447 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1448 // Generate random fold point
1449 let row = rng.random_range(0..=max_fold_point.row());
1450 let max_column = if row < max_fold_point.row() {
1451 fold_snapshot.line_len(row)
1452 } else {
1453 max_fold_point.column()
1454 };
1455 let column = rng.random_range(0..=max_column + 10);
1456 let fold_point = FoldPoint::new(row, column);
1457
1458 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1459 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1460
1461 assert_eq!(
1462 actual, expected,
1463 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1464 fold_point, text
1465 );
1466 }
1467 }
1468
1469 #[gpui::test]
1470 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1471 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1472 let buffer = MultiBuffer::build_simple(text, cx);
1473 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1474 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1475 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1476 let chunks = fold_snapshot.chunks(
1477 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1478 LanguageAwareStyling {
1479 tree_sitter: false,
1480 diagnostics: false,
1481 },
1482 Default::default(),
1483 );
1484 let mut cursor = TabStopCursor::new(chunks);
1485 assert!(cursor.seek_forward(0).is_none());
1486 let mut tab_stops = Vec::new();
1487
1488 let mut all_tab_stops = Vec::new();
1489 let mut byte_offset = 0;
1490 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1491 byte_offset += ch.len_utf8() as u32;
1492
1493 if ch == '\t' {
1494 all_tab_stops.push(TabStop {
1495 byte_offset,
1496 char_offset: offset as u32 + 1,
1497 });
1498 }
1499 }
1500
1501 while let Some(tab_stop) = cursor.seek_forward(u32::MAX) {
1502 tab_stops.push(tab_stop);
1503 }
1504 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1505
1506 assert_eq!(cursor.byte_offset(), byte_offset);
1507 }
1508
1509 #[gpui::test]
1510 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1511 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1512
1513 let buffer = MultiBuffer::build_simple(input, cx);
1514 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1515 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1516 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1517
1518 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1519 let mut cursor = TabStopCursor::new(chunks);
1520
1521 let mut actual_tab_stops = Vec::new();
1522
1523 let mut expected_tab_stops = Vec::new();
1524 let mut byte_offset = 0;
1525 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1526 byte_offset += ch.len_utf8() as u32;
1527
1528 if ch == '\t' {
1529 expected_tab_stops.push(TabStop {
1530 byte_offset,
1531 char_offset: offset as u32 + 1,
1532 });
1533 }
1534 }
1535
1536 while let Some(tab_stop) = cursor.seek_forward(u32::MAX) {
1537 actual_tab_stops.push(tab_stop);
1538 }
1539 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1540
1541 assert_eq!(cursor.byte_offset(), byte_offset);
1542 }
1543
1544 #[gpui::test(iterations = 100)]
1545 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1546 // Generate random input string with up to 512 characters including tabs
1547 let len = rng.random_range(0..=2048);
1548 let mut input = String::with_capacity(len);
1549
1550 let mut skip_tabs = rng.random_bool(0.10);
1551 for idx in 0..len {
1552 if idx % 128 == 0 {
1553 skip_tabs = rng.random_bool(0.10);
1554 }
1555
1556 if rng.random_bool(0.15) && !skip_tabs {
1557 input.push('\t');
1558 } else {
1559 let ch = loop {
1560 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1561 let ch = ascii_code as u8 as char;
1562 if ch != '\t' {
1563 break ch;
1564 }
1565 };
1566 input.push(ch);
1567 }
1568 }
1569
1570 // Build the buffer and create cursor
1571 let buffer = MultiBuffer::build_simple(&input, cx);
1572 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1573 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1574 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1575
1576 // First, collect all expected tab positions
1577 let mut all_tab_stops = Vec::new();
1578 let mut byte_offset = 1;
1579 let mut char_offset = 1;
1580 for ch in buffer_snapshot.text().chars() {
1581 if ch == '\t' {
1582 all_tab_stops.push(TabStop {
1583 byte_offset,
1584 char_offset,
1585 });
1586 }
1587 byte_offset += ch.len_utf8() as u32;
1588 char_offset += 1;
1589 }
1590
1591 // Test with various distances
1592 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1593 // let distances = vec![150];
1594
1595 for distance in distances {
1596 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1597 let mut cursor = TabStopCursor::new(chunks);
1598
1599 let mut found_tab_stops = Vec::new();
1600 let mut position = distance;
1601 while let Some(tab_stop) = cursor.seek_forward(position) {
1602 found_tab_stops.push(tab_stop);
1603 position = distance - tab_stop.byte_offset;
1604 }
1605
1606 let expected_found_tab_stops: Vec<_> = all_tab_stops
1607 .iter()
1608 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1609 .cloned()
1610 .collect();
1611
1612 pretty_assertions::assert_eq!(
1613 found_tab_stops,
1614 expected_found_tab_stops,
1615 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1616 distance,
1617 input
1618 );
1619
1620 let final_position = cursor.byte_offset();
1621 if !found_tab_stops.is_empty() {
1622 let last_tab_stop = found_tab_stops.last().unwrap();
1623 assert!(
1624 final_position >= last_tab_stop.byte_offset,
1625 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1626 final_position,
1627 last_tab_stop.byte_offset,
1628 input
1629 );
1630 }
1631 }
1632 }
1633
1634 #[gpui::test]
1635 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1636 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1637 let buffer = MultiBuffer::build_simple(text, cx);
1638 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1639 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1640 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1641 let chunks = fold_snapshot.chunks(
1642 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1643 LanguageAwareStyling {
1644 tree_sitter: false,
1645 diagnostics: false,
1646 },
1647 Default::default(),
1648 );
1649 let mut cursor = TabStopCursor::new(chunks);
1650 assert!(cursor.seek_forward(0).is_none());
1651
1652 let mut expected_tab_stops = Vec::new();
1653 let mut byte_offset = 0;
1654 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1655 byte_offset += ch.len_utf8() as u32;
1656
1657 if ch == '\t' {
1658 expected_tab_stops.push(TabStop {
1659 byte_offset,
1660 char_offset: i as u32 + 1,
1661 });
1662 }
1663 }
1664
1665 let mut actual_tab_stops = Vec::new();
1666 while let Some(tab_stop) = cursor.seek_forward(u32::MAX) {
1667 actual_tab_stops.push(tab_stop);
1668 }
1669
1670 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1671
1672 assert_eq!(cursor.byte_offset(), byte_offset);
1673 }
1674
1675 #[gpui::test(iterations = 100)]
1676 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1677 // Generate random input string with up to 512 characters including tabs
1678 let len = rng.random_range(0..=2048);
1679 let input = util::RandomCharIter::new(&mut rng)
1680 .take(len)
1681 .collect::<String>();
1682
1683 // Build the buffer and create cursor
1684 let buffer = MultiBuffer::build_simple(&input, cx);
1685 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1686 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1687 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1688
1689 // First, collect all expected tab positions
1690 let mut all_tab_stops = Vec::new();
1691 let mut byte_offset = 0;
1692 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1693 byte_offset += ch.len_utf8() as u32;
1694 if ch == '\t' {
1695 all_tab_stops.push(TabStop {
1696 byte_offset,
1697 char_offset: i as u32 + 1,
1698 });
1699 }
1700 }
1701
1702 // Test with various distances
1703 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1704 let distances = vec![150];
1705
1706 for distance in distances {
1707 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1708 let mut cursor = TabStopCursor::new(chunks);
1709
1710 let mut found_tab_stops = Vec::new();
1711 let mut position = distance;
1712 while let Some(tab_stop) = cursor.seek_forward(position) {
1713 found_tab_stops.push(tab_stop);
1714 position = distance - tab_stop.byte_offset;
1715 }
1716
1717 let expected_found_tab_stops: Vec<_> = all_tab_stops
1718 .iter()
1719 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1720 .cloned()
1721 .collect();
1722
1723 pretty_assertions::assert_eq!(
1724 found_tab_stops,
1725 expected_found_tab_stops,
1726 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1727 distance,
1728 input
1729 );
1730
1731 let final_position = cursor.byte_offset();
1732 if !found_tab_stops.is_empty() {
1733 let last_tab_stop = found_tab_stops.last().unwrap();
1734 assert!(
1735 final_position >= last_tab_stop.byte_offset,
1736 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1737 final_position,
1738 last_tab_stop.byte_offset,
1739 input
1740 );
1741 }
1742 }
1743 }
1744}