1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 #[ztracing::instrument(skip_all)]
24 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
25 let snapshot = TabSnapshot {
26 fold_snapshot,
27 tab_size: tab_size.min(MAX_TABS),
28 max_expansion_column: MAX_EXPANSION_COLUMN,
29 version: 0,
30 };
31 (Self(snapshot.clone()), snapshot)
32 }
33
34 #[cfg(test)]
35 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
36 self.0.max_expansion_column = column;
37 self.0.clone()
38 }
39
40 #[ztracing::instrument(skip_all)]
41 pub fn sync(
42 &mut self,
43 fold_snapshot: FoldSnapshot,
44 mut fold_edits: Vec<FoldEdit>,
45 tab_size: NonZeroU32,
46 ) -> (TabSnapshot, Vec<TabEdit>) {
47 let old_snapshot = &mut self.0;
48 let mut new_snapshot = TabSnapshot {
49 fold_snapshot,
50 tab_size: tab_size.min(MAX_TABS),
51 max_expansion_column: old_snapshot.max_expansion_column,
52 version: old_snapshot.version,
53 };
54
55 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
56 new_snapshot.version += 1;
57 }
58
59 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
60 // Expand each edit to include the next tab on the same line as the edit,
61 // and any subsequent tabs on that line that moved across the tab expansion
62 // boundary.
63 for fold_edit in &mut fold_edits {
64 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
65 let old_end_row_successor_offset = cmp::min(
66 FoldPoint::new(old_end.row() + 1, 0),
67 old_snapshot.fold_snapshot.max_point(),
68 )
69 .to_offset(&old_snapshot.fold_snapshot);
70 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
71
72 let mut offset_from_edit = 0;
73 let mut first_tab_offset = None;
74 let mut last_tab_with_changed_expansion_offset = None;
75 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
76 fold_edit.old.end..old_end_row_successor_offset,
77 false,
78 Highlights::default(),
79 ) {
80 let mut remaining_tabs = chunk.tabs;
81 while remaining_tabs != 0 {
82 let ix = remaining_tabs.trailing_zeros();
83 let offset_from_edit = offset_from_edit + ix;
84 if first_tab_offset.is_none() {
85 first_tab_offset = Some(offset_from_edit);
86 }
87
88 let old_column = old_end.column() + offset_from_edit;
89 let new_column = new_end.column() + offset_from_edit;
90 let was_expanded = old_column < old_snapshot.max_expansion_column;
91 let is_expanded = new_column < new_snapshot.max_expansion_column;
92 if was_expanded != is_expanded {
93 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
94 } else if !was_expanded && !is_expanded {
95 break 'outer;
96 }
97
98 remaining_tabs &= remaining_tabs - 1;
99 }
100
101 offset_from_edit += chunk.text.len() as u32;
102 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
103 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
104 {
105 break;
106 }
107 }
108
109 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
110 fold_edit.old.end.0 += offset as usize + 1;
111 fold_edit.new.end.0 += offset as usize + 1;
112 }
113 }
114
115 let _old_alloc_ptr = fold_edits.as_ptr();
116 // Combine any edits that overlap due to the expansion.
117 let mut fold_edits = fold_edits.into_iter();
118 if let Some(mut first_edit) = fold_edits.next() {
119 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
120 #[allow(clippy::filter_map_identity)]
121 let mut v: Vec<_> = fold_edits
122 .scan(&mut first_edit, |state, edit| {
123 if state.old.end >= edit.old.start {
124 state.old.end = edit.old.end;
125 state.new.end = edit.new.end;
126 Some(None) // Skip this edit, it's merged
127 } else {
128 let new_state = edit;
129 let result = Some(Some(state.clone())); // Yield the previous edit
130 **state = new_state;
131 result
132 }
133 })
134 .filter_map(|x| x)
135 .collect();
136 v.push(first_edit);
137 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
138 v.into_iter()
139 .map(|fold_edit| {
140 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
141 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
142 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
143 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
144 TabEdit {
145 old: old_snapshot.fold_point_to_tab_point(old_start)
146 ..old_snapshot.fold_point_to_tab_point(old_end),
147 new: new_snapshot.fold_point_to_tab_point(new_start)
148 ..new_snapshot.fold_point_to_tab_point(new_end),
149 }
150 })
151 .collect()
152 } else {
153 vec![]
154 }
155 } else {
156 new_snapshot.version += 1;
157 vec![TabEdit {
158 old: TabPoint::zero()..old_snapshot.max_point(),
159 new: TabPoint::zero()..new_snapshot.max_point(),
160 }]
161 };
162 *old_snapshot = new_snapshot;
163 (old_snapshot.clone(), tab_edits)
164 }
165}
166
167#[derive(Clone)]
168pub struct TabSnapshot {
169 pub fold_snapshot: FoldSnapshot,
170 pub tab_size: NonZeroU32,
171 pub max_expansion_column: u32,
172 pub version: usize,
173}
174
175impl std::ops::Deref for TabSnapshot {
176 type Target = FoldSnapshot;
177
178 fn deref(&self) -> &Self::Target {
179 &self.fold_snapshot
180 }
181}
182
183impl TabSnapshot {
184 #[ztracing::instrument(skip_all)]
185 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
186 &self.fold_snapshot.inlay_snapshot.buffer
187 }
188
189 #[ztracing::instrument(skip_all)]
190 pub fn line_len(&self, row: u32) -> u32 {
191 let max_point = self.max_point();
192 if row < max_point.row() {
193 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
194 .0
195 .column
196 } else {
197 max_point.column()
198 }
199 }
200
201 #[ztracing::instrument(skip_all)]
202 pub fn text_summary(&self) -> TextSummary {
203 self.text_summary_for_range(TabPoint::zero()..self.max_point())
204 }
205
206 #[ztracing::instrument(skip_all, fields(rows))]
207 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
208 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
209 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
210 let input_summary = self
211 .fold_snapshot
212 .text_summary_for_range(input_start..input_end);
213
214 let line_end = if range.start.row() == range.end.row() {
215 range.end
216 } else {
217 self.max_point()
218 };
219 let first_line_chars = self
220 .chunks(range.start..line_end, false, Highlights::default())
221 .flat_map(|chunk| chunk.text.chars())
222 .take_while(|&c| c != '\n')
223 .count() as u32;
224
225 let last_line_chars = if range.start.row() == range.end.row() {
226 first_line_chars
227 } else {
228 self.chunks(
229 TabPoint::new(range.end.row(), 0)..range.end,
230 false,
231 Highlights::default(),
232 )
233 .flat_map(|chunk| chunk.text.chars())
234 .count() as u32
235 };
236
237 TextSummary {
238 lines: range.end.0 - range.start.0,
239 first_line_chars,
240 last_line_chars,
241 longest_row: input_summary.longest_row,
242 longest_row_chars: input_summary.longest_row_chars,
243 }
244 }
245
246 #[ztracing::instrument(skip_all)]
247 pub(crate) fn chunks<'a>(
248 &'a self,
249 range: Range<TabPoint>,
250 language_aware: bool,
251 highlights: Highlights<'a>,
252 ) -> TabChunks<'a> {
253 let (input_start, expanded_char_column, to_next_stop) =
254 self.tab_point_to_fold_point(range.start, Bias::Left);
255 let input_column = input_start.column();
256 let input_start = input_start.to_offset(&self.fold_snapshot);
257 let input_end = self
258 .tab_point_to_fold_point(range.end, Bias::Right)
259 .0
260 .to_offset(&self.fold_snapshot);
261 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
262 range.end.column() - range.start.column()
263 } else {
264 to_next_stop
265 };
266
267 TabChunks {
268 snapshot: self,
269 fold_chunks: self.fold_snapshot.chunks(
270 input_start..input_end,
271 language_aware,
272 highlights,
273 ),
274 input_column,
275 column: expanded_char_column,
276 max_expansion_column: self.max_expansion_column,
277 output_position: range.start.0,
278 max_output_position: range.end.0,
279 tab_size: self.tab_size,
280 chunk: Chunk {
281 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
282 is_tab: true,
283 chars: 1u128.unbounded_shl(to_next_stop) - 1,
284 ..Default::default()
285 },
286 inside_leading_tab: to_next_stop > 0,
287 }
288 }
289
290 #[ztracing::instrument(skip_all)]
291 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
292 self.fold_snapshot.row_infos(row)
293 }
294
295 #[cfg(test)]
296 #[ztracing::instrument(skip_all)]
297 pub fn text(&self) -> String {
298 self.chunks(
299 TabPoint::zero()..self.max_point(),
300 false,
301 Highlights::default(),
302 )
303 .map(|chunk| chunk.text)
304 .collect()
305 }
306
307 #[ztracing::instrument(skip_all)]
308 pub fn max_point(&self) -> TabPoint {
309 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
310 }
311
312 #[ztracing::instrument(skip_all)]
313 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
314 self.fold_point_to_tab_point(
315 self.fold_snapshot
316 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
317 )
318 }
319
320 #[ztracing::instrument(skip_all)]
321 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
322 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
323 let tab_cursor = TabStopCursor::new(chunks);
324 let expanded = self.expand_tabs(tab_cursor, input.column());
325 TabPoint::new(input.row(), expanded)
326 }
327
328 #[ztracing::instrument(skip_all)]
329 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
330 TabPointCursor { this: self }
331 }
332
333 #[ztracing::instrument(skip_all)]
334 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
335 let chunks = self
336 .fold_snapshot
337 .chunks_at(FoldPoint::new(output.row(), 0));
338
339 let tab_cursor = TabStopCursor::new(chunks);
340 let expanded = output.column();
341 let (collapsed, expanded_char_column, to_next_stop) =
342 self.collapse_tabs(tab_cursor, expanded, bias);
343
344 (
345 FoldPoint::new(output.row(), collapsed),
346 expanded_char_column,
347 to_next_stop,
348 )
349 }
350
351 #[ztracing::instrument(skip_all)]
352 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
353 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
354 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
355 self.fold_point_to_tab_point(fold_point)
356 }
357
358 #[ztracing::instrument(skip_all)]
359 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
360 let fold_point = self.tab_point_to_fold_point(point, bias).0;
361 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
362 self.fold_snapshot
363 .inlay_snapshot
364 .to_buffer_point(inlay_point)
365 }
366
367 #[ztracing::instrument(skip_all)]
368 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
369 where
370 I: Iterator<Item = Chunk<'a>>,
371 {
372 let tab_size = self.tab_size.get();
373
374 let end_column = column.min(self.max_expansion_column);
375 let mut seek_target = end_column;
376 let mut tab_count = 0;
377 let mut expanded_tab_len = 0;
378
379 while let Some(tab_stop) = cursor.seek(seek_target) {
380 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
381 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
382 tab_count += 1;
383 expanded_tab_len += tab_len;
384
385 seek_target = end_column - cursor.byte_offset;
386 }
387
388 let left_over_char_bytes = if !cursor.is_char_boundary() {
389 cursor.bytes_until_next_char().unwrap_or(0) as u32
390 } else {
391 0
392 };
393
394 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
395 let expanded_bytes =
396 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
397
398 expanded_bytes + column.saturating_sub(collapsed_bytes)
399 }
400
401 #[ztracing::instrument(skip_all)]
402 fn collapse_tabs<'a, I>(
403 &self,
404 mut cursor: TabStopCursor<'a, I>,
405 column: u32,
406 bias: Bias,
407 ) -> (u32, u32, u32)
408 where
409 I: Iterator<Item = Chunk<'a>>,
410 {
411 let tab_size = self.tab_size.get();
412 let mut collapsed_column = column;
413 let mut seek_target = column.min(self.max_expansion_column);
414 let mut tab_count = 0;
415 let mut expanded_tab_len = 0;
416
417 while let Some(tab_stop) = cursor.seek(seek_target) {
418 // Calculate how much we want to expand this tab stop (into spaces)
419 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
420 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
421 // Increment tab count
422 tab_count += 1;
423 // The count of how many spaces we've added to this line in place of tab bytes
424 expanded_tab_len += tab_len;
425
426 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
427 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
428
429 // Did we expand past the search target?
430 if expanded_bytes > column {
431 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
432 // We expanded past the search target, so need to account for the offshoot
433 expanded_chars -= expanded_bytes - column;
434 return match bias {
435 Bias::Left => (
436 cursor.byte_offset() - 1,
437 expanded_chars,
438 expanded_bytes - column,
439 ),
440 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
441 };
442 } else {
443 // otherwise we only want to move the cursor collapse column forward
444 collapsed_column = collapsed_column - tab_len + 1;
445 seek_target = (collapsed_column - cursor.byte_offset)
446 .min(self.max_expansion_column - cursor.byte_offset);
447 }
448 }
449
450 let collapsed_bytes = cursor.byte_offset();
451 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
452 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
453 (
454 collapsed_bytes + column.saturating_sub(expanded_bytes),
455 expanded_chars,
456 0,
457 )
458 }
459}
460
461// todo(lw): Implement TabPointCursor properly
462pub struct TabPointCursor<'this> {
463 this: &'this TabSnapshot,
464}
465
466impl TabPointCursor<'_> {
467 #[ztracing::instrument(skip_all)]
468 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
469 self.this.fold_point_to_tab_point(point)
470 }
471}
472
473#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
474pub struct TabPoint(pub Point);
475
476impl TabPoint {
477 pub fn new(row: u32, column: u32) -> Self {
478 Self(Point::new(row, column))
479 }
480
481 pub fn zero() -> Self {
482 Self::new(0, 0)
483 }
484
485 pub fn row(self) -> u32 {
486 self.0.row
487 }
488
489 pub fn column(self) -> u32 {
490 self.0.column
491 }
492}
493
494impl From<Point> for TabPoint {
495 fn from(point: Point) -> Self {
496 Self(point)
497 }
498}
499
500pub type TabEdit = text::Edit<TabPoint>;
501
502#[derive(Clone, Debug, Default, Eq, PartialEq)]
503pub struct TextSummary {
504 pub lines: Point,
505 pub first_line_chars: u32,
506 pub last_line_chars: u32,
507 pub longest_row: u32,
508 pub longest_row_chars: u32,
509}
510
511impl<'a> From<&'a str> for TextSummary {
512 #[ztracing::instrument(skip_all)]
513 fn from(text: &'a str) -> Self {
514 let sum = text::TextSummary::from(text);
515
516 TextSummary {
517 lines: sum.lines,
518 first_line_chars: sum.first_line_chars,
519 last_line_chars: sum.last_line_chars,
520 longest_row: sum.longest_row,
521 longest_row_chars: sum.longest_row_chars,
522 }
523 }
524}
525
526impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
527 #[ztracing::instrument(skip_all)]
528 fn add_assign(&mut self, other: &'a Self) {
529 let joined_chars = self.last_line_chars + other.first_line_chars;
530 if joined_chars > self.longest_row_chars {
531 self.longest_row = self.lines.row;
532 self.longest_row_chars = joined_chars;
533 }
534 if other.longest_row_chars > self.longest_row_chars {
535 self.longest_row = self.lines.row + other.longest_row;
536 self.longest_row_chars = other.longest_row_chars;
537 }
538
539 if self.lines.row == 0 {
540 self.first_line_chars += other.first_line_chars;
541 }
542
543 if other.lines.row == 0 {
544 self.last_line_chars += other.first_line_chars;
545 } else {
546 self.last_line_chars = other.last_line_chars;
547 }
548
549 self.lines += &other.lines;
550 }
551}
552
553pub struct TabChunks<'a> {
554 snapshot: &'a TabSnapshot,
555 max_expansion_column: u32,
556 max_output_position: Point,
557 tab_size: NonZeroU32,
558 // region: iteration state
559 fold_chunks: FoldChunks<'a>,
560 chunk: Chunk<'a>,
561 column: u32,
562 output_position: Point,
563 input_column: u32,
564 inside_leading_tab: bool,
565 // endregion: iteration state
566}
567
568impl TabChunks<'_> {
569 #[ztracing::instrument(skip_all)]
570 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
571 let (input_start, expanded_char_column, to_next_stop) = self
572 .snapshot
573 .tab_point_to_fold_point(range.start, Bias::Left);
574 let input_column = input_start.column();
575 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
576 let input_end = self
577 .snapshot
578 .tab_point_to_fold_point(range.end, Bias::Right)
579 .0
580 .to_offset(&self.snapshot.fold_snapshot);
581 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
582 range.end.column() - range.start.column()
583 } else {
584 to_next_stop
585 };
586
587 self.fold_chunks.seek(input_start..input_end);
588 self.input_column = input_column;
589 self.column = expanded_char_column;
590 self.output_position = range.start.0;
591 self.max_output_position = range.end.0;
592 self.chunk = Chunk {
593 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
594 is_tab: true,
595 chars: 1u128.unbounded_shl(to_next_stop) - 1,
596 ..Default::default()
597 };
598 self.inside_leading_tab = to_next_stop > 0;
599 }
600}
601
602impl<'a> Iterator for TabChunks<'a> {
603 type Item = Chunk<'a>;
604
605 #[ztracing::instrument(skip_all)]
606 fn next(&mut self) -> Option<Self::Item> {
607 if self.chunk.text.is_empty() {
608 if let Some(chunk) = self.fold_chunks.next() {
609 self.chunk = chunk;
610 if self.inside_leading_tab {
611 self.chunk.text = &self.chunk.text[1..];
612 self.chunk.tabs >>= 1;
613 self.chunk.chars >>= 1;
614 self.chunk.newlines >>= 1;
615 self.inside_leading_tab = false;
616 self.input_column += 1;
617 }
618 } else {
619 return None;
620 }
621 }
622
623 let first_tab_ix = if self.chunk.tabs != 0 {
624 self.chunk.tabs.trailing_zeros() as usize
625 } else {
626 self.chunk.text.len()
627 };
628
629 if first_tab_ix == 0 {
630 self.chunk.text = &self.chunk.text[1..];
631 self.chunk.tabs >>= 1;
632 self.chunk.chars >>= 1;
633 self.chunk.newlines >>= 1;
634
635 let tab_size = if self.input_column < self.max_expansion_column {
636 self.tab_size.get()
637 } else {
638 1
639 };
640 let mut len = tab_size - self.column % tab_size;
641 let next_output_position = cmp::min(
642 self.output_position + Point::new(0, len),
643 self.max_output_position,
644 );
645 len = next_output_position.column - self.output_position.column;
646 self.column += len;
647 self.input_column += 1;
648 self.output_position = next_output_position;
649
650 return Some(Chunk {
651 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
652 is_tab: true,
653 chars: 1u128.unbounded_shl(len) - 1,
654 tabs: 0,
655 newlines: 0,
656 ..self.chunk.clone()
657 });
658 }
659
660 let prefix_len = first_tab_ix;
661 let (prefix, suffix) = self.chunk.text.split_at(prefix_len);
662
663 let mask = 1u128.unbounded_shl(prefix_len as u32).wrapping_sub(1);
664 let prefix_chars = self.chunk.chars & mask;
665 let prefix_tabs = self.chunk.tabs & mask;
666 let prefix_newlines = self.chunk.newlines & mask;
667
668 self.chunk.text = suffix;
669 self.chunk.tabs = self.chunk.tabs.unbounded_shr(prefix_len as u32);
670 self.chunk.chars = self.chunk.chars.unbounded_shr(prefix_len as u32);
671 self.chunk.newlines = self.chunk.newlines.unbounded_shr(prefix_len as u32);
672
673 let newline_count = prefix_newlines.count_ones();
674 if newline_count > 0 {
675 let last_newline_bit = 128 - prefix_newlines.leading_zeros();
676 let chars_after_last_newline =
677 prefix_chars.unbounded_shr(last_newline_bit).count_ones();
678 let bytes_after_last_newline = prefix_len as u32 - last_newline_bit;
679
680 self.column = chars_after_last_newline;
681 self.input_column = bytes_after_last_newline;
682 self.output_position = Point::new(
683 self.output_position.row + newline_count,
684 bytes_after_last_newline,
685 );
686 } else {
687 let char_count = prefix_chars.count_ones();
688 self.column += char_count;
689 if !self.inside_leading_tab {
690 self.input_column += prefix_len as u32;
691 }
692 self.output_position.column += prefix_len as u32;
693 }
694
695 Some(Chunk {
696 text: prefix,
697 chars: prefix_chars,
698 tabs: prefix_tabs,
699 newlines: prefix_newlines,
700 ..self.chunk.clone()
701 })
702 }
703}
704
705#[cfg(test)]
706mod tests {
707 use std::mem;
708
709 use super::*;
710 use crate::{
711 MultiBuffer,
712 display_map::{
713 fold_map::{FoldMap, FoldOffset},
714 inlay_map::InlayMap,
715 },
716 };
717 use multi_buffer::MultiBufferOffset;
718 use rand::{Rng, prelude::StdRng};
719 use util;
720
721 impl TabSnapshot {
722 fn expected_collapse_tabs(
723 &self,
724 chars: impl Iterator<Item = char>,
725 column: u32,
726 bias: Bias,
727 ) -> (u32, u32, u32) {
728 let tab_size = self.tab_size.get();
729
730 let mut expanded_bytes = 0;
731 let mut expanded_chars = 0;
732 let mut collapsed_bytes = 0;
733 for c in chars {
734 if expanded_bytes >= column {
735 break;
736 }
737 if collapsed_bytes >= self.max_expansion_column {
738 break;
739 }
740
741 if c == '\t' {
742 let tab_len = tab_size - (expanded_chars % tab_size);
743 expanded_chars += tab_len;
744 expanded_bytes += tab_len;
745 if expanded_bytes > column {
746 expanded_chars -= expanded_bytes - column;
747 return match bias {
748 Bias::Left => {
749 (collapsed_bytes, expanded_chars, expanded_bytes - column)
750 }
751 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
752 };
753 }
754 } else {
755 expanded_chars += 1;
756 expanded_bytes += c.len_utf8() as u32;
757 }
758
759 if expanded_bytes > column && matches!(bias, Bias::Left) {
760 expanded_chars -= 1;
761 break;
762 }
763
764 collapsed_bytes += c.len_utf8() as u32;
765 }
766
767 (
768 collapsed_bytes + column.saturating_sub(expanded_bytes),
769 expanded_chars,
770 0,
771 )
772 }
773
774 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
775 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
776 let expanded = self.expected_expand_tabs(chars, input.column());
777 TabPoint::new(input.row(), expanded)
778 }
779
780 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
781 let tab_size = self.tab_size.get();
782
783 let mut expanded_chars = 0;
784 let mut expanded_bytes = 0;
785 let mut collapsed_bytes = 0;
786 let end_column = column.min(self.max_expansion_column);
787 for c in chars {
788 if collapsed_bytes >= end_column {
789 break;
790 }
791 if c == '\t' {
792 let tab_len = tab_size - expanded_chars % tab_size;
793 expanded_bytes += tab_len;
794 expanded_chars += tab_len;
795 } else {
796 expanded_bytes += c.len_utf8() as u32;
797 expanded_chars += 1;
798 }
799 collapsed_bytes += c.len_utf8() as u32;
800 }
801
802 expanded_bytes + column.saturating_sub(collapsed_bytes)
803 }
804
805 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
806 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
807 let expanded = output.column();
808 let (collapsed, expanded_char_column, to_next_stop) =
809 self.expected_collapse_tabs(chars, expanded, bias);
810 (
811 FoldPoint::new(output.row(), collapsed),
812 expanded_char_column,
813 to_next_stop,
814 )
815 }
816 }
817
818 #[gpui::test]
819 fn test_expand_tabs(cx: &mut gpui::App) {
820 let test_values = [
821 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
822 (" \twςe", 4),
823 ("fε", 1),
824 ("i❎\t", 3),
825 ];
826 let buffer = MultiBuffer::build_simple("", cx);
827 let buffer_snapshot = buffer.read(cx).snapshot(cx);
828 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
829 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
830 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
831
832 for (text, column) in test_values {
833 let mut tabs = 0u128;
834 let mut chars = 0u128;
835 for (idx, c) in text.char_indices() {
836 if c == '\t' {
837 tabs |= 1 << idx;
838 }
839 chars |= 1 << idx;
840 }
841
842 let chunks = [Chunk {
843 text,
844 tabs,
845 chars,
846 ..Default::default()
847 }];
848
849 let cursor = TabStopCursor::new(chunks);
850
851 assert_eq!(
852 tab_snapshot.expected_expand_tabs(text.chars(), column),
853 tab_snapshot.expand_tabs(cursor, column)
854 );
855 }
856 }
857
858 #[gpui::test]
859 fn test_collapse_tabs(cx: &mut gpui::App) {
860 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
861
862 let buffer = MultiBuffer::build_simple(input, cx);
863 let buffer_snapshot = buffer.read(cx).snapshot(cx);
864 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
865 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
866 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
867
868 for (ix, _) in input.char_indices() {
869 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
870
871 assert_eq!(
872 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
873 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
874 "Failed with tab_point at column {ix}"
875 );
876 assert_eq!(
877 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
878 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
879 "Failed with tab_point at column {ix}"
880 );
881
882 assert_eq!(
883 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
884 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
885 "Failed with tab_point at column {ix}"
886 );
887 assert_eq!(
888 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
889 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
890 "Failed with tab_point at column {ix}"
891 );
892 }
893 }
894
895 #[gpui::test]
896 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
897 // This test reproduces a specific panic where to_fold_point returns incorrect results
898 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
899
900 let text = "γ\tw⭐\n🍐🍗 \t";
901 let buffer = MultiBuffer::build_simple(text, cx);
902 let buffer_snapshot = buffer.read(cx).snapshot(cx);
903 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
904 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
905 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
906
907 // This should panic with the expected vs actual mismatch
908 let tab_point = TabPoint::new(0, 9);
909 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
910 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
911
912 assert_eq!(result, expected);
913 }
914
915 #[gpui::test(iterations = 100)]
916 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
917 // Generate random input string with up to 200 characters including tabs
918 // to stay within the MAX_EXPANSION_COLUMN limit of 256
919 let len = rng.random_range(0..=2048);
920 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
921 let mut input = String::with_capacity(len);
922
923 for _ in 0..len {
924 if rng.random_bool(0.1) {
925 // 10% chance of inserting a tab
926 input.push('\t');
927 } else {
928 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
929 let ch = loop {
930 let ascii_code = rng.random_range(32..=126); // printable ASCII range
931 let ch = ascii_code as u8 as char;
932 if ch != '\t' {
933 break ch;
934 }
935 };
936 input.push(ch);
937 }
938 }
939
940 let buffer = MultiBuffer::build_simple(&input, cx);
941 let buffer_snapshot = buffer.read(cx).snapshot(cx);
942 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
943 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
944 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
945 tab_snapshot.max_expansion_column = rng.random_range(0..323);
946 tab_snapshot.tab_size = tab_size;
947
948 for (ix, _) in input.char_indices() {
949 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
950
951 assert_eq!(
952 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
953 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
954 "Failed with input: {}, with idx: {ix}",
955 input
956 );
957 assert_eq!(
958 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
959 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
960 "Failed with input: {}, with idx: {ix}",
961 input
962 );
963
964 assert_eq!(
965 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
966 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
967 "Failed with input: {}, with idx: {ix}",
968 input
969 );
970 assert_eq!(
971 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
972 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
973 "Failed with input: {}, with idx: {ix}",
974 input
975 );
976 }
977 }
978
979 #[gpui::test]
980 fn test_long_lines(cx: &mut gpui::App) {
981 let max_expansion_column = 12;
982 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
983 let output = "A BC DEF G HI J K L M";
984
985 let buffer = MultiBuffer::build_simple(input, cx);
986 let buffer_snapshot = buffer.read(cx).snapshot(cx);
987 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
988 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
989 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
990
991 tab_snapshot.max_expansion_column = max_expansion_column;
992 assert_eq!(tab_snapshot.text(), output);
993
994 for (ix, c) in input.char_indices() {
995 assert_eq!(
996 tab_snapshot
997 .chunks(
998 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
999 false,
1000 Highlights::default(),
1001 )
1002 .map(|c| c.text)
1003 .collect::<String>(),
1004 &output[ix..],
1005 "text from index {ix}"
1006 );
1007
1008 if c != '\t' {
1009 let input_point = Point::new(0, ix as u32);
1010 let output_point = Point::new(0, output.find(c).unwrap() as u32);
1011 assert_eq!(
1012 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
1013 TabPoint(output_point),
1014 "to_tab_point({input_point:?})"
1015 );
1016 assert_eq!(
1017 tab_snapshot
1018 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
1019 .0,
1020 FoldPoint(input_point),
1021 "to_fold_point({output_point:?})"
1022 );
1023 }
1024 }
1025 }
1026
1027 #[gpui::test]
1028 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
1029 let max_expansion_column = 8;
1030 let input = "abcdefg⋯hij";
1031
1032 let buffer = MultiBuffer::build_simple(input, cx);
1033 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1034 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1035 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1036 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1037
1038 tab_snapshot.max_expansion_column = max_expansion_column;
1039 assert_eq!(tab_snapshot.text(), input);
1040 }
1041
1042 #[gpui::test]
1043 fn test_marking_tabs(cx: &mut gpui::App) {
1044 let input = "\t \thello";
1045
1046 let buffer = MultiBuffer::build_simple(input, cx);
1047 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1048 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1049 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1050 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1051
1052 assert_eq!(
1053 chunks(&tab_snapshot, TabPoint::zero()),
1054 vec![
1055 (" ".to_string(), true),
1056 (" ".to_string(), false),
1057 (" ".to_string(), true),
1058 ("hello".to_string(), false),
1059 ]
1060 );
1061 assert_eq!(
1062 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1063 vec![
1064 (" ".to_string(), true),
1065 (" ".to_string(), false),
1066 (" ".to_string(), true),
1067 ("hello".to_string(), false),
1068 ]
1069 );
1070
1071 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1072 let mut chunks = Vec::new();
1073 let mut was_tab = false;
1074 let mut text = String::new();
1075 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1076 {
1077 if chunk.is_tab != was_tab {
1078 if !text.is_empty() {
1079 chunks.push((mem::take(&mut text), was_tab));
1080 }
1081 was_tab = chunk.is_tab;
1082 }
1083 text.push_str(chunk.text);
1084 }
1085
1086 if !text.is_empty() {
1087 chunks.push((text, was_tab));
1088 }
1089 chunks
1090 }
1091 }
1092
1093 #[gpui::test(iterations = 100)]
1094 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1095 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1096 let len = rng.random_range(0..30);
1097 let buffer = if rng.random() {
1098 let text = util::RandomCharIter::new(&mut rng)
1099 .take(len)
1100 .collect::<String>();
1101 MultiBuffer::build_simple(&text, cx)
1102 } else {
1103 MultiBuffer::build_random(&mut rng, cx)
1104 };
1105 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1106 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1107
1108 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1109 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1110 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1111 fold_map.randomly_mutate(&mut rng);
1112 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1113 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1114 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1115 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1116
1117 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1118 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1119
1120 let text = text::Rope::from(tabs_snapshot.text().as_str());
1121 log::info!(
1122 "TabMap text (tab size: {}): {:?}",
1123 tab_size,
1124 tabs_snapshot.text(),
1125 );
1126
1127 for _ in 0..5 {
1128 let end_row = rng.random_range(0..=text.max_point().row);
1129 let end_column = rng.random_range(0..=text.line_len(end_row));
1130 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1131 let start_row = rng.random_range(0..=text.max_point().row);
1132 let start_column = rng.random_range(0..=text.line_len(start_row));
1133 let mut start =
1134 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1135 if start > end {
1136 mem::swap(&mut start, &mut end);
1137 }
1138
1139 let expected_text = text
1140 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1141 .collect::<String>();
1142 let expected_summary = TextSummary::from(expected_text.as_str());
1143 assert_eq!(
1144 tabs_snapshot
1145 .chunks(start..end, false, Highlights::default())
1146 .map(|c| c.text)
1147 .collect::<String>(),
1148 expected_text,
1149 "chunks({:?}..{:?})",
1150 start,
1151 end
1152 );
1153
1154 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1155 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1156 actual_summary.longest_row = expected_summary.longest_row;
1157 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1158 }
1159 assert_eq!(actual_summary, expected_summary);
1160 }
1161
1162 for row in 0..=text.max_point().row {
1163 assert_eq!(
1164 tabs_snapshot.line_len(row),
1165 text.line_len(row),
1166 "line_len({row})"
1167 );
1168 }
1169 }
1170
1171 #[gpui::test(iterations = 100)]
1172 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1173 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1174 let len = rng.random_range(0..=2000);
1175
1176 // Generate random text using RandomCharIter
1177 let text = util::RandomCharIter::new(&mut rng)
1178 .take(len)
1179 .collect::<String>();
1180
1181 // Create buffer and tab map
1182 let buffer = MultiBuffer::build_simple(&text, cx);
1183 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1184 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1185 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1186 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1187
1188 let mut next_inlay_id = 0;
1189 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1190 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1191 let max_fold_point = fold_snapshot.max_point();
1192 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1193
1194 // Test random fold points
1195 for _ in 0..50 {
1196 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1197 // Generate random fold point
1198 let row = rng.random_range(0..=max_fold_point.row());
1199 let max_column = if row < max_fold_point.row() {
1200 fold_snapshot.line_len(row)
1201 } else {
1202 max_fold_point.column()
1203 };
1204 let column = rng.random_range(0..=max_column + 10);
1205 let fold_point = FoldPoint::new(row, column);
1206
1207 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1208 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1209
1210 assert_eq!(
1211 actual, expected,
1212 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1213 fold_point, text
1214 );
1215 }
1216 }
1217
1218 #[gpui::test]
1219 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1220 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1221 let buffer = MultiBuffer::build_simple(text, cx);
1222 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1223 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1224 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1225 let chunks = fold_snapshot.chunks(
1226 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1227 false,
1228 Default::default(),
1229 );
1230 let mut cursor = TabStopCursor::new(chunks);
1231 assert!(cursor.seek(0).is_none());
1232 let mut tab_stops = Vec::new();
1233
1234 let mut all_tab_stops = Vec::new();
1235 let mut byte_offset = 0;
1236 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1237 byte_offset += ch.len_utf8() as u32;
1238
1239 if ch == '\t' {
1240 all_tab_stops.push(TabStop {
1241 byte_offset,
1242 char_offset: offset as u32 + 1,
1243 });
1244 }
1245 }
1246
1247 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1248 tab_stops.push(tab_stop);
1249 }
1250 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1251
1252 assert_eq!(cursor.byte_offset(), byte_offset);
1253 }
1254
1255 #[gpui::test]
1256 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1257 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1258
1259 let buffer = MultiBuffer::build_simple(input, cx);
1260 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1261 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1262 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1263
1264 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1265 let mut cursor = TabStopCursor::new(chunks);
1266
1267 let mut actual_tab_stops = Vec::new();
1268
1269 let mut expected_tab_stops = Vec::new();
1270 let mut byte_offset = 0;
1271 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1272 byte_offset += ch.len_utf8() as u32;
1273
1274 if ch == '\t' {
1275 expected_tab_stops.push(TabStop {
1276 byte_offset,
1277 char_offset: offset as u32 + 1,
1278 });
1279 }
1280 }
1281
1282 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1283 actual_tab_stops.push(tab_stop);
1284 }
1285 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1286
1287 assert_eq!(cursor.byte_offset(), byte_offset);
1288 }
1289
1290 #[gpui::test(iterations = 100)]
1291 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1292 // Generate random input string with up to 512 characters including tabs
1293 let len = rng.random_range(0..=2048);
1294 let mut input = String::with_capacity(len);
1295
1296 let mut skip_tabs = rng.random_bool(0.10);
1297 for idx in 0..len {
1298 if idx % 128 == 0 {
1299 skip_tabs = rng.random_bool(0.10);
1300 }
1301
1302 if rng.random_bool(0.15) && !skip_tabs {
1303 input.push('\t');
1304 } else {
1305 let ch = loop {
1306 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1307 let ch = ascii_code as u8 as char;
1308 if ch != '\t' {
1309 break ch;
1310 }
1311 };
1312 input.push(ch);
1313 }
1314 }
1315
1316 // Build the buffer and create cursor
1317 let buffer = MultiBuffer::build_simple(&input, cx);
1318 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1319 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1320 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1321
1322 // First, collect all expected tab positions
1323 let mut all_tab_stops = Vec::new();
1324 let mut byte_offset = 1;
1325 let mut char_offset = 1;
1326 for ch in buffer_snapshot.text().chars() {
1327 if ch == '\t' {
1328 all_tab_stops.push(TabStop {
1329 byte_offset,
1330 char_offset,
1331 });
1332 }
1333 byte_offset += ch.len_utf8() as u32;
1334 char_offset += 1;
1335 }
1336
1337 // Test with various distances
1338 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1339 // let distances = vec![150];
1340
1341 for distance in distances {
1342 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1343 let mut cursor = TabStopCursor::new(chunks);
1344
1345 let mut found_tab_stops = Vec::new();
1346 let mut position = distance;
1347 while let Some(tab_stop) = cursor.seek(position) {
1348 found_tab_stops.push(tab_stop);
1349 position = distance - tab_stop.byte_offset;
1350 }
1351
1352 let expected_found_tab_stops: Vec<_> = all_tab_stops
1353 .iter()
1354 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1355 .cloned()
1356 .collect();
1357
1358 pretty_assertions::assert_eq!(
1359 found_tab_stops,
1360 expected_found_tab_stops,
1361 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1362 distance,
1363 input
1364 );
1365
1366 let final_position = cursor.byte_offset();
1367 if !found_tab_stops.is_empty() {
1368 let last_tab_stop = found_tab_stops.last().unwrap();
1369 assert!(
1370 final_position >= last_tab_stop.byte_offset,
1371 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1372 final_position,
1373 last_tab_stop.byte_offset,
1374 input
1375 );
1376 }
1377 }
1378 }
1379
1380 #[gpui::test]
1381 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1382 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1383 let buffer = MultiBuffer::build_simple(text, cx);
1384 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1385 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1386 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1387 let chunks = fold_snapshot.chunks(
1388 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1389 false,
1390 Default::default(),
1391 );
1392 let mut cursor = TabStopCursor::new(chunks);
1393 assert!(cursor.seek(0).is_none());
1394
1395 let mut expected_tab_stops = Vec::new();
1396 let mut byte_offset = 0;
1397 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1398 byte_offset += ch.len_utf8() as u32;
1399
1400 if ch == '\t' {
1401 expected_tab_stops.push(TabStop {
1402 byte_offset,
1403 char_offset: i as u32 + 1,
1404 });
1405 }
1406 }
1407
1408 let mut actual_tab_stops = Vec::new();
1409 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1410 actual_tab_stops.push(tab_stop);
1411 }
1412
1413 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1414
1415 assert_eq!(cursor.byte_offset(), byte_offset);
1416 }
1417
1418 #[gpui::test(iterations = 100)]
1419 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1420 // Generate random input string with up to 512 characters including tabs
1421 let len = rng.random_range(0..=2048);
1422 let input = util::RandomCharIter::new(&mut rng)
1423 .take(len)
1424 .collect::<String>();
1425
1426 // Build the buffer and create cursor
1427 let buffer = MultiBuffer::build_simple(&input, cx);
1428 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1429 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1430 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1431
1432 // First, collect all expected tab positions
1433 let mut all_tab_stops = Vec::new();
1434 let mut byte_offset = 0;
1435 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1436 byte_offset += ch.len_utf8() as u32;
1437 if ch == '\t' {
1438 all_tab_stops.push(TabStop {
1439 byte_offset,
1440 char_offset: i as u32 + 1,
1441 });
1442 }
1443 }
1444
1445 // Test with various distances
1446 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1447 let distances = vec![150];
1448
1449 for distance in distances {
1450 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1451 let mut cursor = TabStopCursor::new(chunks);
1452
1453 let mut found_tab_stops = Vec::new();
1454 let mut position = distance;
1455 while let Some(tab_stop) = cursor.seek(position) {
1456 found_tab_stops.push(tab_stop);
1457 position = distance - tab_stop.byte_offset;
1458 }
1459
1460 let expected_found_tab_stops: Vec<_> = all_tab_stops
1461 .iter()
1462 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1463 .cloned()
1464 .collect();
1465
1466 pretty_assertions::assert_eq!(
1467 found_tab_stops,
1468 expected_found_tab_stops,
1469 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1470 distance,
1471 input
1472 );
1473
1474 let final_position = cursor.byte_offset();
1475 if !found_tab_stops.is_empty() {
1476 let last_tab_stop = found_tab_stops.last().unwrap();
1477 assert!(
1478 final_position >= last_tab_stop.byte_offset,
1479 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1480 final_position,
1481 last_tab_stop.byte_offset,
1482 input
1483 );
1484 }
1485 }
1486 }
1487}
1488
1489struct TabStopCursor<'a, I>
1490where
1491 I: Iterator<Item = Chunk<'a>>,
1492{
1493 chunks: I,
1494 byte_offset: u32,
1495 char_offset: u32,
1496 /// Chunk
1497 /// last tab position iterated through
1498 current_chunk: Option<(Chunk<'a>, u32)>,
1499}
1500
1501impl<'a, I> TabStopCursor<'a, I>
1502where
1503 I: Iterator<Item = Chunk<'a>>,
1504{
1505 #[ztracing::instrument(skip_all)]
1506 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1507 Self {
1508 chunks: chunks.into_iter(),
1509 byte_offset: 0,
1510 char_offset: 0,
1511 current_chunk: None,
1512 }
1513 }
1514
1515 #[ztracing::instrument(skip_all)]
1516 fn bytes_until_next_char(&self) -> Option<usize> {
1517 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1518 let mut idx = *idx;
1519 let mut diff = 0;
1520 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1521 idx -= 1;
1522 diff += 1;
1523 }
1524
1525 if chunk.chars & (1 << idx) != 0 {
1526 Some(
1527 (chunk.text[idx as usize..].chars().next()?)
1528 .len_utf8()
1529 .saturating_sub(diff),
1530 )
1531 } else {
1532 None
1533 }
1534 })
1535 }
1536
1537 #[ztracing::instrument(skip_all)]
1538 fn is_char_boundary(&self) -> bool {
1539 self.current_chunk
1540 .as_ref()
1541 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1542 }
1543
1544 /// distance: length to move forward while searching for the next tab stop
1545 #[ztracing::instrument(skip_all)]
1546 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1547 if distance == 0 {
1548 return None;
1549 }
1550
1551 let mut distance_traversed = 0;
1552
1553 while let Some((mut chunk, chunk_position)) = self
1554 .current_chunk
1555 .take()
1556 .or_else(|| self.chunks.next().zip(Some(0)))
1557 {
1558 if chunk.tabs == 0 {
1559 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1560 if chunk_distance + distance_traversed >= distance {
1561 let overshoot = distance_traversed.abs_diff(distance);
1562
1563 self.byte_offset += overshoot;
1564 self.char_offset += get_char_offset(
1565 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1566 chunk.chars,
1567 );
1568
1569 if chunk_position + overshoot < 128 {
1570 self.current_chunk = Some((chunk, chunk_position + overshoot));
1571 }
1572
1573 return None;
1574 }
1575
1576 self.byte_offset += chunk_distance;
1577 self.char_offset += get_char_offset(
1578 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1579 chunk.chars,
1580 );
1581 distance_traversed += chunk_distance;
1582 continue;
1583 }
1584 let tab_position = chunk.tabs.trailing_zeros() + 1;
1585
1586 if distance_traversed + tab_position - chunk_position > distance {
1587 let cursor_position = distance_traversed.abs_diff(distance);
1588
1589 self.char_offset += get_char_offset(
1590 chunk_position..(chunk_position + cursor_position - 1),
1591 chunk.chars,
1592 );
1593 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1594 self.byte_offset += cursor_position;
1595
1596 return None;
1597 }
1598
1599 self.byte_offset += tab_position - chunk_position;
1600 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1601
1602 let tabstop = TabStop {
1603 char_offset: self.char_offset,
1604 byte_offset: self.byte_offset,
1605 };
1606
1607 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1608
1609 if tab_position as usize != chunk.text.len() {
1610 self.current_chunk = Some((chunk, tab_position));
1611 }
1612
1613 return Some(tabstop);
1614 }
1615
1616 None
1617 }
1618
1619 fn byte_offset(&self) -> u32 {
1620 self.byte_offset
1621 }
1622
1623 fn char_offset(&self) -> u32 {
1624 self.char_offset
1625 }
1626}
1627
1628#[inline(always)]
1629fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1630 if range.start == range.end {
1631 return if (1u128 << range.start) & bit_map == 0 {
1632 0
1633 } else {
1634 1
1635 };
1636 }
1637 let end_shift: u128 = 127u128 - range.end as u128;
1638 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1639 bit_mask = (bit_mask << end_shift) >> end_shift;
1640 let bit_map = bit_map & bit_mask;
1641
1642 bit_map.count_ones()
1643}
1644
1645#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1646struct TabStop {
1647 char_offset: u32,
1648 byte_offset: u32,
1649}