1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 #[ztracing::instrument(skip_all)]
24 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
25 let snapshot = TabSnapshot {
26 fold_snapshot,
27 tab_size: tab_size.min(MAX_TABS),
28 max_expansion_column: MAX_EXPANSION_COLUMN,
29 version: 0,
30 };
31 (Self(snapshot.clone()), snapshot)
32 }
33
34 #[cfg(test)]
35 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
36 self.0.max_expansion_column = column;
37 self.0.clone()
38 }
39
40 #[ztracing::instrument(skip_all)]
41 pub fn sync(
42 &mut self,
43 fold_snapshot: FoldSnapshot,
44 mut fold_edits: Vec<FoldEdit>,
45 tab_size: NonZeroU32,
46 ) -> (TabSnapshot, Vec<TabEdit>) {
47 let old_snapshot = &mut self.0;
48 let mut new_snapshot = TabSnapshot {
49 fold_snapshot,
50 tab_size: tab_size.min(MAX_TABS),
51 max_expansion_column: old_snapshot.max_expansion_column,
52 version: old_snapshot.version,
53 };
54
55 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
56 new_snapshot.version += 1;
57 }
58
59 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
60 // Expand each edit to include the next tab on the same line as the edit,
61 // and any subsequent tabs on that line that moved across the tab expansion
62 // boundary.
63 for fold_edit in &mut fold_edits {
64 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
65 let old_end_row_successor_offset = cmp::min(
66 FoldPoint::new(old_end.row() + 1, 0),
67 old_snapshot.fold_snapshot.max_point(),
68 )
69 .to_offset(&old_snapshot.fold_snapshot);
70 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
71
72 let mut offset_from_edit = 0;
73 let mut first_tab_offset = None;
74 let mut last_tab_with_changed_expansion_offset = None;
75 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
76 fold_edit.old.end..old_end_row_successor_offset,
77 false,
78 Highlights::default(),
79 ) {
80 let mut remaining_tabs = chunk.tabs;
81 while remaining_tabs != 0 {
82 let ix = remaining_tabs.trailing_zeros();
83 let offset_from_edit = offset_from_edit + ix;
84 if first_tab_offset.is_none() {
85 first_tab_offset = Some(offset_from_edit);
86 }
87
88 let old_column = old_end.column() + offset_from_edit;
89 let new_column = new_end.column() + offset_from_edit;
90 let was_expanded = old_column < old_snapshot.max_expansion_column;
91 let is_expanded = new_column < new_snapshot.max_expansion_column;
92 if was_expanded != is_expanded {
93 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
94 } else if !was_expanded && !is_expanded {
95 break 'outer;
96 }
97
98 remaining_tabs &= remaining_tabs - 1;
99 }
100
101 offset_from_edit += chunk.text.len() as u32;
102 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
103 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
104 {
105 break;
106 }
107 }
108
109 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
110 fold_edit.old.end.0 += offset as usize + 1;
111 fold_edit.new.end.0 += offset as usize + 1;
112 }
113 }
114
115 let _old_alloc_ptr = fold_edits.as_ptr();
116 // Combine any edits that overlap due to the expansion.
117 let mut fold_edits = fold_edits.into_iter();
118 if let Some(mut first_edit) = fold_edits.next() {
119 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
120 #[allow(clippy::filter_map_identity)]
121 let mut v: Vec<_> = fold_edits
122 .scan(&mut first_edit, |state, edit| {
123 if state.old.end >= edit.old.start {
124 state.old.end = edit.old.end;
125 state.new.end = edit.new.end;
126 Some(None) // Skip this edit, it's merged
127 } else {
128 let new_state = edit;
129 let result = Some(Some(state.clone())); // Yield the previous edit
130 **state = new_state;
131 result
132 }
133 })
134 .filter_map(|x| x)
135 .collect();
136 v.push(first_edit);
137 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
138 v.into_iter()
139 .map(|fold_edit| {
140 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
141 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
142 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
143 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
144 TabEdit {
145 old: old_snapshot.fold_point_to_tab_point(old_start)
146 ..old_snapshot.fold_point_to_tab_point(old_end),
147 new: new_snapshot.fold_point_to_tab_point(new_start)
148 ..new_snapshot.fold_point_to_tab_point(new_end),
149 }
150 })
151 .collect()
152 } else {
153 vec![]
154 }
155 } else {
156 new_snapshot.version += 1;
157 vec![TabEdit {
158 old: TabPoint::zero()..old_snapshot.max_point(),
159 new: TabPoint::zero()..new_snapshot.max_point(),
160 }]
161 };
162 *old_snapshot = new_snapshot;
163 (old_snapshot.clone(), tab_edits)
164 }
165}
166
167#[derive(Clone)]
168pub struct TabSnapshot {
169 pub fold_snapshot: FoldSnapshot,
170 pub tab_size: NonZeroU32,
171 pub max_expansion_column: u32,
172 pub version: usize,
173}
174
175impl std::ops::Deref for TabSnapshot {
176 type Target = FoldSnapshot;
177
178 fn deref(&self) -> &Self::Target {
179 &self.fold_snapshot
180 }
181}
182
183impl TabSnapshot {
184 #[ztracing::instrument(skip_all)]
185 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
186 &self.fold_snapshot.inlay_snapshot.buffer
187 }
188
189 #[ztracing::instrument(skip_all)]
190 pub fn line_len(&self, row: u32) -> u32 {
191 let max_point = self.max_point();
192 if row < max_point.row() {
193 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
194 .0
195 .column
196 } else {
197 max_point.column()
198 }
199 }
200
201 #[ztracing::instrument(skip_all)]
202 pub fn text_summary(&self) -> TextSummary {
203 self.text_summary_for_range(TabPoint::zero()..self.max_point())
204 }
205
206 #[ztracing::instrument(skip_all, fields(rows))]
207 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
208 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
209 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
210 let input_summary = self
211 .fold_snapshot
212 .text_summary_for_range(input_start..input_end);
213
214 let line_end = if range.start.row() == range.end.row() {
215 range.end
216 } else {
217 self.max_point()
218 };
219 let first_line_chars = self
220 .chunks(range.start..line_end, false, Highlights::default())
221 .flat_map(|chunk| chunk.text.chars())
222 .take_while(|&c| c != '\n')
223 .count() as u32;
224
225 let last_line_chars = if range.start.row() == range.end.row() {
226 first_line_chars
227 } else {
228 self.chunks(
229 TabPoint::new(range.end.row(), 0)..range.end,
230 false,
231 Highlights::default(),
232 )
233 .flat_map(|chunk| chunk.text.chars())
234 .count() as u32
235 };
236
237 TextSummary {
238 lines: range.end.0 - range.start.0,
239 first_line_chars,
240 last_line_chars,
241 longest_row: input_summary.longest_row,
242 longest_row_chars: input_summary.longest_row_chars,
243 }
244 }
245
246 #[ztracing::instrument(skip_all)]
247 pub(crate) fn chunks<'a>(
248 &'a self,
249 range: Range<TabPoint>,
250 language_aware: bool,
251 highlights: Highlights<'a>,
252 ) -> TabChunks<'a> {
253 let (input_start, expanded_char_column, to_next_stop) =
254 self.tab_point_to_fold_point(range.start, Bias::Left);
255 let input_column = input_start.column();
256 let input_start = input_start.to_offset(&self.fold_snapshot);
257 let input_end = self
258 .tab_point_to_fold_point(range.end, Bias::Right)
259 .0
260 .to_offset(&self.fold_snapshot);
261 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
262 range.end.column() - range.start.column()
263 } else {
264 to_next_stop
265 };
266
267 TabChunks {
268 snapshot: self,
269 fold_chunks: self.fold_snapshot.chunks(
270 input_start..input_end,
271 language_aware,
272 highlights,
273 ),
274 input_column,
275 column: expanded_char_column,
276 max_expansion_column: self.max_expansion_column,
277 output_position: range.start.0,
278 max_output_position: range.end.0,
279 tab_size: self.tab_size,
280 chunk: Chunk {
281 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
282 is_tab: true,
283 ..Default::default()
284 },
285 inside_leading_tab: to_next_stop > 0,
286 }
287 }
288
289 #[ztracing::instrument(skip_all)]
290 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
291 self.fold_snapshot.row_infos(row)
292 }
293
294 #[cfg(test)]
295 #[ztracing::instrument(skip_all)]
296 pub fn text(&self) -> String {
297 self.chunks(
298 TabPoint::zero()..self.max_point(),
299 false,
300 Highlights::default(),
301 )
302 .map(|chunk| chunk.text)
303 .collect()
304 }
305
306 #[ztracing::instrument(skip_all)]
307 pub fn max_point(&self) -> TabPoint {
308 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
309 }
310
311 #[ztracing::instrument(skip_all)]
312 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
313 self.fold_point_to_tab_point(
314 self.fold_snapshot
315 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
316 )
317 }
318
319 #[ztracing::instrument(skip_all)]
320 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
321 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
322 let tab_cursor = TabStopCursor::new(chunks);
323 let expanded = self.expand_tabs(tab_cursor, input.column());
324 TabPoint::new(input.row(), expanded)
325 }
326
327 #[ztracing::instrument(skip_all)]
328 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
329 TabPointCursor { this: self }
330 }
331
332 #[ztracing::instrument(skip_all)]
333 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
334 let chunks = self
335 .fold_snapshot
336 .chunks_at(FoldPoint::new(output.row(), 0));
337
338 let tab_cursor = TabStopCursor::new(chunks);
339 let expanded = output.column();
340 let (collapsed, expanded_char_column, to_next_stop) =
341 self.collapse_tabs(tab_cursor, expanded, bias);
342
343 (
344 FoldPoint::new(output.row(), collapsed),
345 expanded_char_column,
346 to_next_stop,
347 )
348 }
349
350 #[ztracing::instrument(skip_all)]
351 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
352 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
353 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
354 self.fold_point_to_tab_point(fold_point)
355 }
356
357 #[ztracing::instrument(skip_all)]
358 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
359 let fold_point = self.tab_point_to_fold_point(point, bias).0;
360 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
361 self.fold_snapshot
362 .inlay_snapshot
363 .to_buffer_point(inlay_point)
364 }
365
366 #[ztracing::instrument(skip_all)]
367 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
368 where
369 I: Iterator<Item = Chunk<'a>>,
370 {
371 let tab_size = self.tab_size.get();
372
373 let end_column = column.min(self.max_expansion_column);
374 let mut seek_target = end_column;
375 let mut tab_count = 0;
376 let mut expanded_tab_len = 0;
377
378 while let Some(tab_stop) = cursor.seek(seek_target) {
379 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
380 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
381 tab_count += 1;
382 expanded_tab_len += tab_len;
383
384 seek_target = end_column - cursor.byte_offset;
385 }
386
387 let left_over_char_bytes = if !cursor.is_char_boundary() {
388 cursor.bytes_until_next_char().unwrap_or(0) as u32
389 } else {
390 0
391 };
392
393 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
394 let expanded_bytes =
395 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
396
397 expanded_bytes + column.saturating_sub(collapsed_bytes)
398 }
399
400 #[ztracing::instrument(skip_all)]
401 fn collapse_tabs<'a, I>(
402 &self,
403 mut cursor: TabStopCursor<'a, I>,
404 column: u32,
405 bias: Bias,
406 ) -> (u32, u32, u32)
407 where
408 I: Iterator<Item = Chunk<'a>>,
409 {
410 let tab_size = self.tab_size.get();
411 let mut collapsed_column = column;
412 let mut seek_target = column.min(self.max_expansion_column);
413 let mut tab_count = 0;
414 let mut expanded_tab_len = 0;
415
416 while let Some(tab_stop) = cursor.seek(seek_target) {
417 // Calculate how much we want to expand this tab stop (into spaces)
418 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
419 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
420 // Increment tab count
421 tab_count += 1;
422 // The count of how many spaces we've added to this line in place of tab bytes
423 expanded_tab_len += tab_len;
424
425 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
426 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
427
428 // Did we expand past the search target?
429 if expanded_bytes > column {
430 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
431 // We expanded past the search target, so need to account for the offshoot
432 expanded_chars -= expanded_bytes - column;
433 return match bias {
434 Bias::Left => (
435 cursor.byte_offset() - 1,
436 expanded_chars,
437 expanded_bytes - column,
438 ),
439 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
440 };
441 } else {
442 // otherwise we only want to move the cursor collapse column forward
443 collapsed_column = collapsed_column - tab_len + 1;
444 seek_target = (collapsed_column - cursor.byte_offset)
445 .min(self.max_expansion_column - cursor.byte_offset);
446 }
447 }
448
449 let collapsed_bytes = cursor.byte_offset();
450 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
451 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
452 (
453 collapsed_bytes + column.saturating_sub(expanded_bytes),
454 expanded_chars,
455 0,
456 )
457 }
458}
459
460// todo(lw): Implement TabPointCursor properly
461pub struct TabPointCursor<'this> {
462 this: &'this TabSnapshot,
463}
464
465impl TabPointCursor<'_> {
466 #[ztracing::instrument(skip_all)]
467 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
468 self.this.fold_point_to_tab_point(point)
469 }
470}
471
472#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
473pub struct TabPoint(pub Point);
474
475impl TabPoint {
476 pub fn new(row: u32, column: u32) -> Self {
477 Self(Point::new(row, column))
478 }
479
480 pub fn zero() -> Self {
481 Self::new(0, 0)
482 }
483
484 pub fn row(self) -> u32 {
485 self.0.row
486 }
487
488 pub fn column(self) -> u32 {
489 self.0.column
490 }
491}
492
493impl From<Point> for TabPoint {
494 fn from(point: Point) -> Self {
495 Self(point)
496 }
497}
498
499pub type TabEdit = text::Edit<TabPoint>;
500
501#[derive(Clone, Debug, Default, Eq, PartialEq)]
502pub struct TextSummary {
503 pub lines: Point,
504 pub first_line_chars: u32,
505 pub last_line_chars: u32,
506 pub longest_row: u32,
507 pub longest_row_chars: u32,
508}
509
510impl<'a> From<&'a str> for TextSummary {
511 #[ztracing::instrument(skip_all)]
512 fn from(text: &'a str) -> Self {
513 let sum = text::TextSummary::from(text);
514
515 TextSummary {
516 lines: sum.lines,
517 first_line_chars: sum.first_line_chars,
518 last_line_chars: sum.last_line_chars,
519 longest_row: sum.longest_row,
520 longest_row_chars: sum.longest_row_chars,
521 }
522 }
523}
524
525impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
526 #[ztracing::instrument(skip_all)]
527 fn add_assign(&mut self, other: &'a Self) {
528 let joined_chars = self.last_line_chars + other.first_line_chars;
529 if joined_chars > self.longest_row_chars {
530 self.longest_row = self.lines.row;
531 self.longest_row_chars = joined_chars;
532 }
533 if other.longest_row_chars > self.longest_row_chars {
534 self.longest_row = self.lines.row + other.longest_row;
535 self.longest_row_chars = other.longest_row_chars;
536 }
537
538 if self.lines.row == 0 {
539 self.first_line_chars += other.first_line_chars;
540 }
541
542 if other.lines.row == 0 {
543 self.last_line_chars += other.first_line_chars;
544 } else {
545 self.last_line_chars = other.last_line_chars;
546 }
547
548 self.lines += &other.lines;
549 }
550}
551
552pub struct TabChunks<'a> {
553 snapshot: &'a TabSnapshot,
554 max_expansion_column: u32,
555 max_output_position: Point,
556 tab_size: NonZeroU32,
557 // region: iteration state
558 fold_chunks: FoldChunks<'a>,
559 chunk: Chunk<'a>,
560 column: u32,
561 output_position: Point,
562 input_column: u32,
563 inside_leading_tab: bool,
564 // endregion: iteration state
565}
566
567impl TabChunks<'_> {
568 #[ztracing::instrument(skip_all)]
569 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
570 let (input_start, expanded_char_column, to_next_stop) = self
571 .snapshot
572 .tab_point_to_fold_point(range.start, Bias::Left);
573 let input_column = input_start.column();
574 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
575 let input_end = self
576 .snapshot
577 .tab_point_to_fold_point(range.end, Bias::Right)
578 .0
579 .to_offset(&self.snapshot.fold_snapshot);
580 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
581 range.end.column() - range.start.column()
582 } else {
583 to_next_stop
584 };
585
586 self.fold_chunks.seek(input_start..input_end);
587 self.input_column = input_column;
588 self.column = expanded_char_column;
589 self.output_position = range.start.0;
590 self.max_output_position = range.end.0;
591 self.chunk = Chunk {
592 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
593 is_tab: true,
594 chars: 1u128.unbounded_shl(to_next_stop) - 1,
595 ..Default::default()
596 };
597 self.inside_leading_tab = to_next_stop > 0;
598 }
599}
600
601impl<'a> Iterator for TabChunks<'a> {
602 type Item = Chunk<'a>;
603
604 #[ztracing::instrument(skip_all)]
605 fn next(&mut self) -> Option<Self::Item> {
606 if self.chunk.text.is_empty() {
607 if let Some(chunk) = self.fold_chunks.next() {
608 self.chunk = chunk;
609 if self.inside_leading_tab {
610 self.chunk.text = &self.chunk.text[1..];
611 self.inside_leading_tab = false;
612 self.input_column += 1;
613 }
614 } else {
615 return None;
616 }
617 }
618
619 //todo(improve performance by using tab cursor)
620 for (ix, c) in self.chunk.text.char_indices() {
621 match c {
622 '\t' if ix > 0 => {
623 let (prefix, suffix) = self.chunk.text.split_at(ix);
624
625 let mask = 1u128.unbounded_shl(ix as u32).wrapping_sub(1);
626 let chars = self.chunk.chars & mask;
627 let tabs = self.chunk.tabs & mask;
628 self.chunk.tabs = self.chunk.tabs.unbounded_shr(ix as u32);
629 self.chunk.chars = self.chunk.chars.unbounded_shr(ix as u32);
630 self.chunk.text = suffix;
631 return Some(Chunk {
632 text: prefix,
633 chars,
634 tabs,
635 ..self.chunk.clone()
636 });
637 }
638 '\t' => {
639 self.chunk.text = &self.chunk.text[1..];
640 self.chunk.tabs >>= 1;
641 self.chunk.chars >>= 1;
642 let tab_size = if self.input_column < self.max_expansion_column {
643 self.tab_size.get()
644 } else {
645 1
646 };
647 let mut len = tab_size - self.column % tab_size;
648 let next_output_position = cmp::min(
649 self.output_position + Point::new(0, len),
650 self.max_output_position,
651 );
652 len = next_output_position.column - self.output_position.column;
653 self.column += len;
654 self.input_column += 1;
655 self.output_position = next_output_position;
656 return Some(Chunk {
657 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
658 is_tab: true,
659 chars: 1u128.unbounded_shl(len) - 1,
660 tabs: 0,
661 ..self.chunk.clone()
662 });
663 }
664 '\n' => {
665 self.column = 0;
666 self.input_column = 0;
667 self.output_position += Point::new(1, 0);
668 }
669 _ => {
670 self.column += 1;
671 if !self.inside_leading_tab {
672 self.input_column += c.len_utf8() as u32;
673 }
674 self.output_position.column += c.len_utf8() as u32;
675 }
676 }
677 }
678
679 Some(mem::take(&mut self.chunk))
680 }
681}
682
683#[cfg(test)]
684mod tests {
685 use super::*;
686 use crate::{
687 MultiBuffer,
688 display_map::{
689 fold_map::{FoldMap, FoldOffset},
690 inlay_map::InlayMap,
691 },
692 };
693 use multi_buffer::MultiBufferOffset;
694 use rand::{Rng, prelude::StdRng};
695 use util;
696
697 impl TabSnapshot {
698 fn expected_collapse_tabs(
699 &self,
700 chars: impl Iterator<Item = char>,
701 column: u32,
702 bias: Bias,
703 ) -> (u32, u32, u32) {
704 let tab_size = self.tab_size.get();
705
706 let mut expanded_bytes = 0;
707 let mut expanded_chars = 0;
708 let mut collapsed_bytes = 0;
709 for c in chars {
710 if expanded_bytes >= column {
711 break;
712 }
713 if collapsed_bytes >= self.max_expansion_column {
714 break;
715 }
716
717 if c == '\t' {
718 let tab_len = tab_size - (expanded_chars % tab_size);
719 expanded_chars += tab_len;
720 expanded_bytes += tab_len;
721 if expanded_bytes > column {
722 expanded_chars -= expanded_bytes - column;
723 return match bias {
724 Bias::Left => {
725 (collapsed_bytes, expanded_chars, expanded_bytes - column)
726 }
727 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
728 };
729 }
730 } else {
731 expanded_chars += 1;
732 expanded_bytes += c.len_utf8() as u32;
733 }
734
735 if expanded_bytes > column && matches!(bias, Bias::Left) {
736 expanded_chars -= 1;
737 break;
738 }
739
740 collapsed_bytes += c.len_utf8() as u32;
741 }
742
743 (
744 collapsed_bytes + column.saturating_sub(expanded_bytes),
745 expanded_chars,
746 0,
747 )
748 }
749
750 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
751 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
752 let expanded = self.expected_expand_tabs(chars, input.column());
753 TabPoint::new(input.row(), expanded)
754 }
755
756 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
757 let tab_size = self.tab_size.get();
758
759 let mut expanded_chars = 0;
760 let mut expanded_bytes = 0;
761 let mut collapsed_bytes = 0;
762 let end_column = column.min(self.max_expansion_column);
763 for c in chars {
764 if collapsed_bytes >= end_column {
765 break;
766 }
767 if c == '\t' {
768 let tab_len = tab_size - expanded_chars % tab_size;
769 expanded_bytes += tab_len;
770 expanded_chars += tab_len;
771 } else {
772 expanded_bytes += c.len_utf8() as u32;
773 expanded_chars += 1;
774 }
775 collapsed_bytes += c.len_utf8() as u32;
776 }
777
778 expanded_bytes + column.saturating_sub(collapsed_bytes)
779 }
780
781 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
782 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
783 let expanded = output.column();
784 let (collapsed, expanded_char_column, to_next_stop) =
785 self.expected_collapse_tabs(chars, expanded, bias);
786 (
787 FoldPoint::new(output.row(), collapsed),
788 expanded_char_column,
789 to_next_stop,
790 )
791 }
792 }
793
794 #[gpui::test]
795 fn test_expand_tabs(cx: &mut gpui::App) {
796 let test_values = [
797 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
798 (" \twςe", 4),
799 ("fε", 1),
800 ("i❎\t", 3),
801 ];
802 let buffer = MultiBuffer::build_simple("", cx);
803 let buffer_snapshot = buffer.read(cx).snapshot(cx);
804 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
805 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
806 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
807
808 for (text, column) in test_values {
809 let mut tabs = 0u128;
810 let mut chars = 0u128;
811 for (idx, c) in text.char_indices() {
812 if c == '\t' {
813 tabs |= 1 << idx;
814 }
815 chars |= 1 << idx;
816 }
817
818 let chunks = [Chunk {
819 text,
820 tabs,
821 chars,
822 ..Default::default()
823 }];
824
825 let cursor = TabStopCursor::new(chunks);
826
827 assert_eq!(
828 tab_snapshot.expected_expand_tabs(text.chars(), column),
829 tab_snapshot.expand_tabs(cursor, column)
830 );
831 }
832 }
833
834 #[gpui::test]
835 fn test_collapse_tabs(cx: &mut gpui::App) {
836 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
837
838 let buffer = MultiBuffer::build_simple(input, cx);
839 let buffer_snapshot = buffer.read(cx).snapshot(cx);
840 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
841 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
842 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
843
844 for (ix, _) in input.char_indices() {
845 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
846
847 assert_eq!(
848 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
849 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
850 "Failed with tab_point at column {ix}"
851 );
852 assert_eq!(
853 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
854 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
855 "Failed with tab_point at column {ix}"
856 );
857
858 assert_eq!(
859 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
860 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
861 "Failed with tab_point at column {ix}"
862 );
863 assert_eq!(
864 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
865 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
866 "Failed with tab_point at column {ix}"
867 );
868 }
869 }
870
871 #[gpui::test]
872 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
873 // This test reproduces a specific panic where to_fold_point returns incorrect results
874 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
875
876 let text = "γ\tw⭐\n🍐🍗 \t";
877 let buffer = MultiBuffer::build_simple(text, cx);
878 let buffer_snapshot = buffer.read(cx).snapshot(cx);
879 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
880 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
881 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
882
883 // This should panic with the expected vs actual mismatch
884 let tab_point = TabPoint::new(0, 9);
885 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
886 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
887
888 assert_eq!(result, expected);
889 }
890
891 #[gpui::test(iterations = 100)]
892 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
893 // Generate random input string with up to 200 characters including tabs
894 // to stay within the MAX_EXPANSION_COLUMN limit of 256
895 let len = rng.random_range(0..=2048);
896 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
897 let mut input = String::with_capacity(len);
898
899 for _ in 0..len {
900 if rng.random_bool(0.1) {
901 // 10% chance of inserting a tab
902 input.push('\t');
903 } else {
904 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
905 let ch = loop {
906 let ascii_code = rng.random_range(32..=126); // printable ASCII range
907 let ch = ascii_code as u8 as char;
908 if ch != '\t' {
909 break ch;
910 }
911 };
912 input.push(ch);
913 }
914 }
915
916 let buffer = MultiBuffer::build_simple(&input, cx);
917 let buffer_snapshot = buffer.read(cx).snapshot(cx);
918 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
919 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
920 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
921 tab_snapshot.max_expansion_column = rng.random_range(0..323);
922 tab_snapshot.tab_size = tab_size;
923
924 for (ix, _) in input.char_indices() {
925 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
926
927 assert_eq!(
928 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
929 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
930 "Failed with input: {}, with idx: {ix}",
931 input
932 );
933 assert_eq!(
934 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
935 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
936 "Failed with input: {}, with idx: {ix}",
937 input
938 );
939
940 assert_eq!(
941 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
942 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
943 "Failed with input: {}, with idx: {ix}",
944 input
945 );
946 assert_eq!(
947 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
948 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
949 "Failed with input: {}, with idx: {ix}",
950 input
951 );
952 }
953 }
954
955 #[gpui::test]
956 fn test_long_lines(cx: &mut gpui::App) {
957 let max_expansion_column = 12;
958 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
959 let output = "A BC DEF G HI J K L M";
960
961 let buffer = MultiBuffer::build_simple(input, cx);
962 let buffer_snapshot = buffer.read(cx).snapshot(cx);
963 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
964 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
965 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
966
967 tab_snapshot.max_expansion_column = max_expansion_column;
968 assert_eq!(tab_snapshot.text(), output);
969
970 for (ix, c) in input.char_indices() {
971 assert_eq!(
972 tab_snapshot
973 .chunks(
974 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
975 false,
976 Highlights::default(),
977 )
978 .map(|c| c.text)
979 .collect::<String>(),
980 &output[ix..],
981 "text from index {ix}"
982 );
983
984 if c != '\t' {
985 let input_point = Point::new(0, ix as u32);
986 let output_point = Point::new(0, output.find(c).unwrap() as u32);
987 assert_eq!(
988 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
989 TabPoint(output_point),
990 "to_tab_point({input_point:?})"
991 );
992 assert_eq!(
993 tab_snapshot
994 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
995 .0,
996 FoldPoint(input_point),
997 "to_fold_point({output_point:?})"
998 );
999 }
1000 }
1001 }
1002
1003 #[gpui::test]
1004 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
1005 let max_expansion_column = 8;
1006 let input = "abcdefg⋯hij";
1007
1008 let buffer = MultiBuffer::build_simple(input, cx);
1009 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1010 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1011 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1012 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1013
1014 tab_snapshot.max_expansion_column = max_expansion_column;
1015 assert_eq!(tab_snapshot.text(), input);
1016 }
1017
1018 #[gpui::test]
1019 fn test_marking_tabs(cx: &mut gpui::App) {
1020 let input = "\t \thello";
1021
1022 let buffer = MultiBuffer::build_simple(input, cx);
1023 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1024 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1025 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1026 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1027
1028 assert_eq!(
1029 chunks(&tab_snapshot, TabPoint::zero()),
1030 vec![
1031 (" ".to_string(), true),
1032 (" ".to_string(), false),
1033 (" ".to_string(), true),
1034 ("hello".to_string(), false),
1035 ]
1036 );
1037 assert_eq!(
1038 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1039 vec![
1040 (" ".to_string(), true),
1041 (" ".to_string(), false),
1042 (" ".to_string(), true),
1043 ("hello".to_string(), false),
1044 ]
1045 );
1046
1047 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1048 let mut chunks = Vec::new();
1049 let mut was_tab = false;
1050 let mut text = String::new();
1051 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1052 {
1053 if chunk.is_tab != was_tab {
1054 if !text.is_empty() {
1055 chunks.push((mem::take(&mut text), was_tab));
1056 }
1057 was_tab = chunk.is_tab;
1058 }
1059 text.push_str(chunk.text);
1060 }
1061
1062 if !text.is_empty() {
1063 chunks.push((text, was_tab));
1064 }
1065 chunks
1066 }
1067 }
1068
1069 #[gpui::test(iterations = 100)]
1070 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1071 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1072 let len = rng.random_range(0..30);
1073 let buffer = if rng.random() {
1074 let text = util::RandomCharIter::new(&mut rng)
1075 .take(len)
1076 .collect::<String>();
1077 MultiBuffer::build_simple(&text, cx)
1078 } else {
1079 MultiBuffer::build_random(&mut rng, cx)
1080 };
1081 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1082 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1083
1084 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1085 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1086 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1087 fold_map.randomly_mutate(&mut rng);
1088 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1089 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1090 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1091 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1092
1093 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1094 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1095
1096 let text = text::Rope::from(tabs_snapshot.text().as_str());
1097 log::info!(
1098 "TabMap text (tab size: {}): {:?}",
1099 tab_size,
1100 tabs_snapshot.text(),
1101 );
1102
1103 for _ in 0..5 {
1104 let end_row = rng.random_range(0..=text.max_point().row);
1105 let end_column = rng.random_range(0..=text.line_len(end_row));
1106 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1107 let start_row = rng.random_range(0..=text.max_point().row);
1108 let start_column = rng.random_range(0..=text.line_len(start_row));
1109 let mut start =
1110 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1111 if start > end {
1112 mem::swap(&mut start, &mut end);
1113 }
1114
1115 let expected_text = text
1116 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1117 .collect::<String>();
1118 let expected_summary = TextSummary::from(expected_text.as_str());
1119 assert_eq!(
1120 tabs_snapshot
1121 .chunks(start..end, false, Highlights::default())
1122 .map(|c| c.text)
1123 .collect::<String>(),
1124 expected_text,
1125 "chunks({:?}..{:?})",
1126 start,
1127 end
1128 );
1129
1130 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1131 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1132 actual_summary.longest_row = expected_summary.longest_row;
1133 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1134 }
1135 assert_eq!(actual_summary, expected_summary);
1136 }
1137
1138 for row in 0..=text.max_point().row {
1139 assert_eq!(
1140 tabs_snapshot.line_len(row),
1141 text.line_len(row),
1142 "line_len({row})"
1143 );
1144 }
1145 }
1146
1147 #[gpui::test(iterations = 100)]
1148 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1149 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1150 let len = rng.random_range(0..=2000);
1151
1152 // Generate random text using RandomCharIter
1153 let text = util::RandomCharIter::new(&mut rng)
1154 .take(len)
1155 .collect::<String>();
1156
1157 // Create buffer and tab map
1158 let buffer = MultiBuffer::build_simple(&text, cx);
1159 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1160 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1161 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1162 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1163
1164 let mut next_inlay_id = 0;
1165 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1166 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1167 let max_fold_point = fold_snapshot.max_point();
1168 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1169
1170 // Test random fold points
1171 for _ in 0..50 {
1172 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1173 // Generate random fold point
1174 let row = rng.random_range(0..=max_fold_point.row());
1175 let max_column = if row < max_fold_point.row() {
1176 fold_snapshot.line_len(row)
1177 } else {
1178 max_fold_point.column()
1179 };
1180 let column = rng.random_range(0..=max_column + 10);
1181 let fold_point = FoldPoint::new(row, column);
1182
1183 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1184 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1185
1186 assert_eq!(
1187 actual, expected,
1188 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1189 fold_point, text
1190 );
1191 }
1192 }
1193
1194 #[gpui::test]
1195 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1196 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1197 let buffer = MultiBuffer::build_simple(text, cx);
1198 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1199 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1200 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1201 let chunks = fold_snapshot.chunks(
1202 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1203 false,
1204 Default::default(),
1205 );
1206 let mut cursor = TabStopCursor::new(chunks);
1207 assert!(cursor.seek(0).is_none());
1208 let mut tab_stops = Vec::new();
1209
1210 let mut all_tab_stops = Vec::new();
1211 let mut byte_offset = 0;
1212 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1213 byte_offset += ch.len_utf8() as u32;
1214
1215 if ch == '\t' {
1216 all_tab_stops.push(TabStop {
1217 byte_offset,
1218 char_offset: offset as u32 + 1,
1219 });
1220 }
1221 }
1222
1223 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1224 tab_stops.push(tab_stop);
1225 }
1226 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1227
1228 assert_eq!(cursor.byte_offset(), byte_offset);
1229 }
1230
1231 #[gpui::test]
1232 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1233 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1234
1235 let buffer = MultiBuffer::build_simple(input, cx);
1236 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1237 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1238 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1239
1240 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1241 let mut cursor = TabStopCursor::new(chunks);
1242
1243 let mut actual_tab_stops = Vec::new();
1244
1245 let mut expected_tab_stops = Vec::new();
1246 let mut byte_offset = 0;
1247 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1248 byte_offset += ch.len_utf8() as u32;
1249
1250 if ch == '\t' {
1251 expected_tab_stops.push(TabStop {
1252 byte_offset,
1253 char_offset: offset as u32 + 1,
1254 });
1255 }
1256 }
1257
1258 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1259 actual_tab_stops.push(tab_stop);
1260 }
1261 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1262
1263 assert_eq!(cursor.byte_offset(), byte_offset);
1264 }
1265
1266 #[gpui::test(iterations = 100)]
1267 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1268 // Generate random input string with up to 512 characters including tabs
1269 let len = rng.random_range(0..=2048);
1270 let mut input = String::with_capacity(len);
1271
1272 let mut skip_tabs = rng.random_bool(0.10);
1273 for idx in 0..len {
1274 if idx % 128 == 0 {
1275 skip_tabs = rng.random_bool(0.10);
1276 }
1277
1278 if rng.random_bool(0.15) && !skip_tabs {
1279 input.push('\t');
1280 } else {
1281 let ch = loop {
1282 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1283 let ch = ascii_code as u8 as char;
1284 if ch != '\t' {
1285 break ch;
1286 }
1287 };
1288 input.push(ch);
1289 }
1290 }
1291
1292 // Build the buffer and create cursor
1293 let buffer = MultiBuffer::build_simple(&input, cx);
1294 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1295 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1296 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1297
1298 // First, collect all expected tab positions
1299 let mut all_tab_stops = Vec::new();
1300 let mut byte_offset = 1;
1301 let mut char_offset = 1;
1302 for ch in buffer_snapshot.text().chars() {
1303 if ch == '\t' {
1304 all_tab_stops.push(TabStop {
1305 byte_offset,
1306 char_offset,
1307 });
1308 }
1309 byte_offset += ch.len_utf8() as u32;
1310 char_offset += 1;
1311 }
1312
1313 // Test with various distances
1314 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1315 // let distances = vec![150];
1316
1317 for distance in distances {
1318 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1319 let mut cursor = TabStopCursor::new(chunks);
1320
1321 let mut found_tab_stops = Vec::new();
1322 let mut position = distance;
1323 while let Some(tab_stop) = cursor.seek(position) {
1324 found_tab_stops.push(tab_stop);
1325 position = distance - tab_stop.byte_offset;
1326 }
1327
1328 let expected_found_tab_stops: Vec<_> = all_tab_stops
1329 .iter()
1330 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1331 .cloned()
1332 .collect();
1333
1334 pretty_assertions::assert_eq!(
1335 found_tab_stops,
1336 expected_found_tab_stops,
1337 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1338 distance,
1339 input
1340 );
1341
1342 let final_position = cursor.byte_offset();
1343 if !found_tab_stops.is_empty() {
1344 let last_tab_stop = found_tab_stops.last().unwrap();
1345 assert!(
1346 final_position >= last_tab_stop.byte_offset,
1347 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1348 final_position,
1349 last_tab_stop.byte_offset,
1350 input
1351 );
1352 }
1353 }
1354 }
1355
1356 #[gpui::test]
1357 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1358 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1359 let buffer = MultiBuffer::build_simple(text, cx);
1360 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1361 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1362 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1363 let chunks = fold_snapshot.chunks(
1364 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1365 false,
1366 Default::default(),
1367 );
1368 let mut cursor = TabStopCursor::new(chunks);
1369 assert!(cursor.seek(0).is_none());
1370
1371 let mut expected_tab_stops = Vec::new();
1372 let mut byte_offset = 0;
1373 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1374 byte_offset += ch.len_utf8() as u32;
1375
1376 if ch == '\t' {
1377 expected_tab_stops.push(TabStop {
1378 byte_offset,
1379 char_offset: i as u32 + 1,
1380 });
1381 }
1382 }
1383
1384 let mut actual_tab_stops = Vec::new();
1385 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1386 actual_tab_stops.push(tab_stop);
1387 }
1388
1389 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1390
1391 assert_eq!(cursor.byte_offset(), byte_offset);
1392 }
1393
1394 #[gpui::test(iterations = 100)]
1395 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1396 // Generate random input string with up to 512 characters including tabs
1397 let len = rng.random_range(0..=2048);
1398 let input = util::RandomCharIter::new(&mut rng)
1399 .take(len)
1400 .collect::<String>();
1401
1402 // Build the buffer and create cursor
1403 let buffer = MultiBuffer::build_simple(&input, cx);
1404 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1405 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1406 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1407
1408 // First, collect all expected tab positions
1409 let mut all_tab_stops = Vec::new();
1410 let mut byte_offset = 0;
1411 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1412 byte_offset += ch.len_utf8() as u32;
1413 if ch == '\t' {
1414 all_tab_stops.push(TabStop {
1415 byte_offset,
1416 char_offset: i as u32 + 1,
1417 });
1418 }
1419 }
1420
1421 // Test with various distances
1422 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1423 let distances = vec![150];
1424
1425 for distance in distances {
1426 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1427 let mut cursor = TabStopCursor::new(chunks);
1428
1429 let mut found_tab_stops = Vec::new();
1430 let mut position = distance;
1431 while let Some(tab_stop) = cursor.seek(position) {
1432 found_tab_stops.push(tab_stop);
1433 position = distance - tab_stop.byte_offset;
1434 }
1435
1436 let expected_found_tab_stops: Vec<_> = all_tab_stops
1437 .iter()
1438 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1439 .cloned()
1440 .collect();
1441
1442 pretty_assertions::assert_eq!(
1443 found_tab_stops,
1444 expected_found_tab_stops,
1445 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1446 distance,
1447 input
1448 );
1449
1450 let final_position = cursor.byte_offset();
1451 if !found_tab_stops.is_empty() {
1452 let last_tab_stop = found_tab_stops.last().unwrap();
1453 assert!(
1454 final_position >= last_tab_stop.byte_offset,
1455 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1456 final_position,
1457 last_tab_stop.byte_offset,
1458 input
1459 );
1460 }
1461 }
1462 }
1463}
1464
1465struct TabStopCursor<'a, I>
1466where
1467 I: Iterator<Item = Chunk<'a>>,
1468{
1469 chunks: I,
1470 byte_offset: u32,
1471 char_offset: u32,
1472 /// Chunk
1473 /// last tab position iterated through
1474 current_chunk: Option<(Chunk<'a>, u32)>,
1475}
1476
1477impl<'a, I> TabStopCursor<'a, I>
1478where
1479 I: Iterator<Item = Chunk<'a>>,
1480{
1481 #[ztracing::instrument(skip_all)]
1482 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1483 Self {
1484 chunks: chunks.into_iter(),
1485 byte_offset: 0,
1486 char_offset: 0,
1487 current_chunk: None,
1488 }
1489 }
1490
1491 #[ztracing::instrument(skip_all)]
1492 fn bytes_until_next_char(&self) -> Option<usize> {
1493 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1494 let mut idx = *idx;
1495 let mut diff = 0;
1496 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1497 idx -= 1;
1498 diff += 1;
1499 }
1500
1501 if chunk.chars & (1 << idx) != 0 {
1502 Some(
1503 (chunk.text[idx as usize..].chars().next()?)
1504 .len_utf8()
1505 .saturating_sub(diff),
1506 )
1507 } else {
1508 None
1509 }
1510 })
1511 }
1512
1513 #[ztracing::instrument(skip_all)]
1514 fn is_char_boundary(&self) -> bool {
1515 self.current_chunk
1516 .as_ref()
1517 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1518 }
1519
1520 /// distance: length to move forward while searching for the next tab stop
1521 #[ztracing::instrument(skip_all)]
1522 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1523 if distance == 0 {
1524 return None;
1525 }
1526
1527 let mut distance_traversed = 0;
1528
1529 while let Some((mut chunk, chunk_position)) = self
1530 .current_chunk
1531 .take()
1532 .or_else(|| self.chunks.next().zip(Some(0)))
1533 {
1534 if chunk.tabs == 0 {
1535 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1536 if chunk_distance + distance_traversed >= distance {
1537 let overshoot = distance_traversed.abs_diff(distance);
1538
1539 self.byte_offset += overshoot;
1540 self.char_offset += get_char_offset(
1541 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1542 chunk.chars,
1543 );
1544
1545 if chunk_position + overshoot < 128 {
1546 self.current_chunk = Some((chunk, chunk_position + overshoot));
1547 }
1548
1549 return None;
1550 }
1551
1552 self.byte_offset += chunk_distance;
1553 self.char_offset += get_char_offset(
1554 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1555 chunk.chars,
1556 );
1557 distance_traversed += chunk_distance;
1558 continue;
1559 }
1560 let tab_position = chunk.tabs.trailing_zeros() + 1;
1561
1562 if distance_traversed + tab_position - chunk_position > distance {
1563 let cursor_position = distance_traversed.abs_diff(distance);
1564
1565 self.char_offset += get_char_offset(
1566 chunk_position..(chunk_position + cursor_position - 1),
1567 chunk.chars,
1568 );
1569 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1570 self.byte_offset += cursor_position;
1571
1572 return None;
1573 }
1574
1575 self.byte_offset += tab_position - chunk_position;
1576 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1577
1578 let tabstop = TabStop {
1579 char_offset: self.char_offset,
1580 byte_offset: self.byte_offset,
1581 };
1582
1583 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1584
1585 if tab_position as usize != chunk.text.len() {
1586 self.current_chunk = Some((chunk, tab_position));
1587 }
1588
1589 return Some(tabstop);
1590 }
1591
1592 None
1593 }
1594
1595 fn byte_offset(&self) -> u32 {
1596 self.byte_offset
1597 }
1598
1599 fn char_offset(&self) -> u32 {
1600 self.char_offset
1601 }
1602}
1603
1604#[inline(always)]
1605fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1606 if range.start == range.end {
1607 return if (1u128 << range.start) & bit_map == 0 {
1608 0
1609 } else {
1610 1
1611 };
1612 }
1613 let end_shift: u128 = 127u128 - range.end as u128;
1614 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1615 bit_mask = (bit_mask << end_shift) >> end_shift;
1616 let bit_map = bit_map & bit_mask;
1617
1618 bit_map.count_ones()
1619}
1620
1621#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1622struct TabStop {
1623 char_offset: u32,
1624 byte_offset: u32,
1625}