1use super::{
2 Highlights,
3 fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot},
4};
5
6use language::Point;
7use multi_buffer::MultiBufferSnapshot;
8use std::{cmp, mem, num::NonZeroU32, ops::Range};
9use sum_tree::Bias;
10
11const MAX_EXPANSION_COLUMN: u32 = 256;
12
13// Handles a tab width <= 128
14const SPACES: &[u8; rope::Chunk::MASK_BITS] = &[b' '; _];
15const MAX_TABS: NonZeroU32 = NonZeroU32::new(SPACES.len() as u32).unwrap();
16
17/// Keeps track of hard tabs in a text buffer.
18///
19/// See the [`display_map` module documentation](crate::display_map) for more information.
20pub struct TabMap(TabSnapshot);
21
22impl TabMap {
23 #[ztracing::instrument(skip_all)]
24 pub fn new(fold_snapshot: FoldSnapshot, tab_size: NonZeroU32) -> (Self, TabSnapshot) {
25 let snapshot = TabSnapshot {
26 fold_snapshot,
27 tab_size: tab_size.min(MAX_TABS),
28 max_expansion_column: MAX_EXPANSION_COLUMN,
29 version: 0,
30 };
31 (Self(snapshot.clone()), snapshot)
32 }
33
34 #[cfg(test)]
35 pub fn set_max_expansion_column(&mut self, column: u32) -> TabSnapshot {
36 self.0.max_expansion_column = column;
37 self.0.clone()
38 }
39
40 #[ztracing::instrument(skip_all)]
41 pub fn sync(
42 &mut self,
43 fold_snapshot: FoldSnapshot,
44 mut fold_edits: Vec<FoldEdit>,
45 tab_size: NonZeroU32,
46 ) -> (TabSnapshot, Vec<TabEdit>) {
47 let old_snapshot = &mut self.0;
48 let mut new_snapshot = TabSnapshot {
49 fold_snapshot,
50 tab_size: tab_size.min(MAX_TABS),
51 max_expansion_column: old_snapshot.max_expansion_column,
52 version: old_snapshot.version,
53 };
54
55 if old_snapshot.fold_snapshot.version != new_snapshot.fold_snapshot.version {
56 new_snapshot.version += 1;
57 }
58
59 let tab_edits = if old_snapshot.tab_size == new_snapshot.tab_size {
60 // Expand each edit to include the next tab on the same line as the edit,
61 // and any subsequent tabs on that line that moved across the tab expansion
62 // boundary.
63 for fold_edit in &mut fold_edits {
64 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
65 let old_end_row_successor_offset = cmp::min(
66 FoldPoint::new(old_end.row() + 1, 0),
67 old_snapshot.fold_snapshot.max_point(),
68 )
69 .to_offset(&old_snapshot.fold_snapshot);
70 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
71
72 let mut offset_from_edit = 0;
73 let mut first_tab_offset = None;
74 let mut last_tab_with_changed_expansion_offset = None;
75 'outer: for chunk in old_snapshot.fold_snapshot.chunks(
76 fold_edit.old.end..old_end_row_successor_offset,
77 false,
78 Highlights::default(),
79 ) {
80 // todo(performance use tabs bitmask)
81 for (ix, _) in chunk.text.match_indices('\t') {
82 let offset_from_edit = offset_from_edit + (ix as u32);
83 if first_tab_offset.is_none() {
84 first_tab_offset = Some(offset_from_edit);
85 }
86
87 let old_column = old_end.column() + offset_from_edit;
88 let new_column = new_end.column() + offset_from_edit;
89 let was_expanded = old_column < old_snapshot.max_expansion_column;
90 let is_expanded = new_column < new_snapshot.max_expansion_column;
91 if was_expanded != is_expanded {
92 last_tab_with_changed_expansion_offset = Some(offset_from_edit);
93 } else if !was_expanded && !is_expanded {
94 break 'outer;
95 }
96 }
97
98 offset_from_edit += chunk.text.len() as u32;
99 if old_end.column() + offset_from_edit >= old_snapshot.max_expansion_column
100 && new_end.column() + offset_from_edit >= new_snapshot.max_expansion_column
101 {
102 break;
103 }
104 }
105
106 if let Some(offset) = last_tab_with_changed_expansion_offset.or(first_tab_offset) {
107 fold_edit.old.end.0 += offset as usize + 1;
108 fold_edit.new.end.0 += offset as usize + 1;
109 }
110 }
111
112 let _old_alloc_ptr = fold_edits.as_ptr();
113 // Combine any edits that overlap due to the expansion.
114 let mut fold_edits = fold_edits.into_iter();
115 if let Some(mut first_edit) = fold_edits.next() {
116 // This code relies on reusing allocations from the Vec<_> - at the time of writing .flatten() prevents them.
117 #[allow(clippy::filter_map_identity)]
118 let mut v: Vec<_> = fold_edits
119 .scan(&mut first_edit, |state, edit| {
120 if state.old.end >= edit.old.start {
121 state.old.end = edit.old.end;
122 state.new.end = edit.new.end;
123 Some(None) // Skip this edit, it's merged
124 } else {
125 let new_state = edit;
126 let result = Some(Some(state.clone())); // Yield the previous edit
127 **state = new_state;
128 result
129 }
130 })
131 .filter_map(|x| x)
132 .collect();
133 v.push(first_edit);
134 debug_assert_eq!(v.as_ptr(), _old_alloc_ptr, "Fold edits were reallocated");
135 v.into_iter()
136 .map(|fold_edit| {
137 let old_start = fold_edit.old.start.to_point(&old_snapshot.fold_snapshot);
138 let old_end = fold_edit.old.end.to_point(&old_snapshot.fold_snapshot);
139 let new_start = fold_edit.new.start.to_point(&new_snapshot.fold_snapshot);
140 let new_end = fold_edit.new.end.to_point(&new_snapshot.fold_snapshot);
141 TabEdit {
142 old: old_snapshot.fold_point_to_tab_point(old_start)
143 ..old_snapshot.fold_point_to_tab_point(old_end),
144 new: new_snapshot.fold_point_to_tab_point(new_start)
145 ..new_snapshot.fold_point_to_tab_point(new_end),
146 }
147 })
148 .collect()
149 } else {
150 vec![]
151 }
152 } else {
153 new_snapshot.version += 1;
154 vec![TabEdit {
155 old: TabPoint::zero()..old_snapshot.max_point(),
156 new: TabPoint::zero()..new_snapshot.max_point(),
157 }]
158 };
159 *old_snapshot = new_snapshot;
160 (old_snapshot.clone(), tab_edits)
161 }
162}
163
164#[derive(Clone)]
165pub struct TabSnapshot {
166 pub fold_snapshot: FoldSnapshot,
167 pub tab_size: NonZeroU32,
168 pub max_expansion_column: u32,
169 pub version: usize,
170}
171
172impl std::ops::Deref for TabSnapshot {
173 type Target = FoldSnapshot;
174
175 fn deref(&self) -> &Self::Target {
176 &self.fold_snapshot
177 }
178}
179
180impl TabSnapshot {
181 #[ztracing::instrument(skip_all)]
182 pub fn buffer_snapshot(&self) -> &MultiBufferSnapshot {
183 &self.fold_snapshot.inlay_snapshot.buffer
184 }
185
186 #[ztracing::instrument(skip_all)]
187 pub fn line_len(&self, row: u32) -> u32 {
188 let max_point = self.max_point();
189 if row < max_point.row() {
190 self.fold_point_to_tab_point(FoldPoint::new(row, self.fold_snapshot.line_len(row)))
191 .0
192 .column
193 } else {
194 max_point.column()
195 }
196 }
197
198 #[ztracing::instrument(skip_all)]
199 pub fn text_summary(&self) -> TextSummary {
200 self.text_summary_for_range(TabPoint::zero()..self.max_point())
201 }
202
203 #[ztracing::instrument(skip_all, fields(rows))]
204 pub fn text_summary_for_range(&self, range: Range<TabPoint>) -> TextSummary {
205 let input_start = self.tab_point_to_fold_point(range.start, Bias::Left).0;
206 let input_end = self.tab_point_to_fold_point(range.end, Bias::Right).0;
207 let input_summary = self
208 .fold_snapshot
209 .text_summary_for_range(input_start..input_end);
210
211 let line_end = if range.start.row() == range.end.row() {
212 range.end
213 } else {
214 self.max_point()
215 };
216 let first_line_chars = self
217 .chunks(range.start..line_end, false, Highlights::default())
218 .flat_map(|chunk| chunk.text.chars())
219 .take_while(|&c| c != '\n')
220 .count() as u32;
221
222 let last_line_chars = if range.start.row() == range.end.row() {
223 first_line_chars
224 } else {
225 self.chunks(
226 TabPoint::new(range.end.row(), 0)..range.end,
227 false,
228 Highlights::default(),
229 )
230 .flat_map(|chunk| chunk.text.chars())
231 .count() as u32
232 };
233
234 TextSummary {
235 lines: range.end.0 - range.start.0,
236 first_line_chars,
237 last_line_chars,
238 longest_row: input_summary.longest_row,
239 longest_row_chars: input_summary.longest_row_chars,
240 }
241 }
242
243 #[ztracing::instrument(skip_all)]
244 pub(crate) fn chunks<'a>(
245 &'a self,
246 range: Range<TabPoint>,
247 language_aware: bool,
248 highlights: Highlights<'a>,
249 ) -> TabChunks<'a> {
250 let (input_start, expanded_char_column, to_next_stop) =
251 self.tab_point_to_fold_point(range.start, Bias::Left);
252 let input_column = input_start.column();
253 let input_start = input_start.to_offset(&self.fold_snapshot);
254 let input_end = self
255 .tab_point_to_fold_point(range.end, Bias::Right)
256 .0
257 .to_offset(&self.fold_snapshot);
258 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
259 range.end.column() - range.start.column()
260 } else {
261 to_next_stop
262 };
263
264 TabChunks {
265 snapshot: self,
266 fold_chunks: self.fold_snapshot.chunks(
267 input_start..input_end,
268 language_aware,
269 highlights,
270 ),
271 input_column,
272 column: expanded_char_column,
273 max_expansion_column: self.max_expansion_column,
274 output_position: range.start.0,
275 max_output_position: range.end.0,
276 tab_size: self.tab_size,
277 chunk: Chunk {
278 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
279 is_tab: true,
280 ..Default::default()
281 },
282 inside_leading_tab: to_next_stop > 0,
283 }
284 }
285
286 #[ztracing::instrument(skip_all)]
287 pub fn rows(&self, row: u32) -> fold_map::FoldRows<'_> {
288 self.fold_snapshot.row_infos(row)
289 }
290
291 #[cfg(test)]
292 #[ztracing::instrument(skip_all)]
293 pub fn text(&self) -> String {
294 self.chunks(
295 TabPoint::zero()..self.max_point(),
296 false,
297 Highlights::default(),
298 )
299 .map(|chunk| chunk.text)
300 .collect()
301 }
302
303 #[ztracing::instrument(skip_all)]
304 pub fn max_point(&self) -> TabPoint {
305 self.fold_point_to_tab_point(self.fold_snapshot.max_point())
306 }
307
308 #[ztracing::instrument(skip_all)]
309 pub fn clip_point(&self, point: TabPoint, bias: Bias) -> TabPoint {
310 self.fold_point_to_tab_point(
311 self.fold_snapshot
312 .clip_point(self.tab_point_to_fold_point(point, bias).0, bias),
313 )
314 }
315
316 #[ztracing::instrument(skip_all)]
317 pub fn fold_point_to_tab_point(&self, input: FoldPoint) -> TabPoint {
318 let chunks = self.fold_snapshot.chunks_at(FoldPoint::new(input.row(), 0));
319 let tab_cursor = TabStopCursor::new(chunks);
320 let expanded = self.expand_tabs(tab_cursor, input.column());
321 TabPoint::new(input.row(), expanded)
322 }
323
324 #[ztracing::instrument(skip_all)]
325 pub fn tab_point_cursor(&self) -> TabPointCursor<'_> {
326 TabPointCursor { this: self }
327 }
328
329 #[ztracing::instrument(skip_all)]
330 pub fn tab_point_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
331 let chunks = self
332 .fold_snapshot
333 .chunks_at(FoldPoint::new(output.row(), 0));
334
335 let tab_cursor = TabStopCursor::new(chunks);
336 let expanded = output.column();
337 let (collapsed, expanded_char_column, to_next_stop) =
338 self.collapse_tabs(tab_cursor, expanded, bias);
339
340 (
341 FoldPoint::new(output.row(), collapsed),
342 expanded_char_column,
343 to_next_stop,
344 )
345 }
346
347 #[ztracing::instrument(skip_all)]
348 pub fn point_to_tab_point(&self, point: Point, bias: Bias) -> TabPoint {
349 let inlay_point = self.fold_snapshot.inlay_snapshot.to_inlay_point(point);
350 let fold_point = self.fold_snapshot.to_fold_point(inlay_point, bias);
351 self.fold_point_to_tab_point(fold_point)
352 }
353
354 #[ztracing::instrument(skip_all)]
355 pub fn tab_point_to_point(&self, point: TabPoint, bias: Bias) -> Point {
356 let fold_point = self.tab_point_to_fold_point(point, bias).0;
357 let inlay_point = fold_point.to_inlay_point(&self.fold_snapshot);
358 self.fold_snapshot
359 .inlay_snapshot
360 .to_buffer_point(inlay_point)
361 }
362
363 #[ztracing::instrument(skip_all)]
364 fn expand_tabs<'a, I>(&self, mut cursor: TabStopCursor<'a, I>, column: u32) -> u32
365 where
366 I: Iterator<Item = Chunk<'a>>,
367 {
368 let tab_size = self.tab_size.get();
369
370 let end_column = column.min(self.max_expansion_column);
371 let mut seek_target = end_column;
372 let mut tab_count = 0;
373 let mut expanded_tab_len = 0;
374
375 while let Some(tab_stop) = cursor.seek(seek_target) {
376 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
377 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
378 tab_count += 1;
379 expanded_tab_len += tab_len;
380
381 seek_target = end_column - cursor.byte_offset;
382 }
383
384 let left_over_char_bytes = if !cursor.is_char_boundary() {
385 cursor.bytes_until_next_char().unwrap_or(0) as u32
386 } else {
387 0
388 };
389
390 let collapsed_bytes = cursor.byte_offset() + left_over_char_bytes;
391 let expanded_bytes =
392 cursor.byte_offset() + expanded_tab_len - tab_count + left_over_char_bytes;
393
394 expanded_bytes + column.saturating_sub(collapsed_bytes)
395 }
396
397 #[ztracing::instrument(skip_all)]
398 fn collapse_tabs<'a, I>(
399 &self,
400 mut cursor: TabStopCursor<'a, I>,
401 column: u32,
402 bias: Bias,
403 ) -> (u32, u32, u32)
404 where
405 I: Iterator<Item = Chunk<'a>>,
406 {
407 let tab_size = self.tab_size.get();
408 let mut collapsed_column = column;
409 let mut seek_target = column.min(self.max_expansion_column);
410 let mut tab_count = 0;
411 let mut expanded_tab_len = 0;
412
413 while let Some(tab_stop) = cursor.seek(seek_target) {
414 // Calculate how much we want to expand this tab stop (into spaces)
415 let expanded_chars_old = tab_stop.char_offset + expanded_tab_len - tab_count;
416 let tab_len = tab_size - ((expanded_chars_old - 1) % tab_size);
417 // Increment tab count
418 tab_count += 1;
419 // The count of how many spaces we've added to this line in place of tab bytes
420 expanded_tab_len += tab_len;
421
422 // The count of bytes at this point in the iteration while considering tab_count and previous expansions
423 let expanded_bytes = tab_stop.byte_offset + expanded_tab_len - tab_count;
424
425 // Did we expand past the search target?
426 if expanded_bytes > column {
427 let mut expanded_chars = tab_stop.char_offset + expanded_tab_len - tab_count;
428 // We expanded past the search target, so need to account for the offshoot
429 expanded_chars -= expanded_bytes - column;
430 return match bias {
431 Bias::Left => (
432 cursor.byte_offset() - 1,
433 expanded_chars,
434 expanded_bytes - column,
435 ),
436 Bias::Right => (cursor.byte_offset(), expanded_chars, 0),
437 };
438 } else {
439 // otherwise we only want to move the cursor collapse column forward
440 collapsed_column = collapsed_column - tab_len + 1;
441 seek_target = (collapsed_column - cursor.byte_offset)
442 .min(self.max_expansion_column - cursor.byte_offset);
443 }
444 }
445
446 let collapsed_bytes = cursor.byte_offset();
447 let expanded_bytes = cursor.byte_offset() + expanded_tab_len - tab_count;
448 let expanded_chars = cursor.char_offset() + expanded_tab_len - tab_count;
449 (
450 collapsed_bytes + column.saturating_sub(expanded_bytes),
451 expanded_chars,
452 0,
453 )
454 }
455}
456
457// todo(lw): Implement TabPointCursor properly
458pub struct TabPointCursor<'this> {
459 this: &'this TabSnapshot,
460}
461
462impl TabPointCursor<'_> {
463 #[ztracing::instrument(skip_all)]
464 pub fn map(&mut self, point: FoldPoint) -> TabPoint {
465 self.this.fold_point_to_tab_point(point)
466 }
467}
468
469#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
470pub struct TabPoint(pub Point);
471
472impl TabPoint {
473 pub fn new(row: u32, column: u32) -> Self {
474 Self(Point::new(row, column))
475 }
476
477 pub fn zero() -> Self {
478 Self::new(0, 0)
479 }
480
481 pub fn row(self) -> u32 {
482 self.0.row
483 }
484
485 pub fn column(self) -> u32 {
486 self.0.column
487 }
488}
489
490impl From<Point> for TabPoint {
491 fn from(point: Point) -> Self {
492 Self(point)
493 }
494}
495
496pub type TabEdit = text::Edit<TabPoint>;
497
498#[derive(Clone, Debug, Default, Eq, PartialEq)]
499pub struct TextSummary {
500 pub lines: Point,
501 pub first_line_chars: u32,
502 pub last_line_chars: u32,
503 pub longest_row: u32,
504 pub longest_row_chars: u32,
505}
506
507impl<'a> From<&'a str> for TextSummary {
508 #[ztracing::instrument(skip_all)]
509 fn from(text: &'a str) -> Self {
510 let sum = text::TextSummary::from(text);
511
512 TextSummary {
513 lines: sum.lines,
514 first_line_chars: sum.first_line_chars,
515 last_line_chars: sum.last_line_chars,
516 longest_row: sum.longest_row,
517 longest_row_chars: sum.longest_row_chars,
518 }
519 }
520}
521
522impl<'a> std::ops::AddAssign<&'a Self> for TextSummary {
523 #[ztracing::instrument(skip_all)]
524 fn add_assign(&mut self, other: &'a Self) {
525 let joined_chars = self.last_line_chars + other.first_line_chars;
526 if joined_chars > self.longest_row_chars {
527 self.longest_row = self.lines.row;
528 self.longest_row_chars = joined_chars;
529 }
530 if other.longest_row_chars > self.longest_row_chars {
531 self.longest_row = self.lines.row + other.longest_row;
532 self.longest_row_chars = other.longest_row_chars;
533 }
534
535 if self.lines.row == 0 {
536 self.first_line_chars += other.first_line_chars;
537 }
538
539 if other.lines.row == 0 {
540 self.last_line_chars += other.first_line_chars;
541 } else {
542 self.last_line_chars = other.last_line_chars;
543 }
544
545 self.lines += &other.lines;
546 }
547}
548
549pub struct TabChunks<'a> {
550 snapshot: &'a TabSnapshot,
551 max_expansion_column: u32,
552 max_output_position: Point,
553 tab_size: NonZeroU32,
554 // region: iteration state
555 fold_chunks: FoldChunks<'a>,
556 chunk: Chunk<'a>,
557 column: u32,
558 output_position: Point,
559 input_column: u32,
560 inside_leading_tab: bool,
561 // endregion: iteration state
562}
563
564impl TabChunks<'_> {
565 #[ztracing::instrument(skip_all)]
566 pub(crate) fn seek(&mut self, range: Range<TabPoint>) {
567 let (input_start, expanded_char_column, to_next_stop) = self
568 .snapshot
569 .tab_point_to_fold_point(range.start, Bias::Left);
570 let input_column = input_start.column();
571 let input_start = input_start.to_offset(&self.snapshot.fold_snapshot);
572 let input_end = self
573 .snapshot
574 .tab_point_to_fold_point(range.end, Bias::Right)
575 .0
576 .to_offset(&self.snapshot.fold_snapshot);
577 let to_next_stop = if range.start.0 + Point::new(0, to_next_stop) > range.end.0 {
578 range.end.column() - range.start.column()
579 } else {
580 to_next_stop
581 };
582
583 self.fold_chunks.seek(input_start..input_end);
584 self.input_column = input_column;
585 self.column = expanded_char_column;
586 self.output_position = range.start.0;
587 self.max_output_position = range.end.0;
588 self.chunk = Chunk {
589 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..to_next_stop as usize]) },
590 is_tab: true,
591 chars: 1u128.unbounded_shl(to_next_stop) - 1,
592 ..Default::default()
593 };
594 self.inside_leading_tab = to_next_stop > 0;
595 }
596}
597
598impl<'a> Iterator for TabChunks<'a> {
599 type Item = Chunk<'a>;
600
601 #[ztracing::instrument(skip_all)]
602 fn next(&mut self) -> Option<Self::Item> {
603 if self.chunk.text.is_empty() {
604 if let Some(chunk) = self.fold_chunks.next() {
605 self.chunk = chunk;
606 if self.inside_leading_tab {
607 self.chunk.text = &self.chunk.text[1..];
608 self.inside_leading_tab = false;
609 self.input_column += 1;
610 }
611 } else {
612 return None;
613 }
614 }
615
616 //todo(improve performance by using tab cursor)
617 for (ix, c) in self.chunk.text.char_indices() {
618 match c {
619 '\t' if ix > 0 => {
620 let (prefix, suffix) = self.chunk.text.split_at(ix);
621
622 let mask = 1u128.unbounded_shl(ix as u32).wrapping_sub(1);
623 let chars = self.chunk.chars & mask;
624 let tabs = self.chunk.tabs & mask;
625 self.chunk.tabs = self.chunk.tabs.unbounded_shr(ix as u32);
626 self.chunk.chars = self.chunk.chars.unbounded_shr(ix as u32);
627 self.chunk.text = suffix;
628 return Some(Chunk {
629 text: prefix,
630 chars,
631 tabs,
632 ..self.chunk.clone()
633 });
634 }
635 '\t' => {
636 self.chunk.text = &self.chunk.text[1..];
637 self.chunk.tabs >>= 1;
638 self.chunk.chars >>= 1;
639 let tab_size = if self.input_column < self.max_expansion_column {
640 self.tab_size.get()
641 } else {
642 1
643 };
644 let mut len = tab_size - self.column % tab_size;
645 let next_output_position = cmp::min(
646 self.output_position + Point::new(0, len),
647 self.max_output_position,
648 );
649 len = next_output_position.column - self.output_position.column;
650 self.column += len;
651 self.input_column += 1;
652 self.output_position = next_output_position;
653 return Some(Chunk {
654 text: unsafe { std::str::from_utf8_unchecked(&SPACES[..len as usize]) },
655 is_tab: true,
656 chars: 1u128.unbounded_shl(len) - 1,
657 tabs: 0,
658 ..self.chunk.clone()
659 });
660 }
661 '\n' => {
662 self.column = 0;
663 self.input_column = 0;
664 self.output_position += Point::new(1, 0);
665 }
666 _ => {
667 self.column += 1;
668 if !self.inside_leading_tab {
669 self.input_column += c.len_utf8() as u32;
670 }
671 self.output_position.column += c.len_utf8() as u32;
672 }
673 }
674 }
675
676 Some(mem::take(&mut self.chunk))
677 }
678}
679
680#[cfg(test)]
681mod tests {
682 use super::*;
683 use crate::{
684 MultiBuffer,
685 display_map::{
686 fold_map::{FoldMap, FoldOffset},
687 inlay_map::InlayMap,
688 },
689 };
690 use multi_buffer::MultiBufferOffset;
691 use rand::{Rng, prelude::StdRng};
692 use util;
693
694 impl TabSnapshot {
695 fn expected_collapse_tabs(
696 &self,
697 chars: impl Iterator<Item = char>,
698 column: u32,
699 bias: Bias,
700 ) -> (u32, u32, u32) {
701 let tab_size = self.tab_size.get();
702
703 let mut expanded_bytes = 0;
704 let mut expanded_chars = 0;
705 let mut collapsed_bytes = 0;
706 for c in chars {
707 if expanded_bytes >= column {
708 break;
709 }
710 if collapsed_bytes >= self.max_expansion_column {
711 break;
712 }
713
714 if c == '\t' {
715 let tab_len = tab_size - (expanded_chars % tab_size);
716 expanded_chars += tab_len;
717 expanded_bytes += tab_len;
718 if expanded_bytes > column {
719 expanded_chars -= expanded_bytes - column;
720 return match bias {
721 Bias::Left => {
722 (collapsed_bytes, expanded_chars, expanded_bytes - column)
723 }
724 Bias::Right => (collapsed_bytes + 1, expanded_chars, 0),
725 };
726 }
727 } else {
728 expanded_chars += 1;
729 expanded_bytes += c.len_utf8() as u32;
730 }
731
732 if expanded_bytes > column && matches!(bias, Bias::Left) {
733 expanded_chars -= 1;
734 break;
735 }
736
737 collapsed_bytes += c.len_utf8() as u32;
738 }
739
740 (
741 collapsed_bytes + column.saturating_sub(expanded_bytes),
742 expanded_chars,
743 0,
744 )
745 }
746
747 pub fn expected_to_tab_point(&self, input: FoldPoint) -> TabPoint {
748 let chars = self.fold_snapshot.chars_at(FoldPoint::new(input.row(), 0));
749 let expanded = self.expected_expand_tabs(chars, input.column());
750 TabPoint::new(input.row(), expanded)
751 }
752
753 fn expected_expand_tabs(&self, chars: impl Iterator<Item = char>, column: u32) -> u32 {
754 let tab_size = self.tab_size.get();
755
756 let mut expanded_chars = 0;
757 let mut expanded_bytes = 0;
758 let mut collapsed_bytes = 0;
759 let end_column = column.min(self.max_expansion_column);
760 for c in chars {
761 if collapsed_bytes >= end_column {
762 break;
763 }
764 if c == '\t' {
765 let tab_len = tab_size - expanded_chars % tab_size;
766 expanded_bytes += tab_len;
767 expanded_chars += tab_len;
768 } else {
769 expanded_bytes += c.len_utf8() as u32;
770 expanded_chars += 1;
771 }
772 collapsed_bytes += c.len_utf8() as u32;
773 }
774
775 expanded_bytes + column.saturating_sub(collapsed_bytes)
776 }
777
778 fn expected_to_fold_point(&self, output: TabPoint, bias: Bias) -> (FoldPoint, u32, u32) {
779 let chars = self.fold_snapshot.chars_at(FoldPoint::new(output.row(), 0));
780 let expanded = output.column();
781 let (collapsed, expanded_char_column, to_next_stop) =
782 self.expected_collapse_tabs(chars, expanded, bias);
783 (
784 FoldPoint::new(output.row(), collapsed),
785 expanded_char_column,
786 to_next_stop,
787 )
788 }
789 }
790
791 #[gpui::test]
792 fn test_expand_tabs(cx: &mut gpui::App) {
793 let test_values = [
794 ("κg🏀 f\nwo🏀❌by🍐❎β🍗c\tβ❎ \ncλ🎉", 17),
795 (" \twςe", 4),
796 ("fε", 1),
797 ("i❎\t", 3),
798 ];
799 let buffer = MultiBuffer::build_simple("", cx);
800 let buffer_snapshot = buffer.read(cx).snapshot(cx);
801 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
802 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
803 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
804
805 for (text, column) in test_values {
806 let mut tabs = 0u128;
807 let mut chars = 0u128;
808 for (idx, c) in text.char_indices() {
809 if c == '\t' {
810 tabs |= 1 << idx;
811 }
812 chars |= 1 << idx;
813 }
814
815 let chunks = [Chunk {
816 text,
817 tabs,
818 chars,
819 ..Default::default()
820 }];
821
822 let cursor = TabStopCursor::new(chunks);
823
824 assert_eq!(
825 tab_snapshot.expected_expand_tabs(text.chars(), column),
826 tab_snapshot.expand_tabs(cursor, column)
827 );
828 }
829 }
830
831 #[gpui::test]
832 fn test_collapse_tabs(cx: &mut gpui::App) {
833 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
834
835 let buffer = MultiBuffer::build_simple(input, cx);
836 let buffer_snapshot = buffer.read(cx).snapshot(cx);
837 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
838 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
839 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
840
841 for (ix, _) in input.char_indices() {
842 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
843
844 assert_eq!(
845 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
846 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
847 "Failed with tab_point at column {ix}"
848 );
849 assert_eq!(
850 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
851 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
852 "Failed with tab_point at column {ix}"
853 );
854
855 assert_eq!(
856 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
857 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
858 "Failed with tab_point at column {ix}"
859 );
860 assert_eq!(
861 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
862 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
863 "Failed with tab_point at column {ix}"
864 );
865 }
866 }
867
868 #[gpui::test]
869 fn test_to_fold_point_panic_reproduction(cx: &mut gpui::App) {
870 // This test reproduces a specific panic where to_fold_point returns incorrect results
871 let _text = "use macro_rules_attribute::apply;\nuse serde_json::Value;\nuse smol::{\n io::AsyncReadExt,\n process::{Command, Stdio},\n};\nuse smol_macros::main;\nuse std::io;\n\nfn test_random() {\n // Generate a random value\n let random_value = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap()\n .as_secs()\n % 100;\n\n // Create some complex nested data structures\n let mut vector = Vec::new();\n for i in 0..random_value {\n vector.push(i);\n }\n ";
872
873 let text = "γ\tw⭐\n🍐🍗 \t";
874 let buffer = MultiBuffer::build_simple(text, cx);
875 let buffer_snapshot = buffer.read(cx).snapshot(cx);
876 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
877 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
878 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
879
880 // This should panic with the expected vs actual mismatch
881 let tab_point = TabPoint::new(0, 9);
882 let result = tab_snapshot.tab_point_to_fold_point(tab_point, Bias::Left);
883 let expected = tab_snapshot.expected_to_fold_point(tab_point, Bias::Left);
884
885 assert_eq!(result, expected);
886 }
887
888 #[gpui::test(iterations = 100)]
889 fn test_collapse_tabs_random(cx: &mut gpui::App, mut rng: StdRng) {
890 // Generate random input string with up to 200 characters including tabs
891 // to stay within the MAX_EXPANSION_COLUMN limit of 256
892 let len = rng.random_range(0..=2048);
893 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
894 let mut input = String::with_capacity(len);
895
896 for _ in 0..len {
897 if rng.random_bool(0.1) {
898 // 10% chance of inserting a tab
899 input.push('\t');
900 } else {
901 // 90% chance of inserting a random ASCII character (excluding tab, newline, carriage return)
902 let ch = loop {
903 let ascii_code = rng.random_range(32..=126); // printable ASCII range
904 let ch = ascii_code as u8 as char;
905 if ch != '\t' {
906 break ch;
907 }
908 };
909 input.push(ch);
910 }
911 }
912
913 let buffer = MultiBuffer::build_simple(&input, cx);
914 let buffer_snapshot = buffer.read(cx).snapshot(cx);
915 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
916 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
917 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
918 tab_snapshot.max_expansion_column = rng.random_range(0..323);
919 tab_snapshot.tab_size = tab_size;
920
921 for (ix, _) in input.char_indices() {
922 let range = TabPoint::new(0, ix as u32)..tab_snapshot.max_point();
923
924 assert_eq!(
925 tab_snapshot.expected_to_fold_point(range.start, Bias::Left),
926 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Left),
927 "Failed with input: {}, with idx: {ix}",
928 input
929 );
930 assert_eq!(
931 tab_snapshot.expected_to_fold_point(range.start, Bias::Right),
932 tab_snapshot.tab_point_to_fold_point(range.start, Bias::Right),
933 "Failed with input: {}, with idx: {ix}",
934 input
935 );
936
937 assert_eq!(
938 tab_snapshot.expected_to_fold_point(range.end, Bias::Left),
939 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Left),
940 "Failed with input: {}, with idx: {ix}",
941 input
942 );
943 assert_eq!(
944 tab_snapshot.expected_to_fold_point(range.end, Bias::Right),
945 tab_snapshot.tab_point_to_fold_point(range.end, Bias::Right),
946 "Failed with input: {}, with idx: {ix}",
947 input
948 );
949 }
950 }
951
952 #[gpui::test]
953 fn test_long_lines(cx: &mut gpui::App) {
954 let max_expansion_column = 12;
955 let input = "A\tBC\tDEF\tG\tHI\tJ\tK\tL\tM";
956 let output = "A BC DEF G HI J K L M";
957
958 let buffer = MultiBuffer::build_simple(input, cx);
959 let buffer_snapshot = buffer.read(cx).snapshot(cx);
960 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
961 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
962 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
963
964 tab_snapshot.max_expansion_column = max_expansion_column;
965 assert_eq!(tab_snapshot.text(), output);
966
967 for (ix, c) in input.char_indices() {
968 assert_eq!(
969 tab_snapshot
970 .chunks(
971 TabPoint::new(0, ix as u32)..tab_snapshot.max_point(),
972 false,
973 Highlights::default(),
974 )
975 .map(|c| c.text)
976 .collect::<String>(),
977 &output[ix..],
978 "text from index {ix}"
979 );
980
981 if c != '\t' {
982 let input_point = Point::new(0, ix as u32);
983 let output_point = Point::new(0, output.find(c).unwrap() as u32);
984 assert_eq!(
985 tab_snapshot.fold_point_to_tab_point(FoldPoint(input_point)),
986 TabPoint(output_point),
987 "to_tab_point({input_point:?})"
988 );
989 assert_eq!(
990 tab_snapshot
991 .tab_point_to_fold_point(TabPoint(output_point), Bias::Left)
992 .0,
993 FoldPoint(input_point),
994 "to_fold_point({output_point:?})"
995 );
996 }
997 }
998 }
999
1000 #[gpui::test]
1001 fn test_long_lines_with_character_spanning_max_expansion_column(cx: &mut gpui::App) {
1002 let max_expansion_column = 8;
1003 let input = "abcdefg⋯hij";
1004
1005 let buffer = MultiBuffer::build_simple(input, cx);
1006 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1007 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1008 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1009 let (_, mut tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1010
1011 tab_snapshot.max_expansion_column = max_expansion_column;
1012 assert_eq!(tab_snapshot.text(), input);
1013 }
1014
1015 #[gpui::test]
1016 fn test_marking_tabs(cx: &mut gpui::App) {
1017 let input = "\t \thello";
1018
1019 let buffer = MultiBuffer::build_simple(input, cx);
1020 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1021 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1022 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1023 let (_, tab_snapshot) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
1024
1025 assert_eq!(
1026 chunks(&tab_snapshot, TabPoint::zero()),
1027 vec![
1028 (" ".to_string(), true),
1029 (" ".to_string(), false),
1030 (" ".to_string(), true),
1031 ("hello".to_string(), false),
1032 ]
1033 );
1034 assert_eq!(
1035 chunks(&tab_snapshot, TabPoint::new(0, 2)),
1036 vec![
1037 (" ".to_string(), true),
1038 (" ".to_string(), false),
1039 (" ".to_string(), true),
1040 ("hello".to_string(), false),
1041 ]
1042 );
1043
1044 fn chunks(snapshot: &TabSnapshot, start: TabPoint) -> Vec<(String, bool)> {
1045 let mut chunks = Vec::new();
1046 let mut was_tab = false;
1047 let mut text = String::new();
1048 for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default())
1049 {
1050 if chunk.is_tab != was_tab {
1051 if !text.is_empty() {
1052 chunks.push((mem::take(&mut text), was_tab));
1053 }
1054 was_tab = chunk.is_tab;
1055 }
1056 text.push_str(chunk.text);
1057 }
1058
1059 if !text.is_empty() {
1060 chunks.push((text, was_tab));
1061 }
1062 chunks
1063 }
1064 }
1065
1066 #[gpui::test(iterations = 100)]
1067 fn test_random_tabs(cx: &mut gpui::App, mut rng: StdRng) {
1068 let tab_size = NonZeroU32::new(rng.random_range(1..=4)).unwrap();
1069 let len = rng.random_range(0..30);
1070 let buffer = if rng.random() {
1071 let text = util::RandomCharIter::new(&mut rng)
1072 .take(len)
1073 .collect::<String>();
1074 MultiBuffer::build_simple(&text, cx)
1075 } else {
1076 MultiBuffer::build_random(&mut rng, cx)
1077 };
1078 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1079 log::info!("Buffer text: {:?}", buffer_snapshot.text());
1080
1081 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1082 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1083 let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
1084 fold_map.randomly_mutate(&mut rng);
1085 let (fold_snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
1086 log::info!("FoldMap text: {:?}", fold_snapshot.text());
1087 let (inlay_snapshot, _) = inlay_map.randomly_mutate(&mut 0, &mut rng);
1088 log::info!("InlayMap text: {:?}", inlay_snapshot.text());
1089
1090 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1091 let tabs_snapshot = tab_map.set_max_expansion_column(32);
1092
1093 let text = text::Rope::from(tabs_snapshot.text().as_str());
1094 log::info!(
1095 "TabMap text (tab size: {}): {:?}",
1096 tab_size,
1097 tabs_snapshot.text(),
1098 );
1099
1100 for _ in 0..5 {
1101 let end_row = rng.random_range(0..=text.max_point().row);
1102 let end_column = rng.random_range(0..=text.line_len(end_row));
1103 let mut end = TabPoint(text.clip_point(Point::new(end_row, end_column), Bias::Right));
1104 let start_row = rng.random_range(0..=text.max_point().row);
1105 let start_column = rng.random_range(0..=text.line_len(start_row));
1106 let mut start =
1107 TabPoint(text.clip_point(Point::new(start_row, start_column), Bias::Left));
1108 if start > end {
1109 mem::swap(&mut start, &mut end);
1110 }
1111
1112 let expected_text = text
1113 .chunks_in_range(text.point_to_offset(start.0)..text.point_to_offset(end.0))
1114 .collect::<String>();
1115 let expected_summary = TextSummary::from(expected_text.as_str());
1116 assert_eq!(
1117 tabs_snapshot
1118 .chunks(start..end, false, Highlights::default())
1119 .map(|c| c.text)
1120 .collect::<String>(),
1121 expected_text,
1122 "chunks({:?}..{:?})",
1123 start,
1124 end
1125 );
1126
1127 let mut actual_summary = tabs_snapshot.text_summary_for_range(start..end);
1128 if tab_size.get() > 1 && inlay_snapshot.text().contains('\t') {
1129 actual_summary.longest_row = expected_summary.longest_row;
1130 actual_summary.longest_row_chars = expected_summary.longest_row_chars;
1131 }
1132 assert_eq!(actual_summary, expected_summary);
1133 }
1134
1135 for row in 0..=text.max_point().row {
1136 assert_eq!(
1137 tabs_snapshot.line_len(row),
1138 text.line_len(row),
1139 "line_len({row})"
1140 );
1141 }
1142 }
1143
1144 #[gpui::test(iterations = 100)]
1145 fn test_to_tab_point_random(cx: &mut gpui::App, mut rng: StdRng) {
1146 let tab_size = NonZeroU32::new(rng.random_range(1..=16)).unwrap();
1147 let len = rng.random_range(0..=2000);
1148
1149 // Generate random text using RandomCharIter
1150 let text = util::RandomCharIter::new(&mut rng)
1151 .take(len)
1152 .collect::<String>();
1153
1154 // Create buffer and tab map
1155 let buffer = MultiBuffer::build_simple(&text, cx);
1156 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1157 let (mut inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1158 let (mut fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
1159 let (mut tab_map, _) = TabMap::new(fold_snapshot, tab_size);
1160
1161 let mut next_inlay_id = 0;
1162 let (inlay_snapshot, inlay_edits) = inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
1163 let (fold_snapshot, fold_edits) = fold_map.read(inlay_snapshot, inlay_edits);
1164 let max_fold_point = fold_snapshot.max_point();
1165 let (mut tab_snapshot, _) = tab_map.sync(fold_snapshot.clone(), fold_edits, tab_size);
1166
1167 // Test random fold points
1168 for _ in 0..50 {
1169 tab_snapshot.max_expansion_column = rng.random_range(0..=256);
1170 // Generate random fold point
1171 let row = rng.random_range(0..=max_fold_point.row());
1172 let max_column = if row < max_fold_point.row() {
1173 fold_snapshot.line_len(row)
1174 } else {
1175 max_fold_point.column()
1176 };
1177 let column = rng.random_range(0..=max_column + 10);
1178 let fold_point = FoldPoint::new(row, column);
1179
1180 let actual = tab_snapshot.fold_point_to_tab_point(fold_point);
1181 let expected = tab_snapshot.expected_to_tab_point(fold_point);
1182
1183 assert_eq!(
1184 actual, expected,
1185 "to_tab_point mismatch for fold_point {:?} in text {:?}",
1186 fold_point, text
1187 );
1188 }
1189 }
1190
1191 #[gpui::test]
1192 fn test_tab_stop_cursor_utf8(cx: &mut gpui::App) {
1193 let text = "\tfoo\tbarbarbar\t\tbaz\n";
1194 let buffer = MultiBuffer::build_simple(text, cx);
1195 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1196 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1197 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1198 let chunks = fold_snapshot.chunks(
1199 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1200 false,
1201 Default::default(),
1202 );
1203 let mut cursor = TabStopCursor::new(chunks);
1204 assert!(cursor.seek(0).is_none());
1205 let mut tab_stops = Vec::new();
1206
1207 let mut all_tab_stops = Vec::new();
1208 let mut byte_offset = 0;
1209 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1210 byte_offset += ch.len_utf8() as u32;
1211
1212 if ch == '\t' {
1213 all_tab_stops.push(TabStop {
1214 byte_offset,
1215 char_offset: offset as u32 + 1,
1216 });
1217 }
1218 }
1219
1220 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1221 tab_stops.push(tab_stop);
1222 }
1223 pretty_assertions::assert_eq!(tab_stops.as_slice(), all_tab_stops.as_slice(),);
1224
1225 assert_eq!(cursor.byte_offset(), byte_offset);
1226 }
1227
1228 #[gpui::test]
1229 fn test_tab_stop_with_end_range_utf8(cx: &mut gpui::App) {
1230 let input = "A\tBC\t"; // DEF\tG\tHI\tJ\tK\tL\tM
1231
1232 let buffer = MultiBuffer::build_simple(input, cx);
1233 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1234 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1235 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1236
1237 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1238 let mut cursor = TabStopCursor::new(chunks);
1239
1240 let mut actual_tab_stops = Vec::new();
1241
1242 let mut expected_tab_stops = Vec::new();
1243 let mut byte_offset = 0;
1244 for (offset, ch) in buffer.read(cx).snapshot(cx).text().char_indices() {
1245 byte_offset += ch.len_utf8() as u32;
1246
1247 if ch == '\t' {
1248 expected_tab_stops.push(TabStop {
1249 byte_offset,
1250 char_offset: offset as u32 + 1,
1251 });
1252 }
1253 }
1254
1255 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1256 actual_tab_stops.push(tab_stop);
1257 }
1258 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1259
1260 assert_eq!(cursor.byte_offset(), byte_offset);
1261 }
1262
1263 #[gpui::test(iterations = 100)]
1264 fn test_tab_stop_cursor_random_utf8(cx: &mut gpui::App, mut rng: StdRng) {
1265 // Generate random input string with up to 512 characters including tabs
1266 let len = rng.random_range(0..=2048);
1267 let mut input = String::with_capacity(len);
1268
1269 let mut skip_tabs = rng.random_bool(0.10);
1270 for idx in 0..len {
1271 if idx % 128 == 0 {
1272 skip_tabs = rng.random_bool(0.10);
1273 }
1274
1275 if rng.random_bool(0.15) && !skip_tabs {
1276 input.push('\t');
1277 } else {
1278 let ch = loop {
1279 let ascii_code = rng.random_range(32..=126); // printable ASCII range
1280 let ch = ascii_code as u8 as char;
1281 if ch != '\t' {
1282 break ch;
1283 }
1284 };
1285 input.push(ch);
1286 }
1287 }
1288
1289 // Build the buffer and create cursor
1290 let buffer = MultiBuffer::build_simple(&input, cx);
1291 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1292 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1293 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1294
1295 // First, collect all expected tab positions
1296 let mut all_tab_stops = Vec::new();
1297 let mut byte_offset = 1;
1298 let mut char_offset = 1;
1299 for ch in buffer_snapshot.text().chars() {
1300 if ch == '\t' {
1301 all_tab_stops.push(TabStop {
1302 byte_offset,
1303 char_offset,
1304 });
1305 }
1306 byte_offset += ch.len_utf8() as u32;
1307 char_offset += 1;
1308 }
1309
1310 // Test with various distances
1311 let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1312 // let distances = vec![150];
1313
1314 for distance in distances {
1315 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1316 let mut cursor = TabStopCursor::new(chunks);
1317
1318 let mut found_tab_stops = Vec::new();
1319 let mut position = distance;
1320 while let Some(tab_stop) = cursor.seek(position) {
1321 found_tab_stops.push(tab_stop);
1322 position = distance - tab_stop.byte_offset;
1323 }
1324
1325 let expected_found_tab_stops: Vec<_> = all_tab_stops
1326 .iter()
1327 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1328 .cloned()
1329 .collect();
1330
1331 pretty_assertions::assert_eq!(
1332 found_tab_stops,
1333 expected_found_tab_stops,
1334 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1335 distance,
1336 input
1337 );
1338
1339 let final_position = cursor.byte_offset();
1340 if !found_tab_stops.is_empty() {
1341 let last_tab_stop = found_tab_stops.last().unwrap();
1342 assert!(
1343 final_position >= last_tab_stop.byte_offset,
1344 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1345 final_position,
1346 last_tab_stop.byte_offset,
1347 input
1348 );
1349 }
1350 }
1351 }
1352
1353 #[gpui::test]
1354 fn test_tab_stop_cursor_utf16(cx: &mut gpui::App) {
1355 let text = "\r\t😁foo\tb😀arbar🤯bar\t\tbaz\n";
1356 let buffer = MultiBuffer::build_simple(text, cx);
1357 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1358 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot);
1359 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1360 let chunks = fold_snapshot.chunks(
1361 FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(),
1362 false,
1363 Default::default(),
1364 );
1365 let mut cursor = TabStopCursor::new(chunks);
1366 assert!(cursor.seek(0).is_none());
1367
1368 let mut expected_tab_stops = Vec::new();
1369 let mut byte_offset = 0;
1370 for (i, ch) in fold_snapshot.chars_at(FoldPoint::new(0, 0)).enumerate() {
1371 byte_offset += ch.len_utf8() as u32;
1372
1373 if ch == '\t' {
1374 expected_tab_stops.push(TabStop {
1375 byte_offset,
1376 char_offset: i as u32 + 1,
1377 });
1378 }
1379 }
1380
1381 let mut actual_tab_stops = Vec::new();
1382 while let Some(tab_stop) = cursor.seek(u32::MAX) {
1383 actual_tab_stops.push(tab_stop);
1384 }
1385
1386 pretty_assertions::assert_eq!(actual_tab_stops.as_slice(), expected_tab_stops.as_slice(),);
1387
1388 assert_eq!(cursor.byte_offset(), byte_offset);
1389 }
1390
1391 #[gpui::test(iterations = 100)]
1392 fn test_tab_stop_cursor_random_utf16(cx: &mut gpui::App, mut rng: StdRng) {
1393 // Generate random input string with up to 512 characters including tabs
1394 let len = rng.random_range(0..=2048);
1395 let input = util::RandomCharIter::new(&mut rng)
1396 .take(len)
1397 .collect::<String>();
1398
1399 // Build the buffer and create cursor
1400 let buffer = MultiBuffer::build_simple(&input, cx);
1401 let buffer_snapshot = buffer.read(cx).snapshot(cx);
1402 let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
1403 let (_, fold_snapshot) = FoldMap::new(inlay_snapshot);
1404
1405 // First, collect all expected tab positions
1406 let mut all_tab_stops = Vec::new();
1407 let mut byte_offset = 0;
1408 for (i, ch) in buffer_snapshot.text().chars().enumerate() {
1409 byte_offset += ch.len_utf8() as u32;
1410 if ch == '\t' {
1411 all_tab_stops.push(TabStop {
1412 byte_offset,
1413 char_offset: i as u32 + 1,
1414 });
1415 }
1416 }
1417
1418 // Test with various distances
1419 // let distances = vec![1, 5, 10, 50, 100, u32::MAX];
1420 let distances = vec![150];
1421
1422 for distance in distances {
1423 let chunks = fold_snapshot.chunks_at(FoldPoint::new(0, 0));
1424 let mut cursor = TabStopCursor::new(chunks);
1425
1426 let mut found_tab_stops = Vec::new();
1427 let mut position = distance;
1428 while let Some(tab_stop) = cursor.seek(position) {
1429 found_tab_stops.push(tab_stop);
1430 position = distance - tab_stop.byte_offset;
1431 }
1432
1433 let expected_found_tab_stops: Vec<_> = all_tab_stops
1434 .iter()
1435 .take_while(|tab_stop| tab_stop.byte_offset <= distance)
1436 .cloned()
1437 .collect();
1438
1439 pretty_assertions::assert_eq!(
1440 found_tab_stops,
1441 expected_found_tab_stops,
1442 "TabStopCursor output mismatch for distance {}. Input: {:?}",
1443 distance,
1444 input
1445 );
1446
1447 let final_position = cursor.byte_offset();
1448 if !found_tab_stops.is_empty() {
1449 let last_tab_stop = found_tab_stops.last().unwrap();
1450 assert!(
1451 final_position >= last_tab_stop.byte_offset,
1452 "Cursor final position {} is before last tab stop {}. Input: {:?}",
1453 final_position,
1454 last_tab_stop.byte_offset,
1455 input
1456 );
1457 }
1458 }
1459 }
1460}
1461
1462struct TabStopCursor<'a, I>
1463where
1464 I: Iterator<Item = Chunk<'a>>,
1465{
1466 chunks: I,
1467 byte_offset: u32,
1468 char_offset: u32,
1469 /// Chunk
1470 /// last tab position iterated through
1471 current_chunk: Option<(Chunk<'a>, u32)>,
1472}
1473
1474impl<'a, I> TabStopCursor<'a, I>
1475where
1476 I: Iterator<Item = Chunk<'a>>,
1477{
1478 #[ztracing::instrument(skip_all)]
1479 fn new(chunks: impl IntoIterator<Item = Chunk<'a>, IntoIter = I>) -> Self {
1480 Self {
1481 chunks: chunks.into_iter(),
1482 byte_offset: 0,
1483 char_offset: 0,
1484 current_chunk: None,
1485 }
1486 }
1487
1488 #[ztracing::instrument(skip_all)]
1489 fn bytes_until_next_char(&self) -> Option<usize> {
1490 self.current_chunk.as_ref().and_then(|(chunk, idx)| {
1491 let mut idx = *idx;
1492 let mut diff = 0;
1493 while idx > 0 && chunk.chars & (1u128.unbounded_shl(idx)) == 0 {
1494 idx -= 1;
1495 diff += 1;
1496 }
1497
1498 if chunk.chars & (1 << idx) != 0 {
1499 Some(
1500 (chunk.text[idx as usize..].chars().next()?)
1501 .len_utf8()
1502 .saturating_sub(diff),
1503 )
1504 } else {
1505 None
1506 }
1507 })
1508 }
1509
1510 #[ztracing::instrument(skip_all)]
1511 fn is_char_boundary(&self) -> bool {
1512 self.current_chunk
1513 .as_ref()
1514 .is_some_and(|(chunk, idx)| (chunk.chars & 1u128.unbounded_shl(*idx)) != 0)
1515 }
1516
1517 /// distance: length to move forward while searching for the next tab stop
1518 #[ztracing::instrument(skip_all)]
1519 fn seek(&mut self, distance: u32) -> Option<TabStop> {
1520 if distance == 0 {
1521 return None;
1522 }
1523
1524 let mut distance_traversed = 0;
1525
1526 while let Some((mut chunk, chunk_position)) = self
1527 .current_chunk
1528 .take()
1529 .or_else(|| self.chunks.next().zip(Some(0)))
1530 {
1531 if chunk.tabs == 0 {
1532 let chunk_distance = chunk.text.len() as u32 - chunk_position;
1533 if chunk_distance + distance_traversed >= distance {
1534 let overshoot = distance_traversed.abs_diff(distance);
1535
1536 self.byte_offset += overshoot;
1537 self.char_offset += get_char_offset(
1538 chunk_position..(chunk_position + overshoot).saturating_sub(1),
1539 chunk.chars,
1540 );
1541
1542 if chunk_position + overshoot < 128 {
1543 self.current_chunk = Some((chunk, chunk_position + overshoot));
1544 }
1545
1546 return None;
1547 }
1548
1549 self.byte_offset += chunk_distance;
1550 self.char_offset += get_char_offset(
1551 chunk_position..(chunk_position + chunk_distance).saturating_sub(1),
1552 chunk.chars,
1553 );
1554 distance_traversed += chunk_distance;
1555 continue;
1556 }
1557 let tab_position = chunk.tabs.trailing_zeros() + 1;
1558
1559 if distance_traversed + tab_position - chunk_position > distance {
1560 let cursor_position = distance_traversed.abs_diff(distance);
1561
1562 self.char_offset += get_char_offset(
1563 chunk_position..(chunk_position + cursor_position - 1),
1564 chunk.chars,
1565 );
1566 self.current_chunk = Some((chunk, cursor_position + chunk_position));
1567 self.byte_offset += cursor_position;
1568
1569 return None;
1570 }
1571
1572 self.byte_offset += tab_position - chunk_position;
1573 self.char_offset += get_char_offset(chunk_position..(tab_position - 1), chunk.chars);
1574
1575 let tabstop = TabStop {
1576 char_offset: self.char_offset,
1577 byte_offset: self.byte_offset,
1578 };
1579
1580 chunk.tabs = (chunk.tabs - 1) & chunk.tabs;
1581
1582 if tab_position as usize != chunk.text.len() {
1583 self.current_chunk = Some((chunk, tab_position));
1584 }
1585
1586 return Some(tabstop);
1587 }
1588
1589 None
1590 }
1591
1592 fn byte_offset(&self) -> u32 {
1593 self.byte_offset
1594 }
1595
1596 fn char_offset(&self) -> u32 {
1597 self.char_offset
1598 }
1599}
1600
1601#[inline(always)]
1602fn get_char_offset(range: Range<u32>, bit_map: u128) -> u32 {
1603 if range.start == range.end {
1604 return if (1u128 << range.start) & bit_map == 0 {
1605 0
1606 } else {
1607 1
1608 };
1609 }
1610 let end_shift: u128 = 127u128 - range.end as u128;
1611 let mut bit_mask = (u128::MAX >> range.start) << range.start;
1612 bit_mask = (bit_mask << end_shift) >> end_shift;
1613 let bit_map = bit_map & bit_mask;
1614
1615 bit_map.count_ones()
1616}
1617
1618#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1619struct TabStop {
1620 char_offset: u32,
1621 byte_offset: u32,
1622}