From 3ed1c32bf9a1ebb485e3da6cabc8b3c0a423beea Mon Sep 17 00:00:00 2001 From: Xin Zhao Date: Tue, 7 Apr 2026 22:15:33 +0800 Subject: [PATCH] editor: Fix diagnostic rendering when semantic tokens set to full (#53008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content is consistent with the [UI/UX checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) - [ ] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable Closes #50212 There are two unreasonable coupling account for this issue, the coupling of `use_tree_sitter` with `languge_aware` in https://github.com/zed-industries/zed/blob/7892b932795911516f26f3c1c1c72249ed181ba8/crates/editor/src/element.rs#L3820-L3822 and the coupling of `language_aware` with `diagnostics` in https://github.com/zed-industries/zed/blob/7892b932795911516f26f3c1c1c72249ed181ba8/crates/language/src/buffer.rs#L3736-L3746 Because of these couplings, when the editor stops using Tree-sitter highlighting when `"semantic_tokens"` set to `"full"`, it also accidentally stops fetching diagnostic information. This is why error and warning underlines disappear. I’ve fixed this by adding a separate `use_tree_sitter` parameter to `highlighted_chunks`. This way, we can keep `language_aware` true to get the diagnostic data we need, but still decide whether or not to apply Tree-sitter highlights. I chose to fix this at the `highlighted_chunks` level because I’m worried that changing the logic in the deeper layers of the DisplayMap or Buffer might have too many side effects that are hard to predict. This approach feels like a safer way to solve the problem. Release Notes: - Fixed a bug where diagnostic underlines disappeared when "semantic_tokens" set to "full" --------- Co-authored-by: Kirill Bulatov --- crates/editor/src/display_map.rs | 48 +++++-- crates/editor/src/display_map/block_map.rs | 14 +- .../src/display_map/custom_highlights.rs | 9 +- crates/editor/src/display_map/fold_map.rs | 33 ++++- crates/editor/src/display_map/inlay_map.rs | 37 +++-- crates/editor/src/display_map/tab_map.rs | 71 ++++++++-- crates/editor/src/display_map/wrap_map.rs | 19 ++- crates/editor/src/editor.rs | 22 ++- crates/editor/src/element.rs | 17 ++- crates/editor/src/semantic_tokens.rs | 132 +++++++++++++++++- crates/language/src/buffer.rs | 43 +++++- crates/language/src/buffer_tests.rs | 8 +- crates/multi_buffer/src/multi_buffer.rs | 33 +++-- crates/multi_buffer/src/multi_buffer_tests.rs | 24 +++- crates/outline_panel/src/outline_panel.rs | 13 +- crates/project/src/lsp_store.rs | 15 +- .../tests/integration/project_tests.rs | 15 +- crates/vim/src/state.rs | 12 +- 18 files changed, 468 insertions(+), 97 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index f95f1030276015af4825119fc98ac68b876d0e5f..7cb8040e282a47d27cf5d7b33e5453295b4f645f 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -98,7 +98,7 @@ use gpui::{ WeakEntity, }; use language::{ - Point, Subscription as BufferSubscription, + LanguageAwareStyling, Point, Subscription as BufferSubscription, language_settings::{AllLanguageSettings, LanguageSettings}, }; @@ -1769,7 +1769,10 @@ impl DisplaySnapshot { self.block_snapshot .chunks( BlockRow(display_row.0)..BlockRow(self.max_point().row().next_row().0), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, self.masked, Highlights::default(), ) @@ -1783,7 +1786,10 @@ impl DisplaySnapshot { self.block_snapshot .chunks( BlockRow(row)..BlockRow(row + 1), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, self.masked, Highlights::default(), ) @@ -1798,7 +1804,7 @@ impl DisplaySnapshot { pub fn chunks( &self, display_rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlight_styles: HighlightStyles, ) -> DisplayChunks<'_> { self.block_snapshot.chunks( @@ -1818,7 +1824,7 @@ impl DisplaySnapshot { pub fn highlighted_chunks<'a>( &'a self, display_rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, editor_style: &'a EditorStyle, ) -> impl Iterator> { self.chunks( @@ -1910,7 +1916,10 @@ impl DisplaySnapshot { let chunks = custom_highlights::CustomHighlightsChunks::new( multibuffer_range, - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, None, Some(&self.semantic_token_highlights), multibuffer, @@ -1961,7 +1970,14 @@ impl DisplaySnapshot { let mut line = String::new(); let range = display_row..display_row.next_row(); - for chunk in self.highlighted_chunks(range, false, editor_style) { + for chunk in self.highlighted_chunks( + range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + editor_style, + ) { line.push_str(chunk.text); let text_style = if let Some(style) = chunk.style { @@ -3388,7 +3404,14 @@ pub mod tests { let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); let mut chunks = Vec::<(String, Option, Rgba)>::new(); - for chunk in snapshot.chunks(DisplayRow(0)..DisplayRow(5), true, Default::default()) { + for chunk in snapshot.chunks( + DisplayRow(0)..DisplayRow(5), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + Default::default(), + ) { let color = chunk .highlight_style .and_then(|style| style.color) @@ -3940,7 +3963,14 @@ pub mod tests { ) -> Vec<(String, Option, Option)> { let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); let mut chunks: Vec<(String, Option, Option)> = Vec::new(); - for chunk in snapshot.chunks(rows, true, HighlightStyles::default()) { + for chunk in snapshot.chunks( + rows, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + HighlightStyles::default(), + ) { let syntax_color = chunk .syntax_highlight_id .and_then(|id| theme.get(id)?.color); diff --git a/crates/editor/src/display_map/block_map.rs b/crates/editor/src/display_map/block_map.rs index 67318e3300e73085fe40c2e22edfcd06778902c8..17fa7e3de4a361f6728664e76368583788053cfd 100644 --- a/crates/editor/src/display_map/block_map.rs +++ b/crates/editor/src/display_map/block_map.rs @@ -9,7 +9,7 @@ use crate::{ }; use collections::{Bound, HashMap, HashSet}; use gpui::{AnyElement, App, EntityId, Pixels, Window}; -use language::{Patch, Point}; +use language::{LanguageAwareStyling, Patch, Point}; use multi_buffer::{ Anchor, ExcerptBoundaryInfo, MultiBuffer, MultiBufferOffset, MultiBufferPoint, MultiBufferRow, MultiBufferSnapshot, RowInfo, ToOffset, ToPoint as _, @@ -2140,7 +2140,10 @@ impl BlockSnapshot { pub fn text(&self) -> String { self.chunks( BlockRow(0)..self.transforms.summary().output_rows, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, false, Highlights::default(), ) @@ -2152,7 +2155,7 @@ impl BlockSnapshot { pub(crate) fn chunks<'a>( &'a self, rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, masked: bool, highlights: Highlights<'a>, ) -> BlockChunks<'a> { @@ -4300,7 +4303,10 @@ mod tests { let actual_text = blocks_snapshot .chunks( BlockRow(start_row as u32)..BlockRow(end_row as u32), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, false, Highlights::default(), ) diff --git a/crates/editor/src/display_map/custom_highlights.rs b/crates/editor/src/display_map/custom_highlights.rs index 39eabef2f9627b8088dc826ec64379bf76a6c9fa..6e93e562172decb0843da35c7f55fafd92ed21cc 100644 --- a/crates/editor/src/display_map/custom_highlights.rs +++ b/crates/editor/src/display_map/custom_highlights.rs @@ -1,6 +1,6 @@ use collections::BTreeMap; use gpui::HighlightStyle; -use language::Chunk; +use language::{Chunk, LanguageAwareStyling}; use multi_buffer::{MultiBufferChunks, MultiBufferOffset, MultiBufferSnapshot, ToOffset as _}; use std::{ cmp, @@ -34,7 +34,7 @@ impl<'a> CustomHighlightsChunks<'a> { #[ztracing::instrument(skip_all)] pub fn new( range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, text_highlights: Option<&'a TextHighlights>, semantic_token_highlights: Option<&'a SemanticTokensHighlights>, multibuffer_snapshot: &'a MultiBufferSnapshot, @@ -308,7 +308,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = CustomHighlightsChunks::new( MultiBufferOffset(0)..buffer_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, None, None, &buffer_snapshot, diff --git a/crates/editor/src/display_map/fold_map.rs b/crates/editor/src/display_map/fold_map.rs index 1554bb96dab0e2f76a17df1396bd945f332af208..4c6c04b86cc3e2fb9ef10be58c14faae623dc65f 100644 --- a/crates/editor/src/display_map/fold_map.rs +++ b/crates/editor/src/display_map/fold_map.rs @@ -5,7 +5,7 @@ use super::{ inlay_map::{InlayBufferRows, InlayChunks, InlayEdit, InlayOffset, InlayPoint, InlaySnapshot}, }; use gpui::{AnyElement, App, ElementId, HighlightStyle, Pixels, SharedString, Stateful, Window}; -use language::{Edit, HighlightId, Point}; +use language::{Edit, HighlightId, LanguageAwareStyling, Point}; use multi_buffer::{ Anchor, AnchorRangeExt, MBTextSummary, MultiBufferOffset, MultiBufferRow, MultiBufferSnapshot, RowInfo, ToOffset, @@ -707,7 +707,10 @@ impl FoldSnapshot { pub fn text(&self) -> String { self.chunks( FoldOffset(MultiBufferOffset(0))..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|c| c.text) @@ -909,7 +912,7 @@ impl FoldSnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> FoldChunks<'a> { let mut transform_cursor = self @@ -954,7 +957,10 @@ impl FoldSnapshot { pub fn chars_at(&self, start: FoldPoint) -> impl '_ + Iterator { self.chunks( start.to_offset(self)..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .flat_map(|chunk| chunk.text.chars()) @@ -964,7 +970,10 @@ impl FoldSnapshot { pub fn chunks_at(&self, start: FoldPoint) -> FoldChunks<'_> { self.chunks( start.to_offset(self)..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) } @@ -2131,7 +2140,14 @@ mod tests { let text = &expected_text[start.0.0..end.0.0]; assert_eq!( snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default() + ) .map(|c| c.text) .collect::(), text, @@ -2303,7 +2319,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = snapshot.chunks( FoldOffset(MultiBufferOffset(0))..FoldOffset(snapshot.len().0), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); diff --git a/crates/editor/src/display_map/inlay_map.rs b/crates/editor/src/display_map/inlay_map.rs index 47ca295ccb1a08768ce129b92d10506294a9cf78..698b58682d7ef7682094e7728f419348fd5d32d9 100644 --- a/crates/editor/src/display_map/inlay_map.rs +++ b/crates/editor/src/display_map/inlay_map.rs @@ -10,7 +10,7 @@ use crate::{ inlays::{Inlay, InlayContent}, }; use collections::BTreeSet; -use language::{Chunk, Edit, Point, TextSummary}; +use language::{Chunk, Edit, LanguageAwareStyling, Point, TextSummary}; use multi_buffer::{ MBTextSummary, MultiBufferOffset, MultiBufferRow, MultiBufferRows, MultiBufferSnapshot, RowInfo, ToOffset, @@ -1200,7 +1200,7 @@ impl InlaySnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> InlayChunks<'a> { let mut cursor = self @@ -1234,9 +1234,16 @@ impl InlaySnapshot { #[cfg(test)] #[ztracing::instrument(skip_all)] pub fn text(&self) -> String { - self.chunks(Default::default()..self.len(), false, Highlights::default()) - .map(|chunk| chunk.chunk.text) - .collect() + self.chunks( + Default::default()..self.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) + .map(|chunk| chunk.chunk.text) + .collect() } #[ztracing::instrument(skip_all)] @@ -1979,7 +1986,10 @@ mod tests { let actual_text = inlay_snapshot .chunks( range, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights { text_highlights: Some(&text_highlights), inlay_highlights: Some(&inlay_highlights), @@ -2158,7 +2168,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = snapshot.chunks( InlayOffset(MultiBufferOffset(0))..snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); @@ -2293,7 +2306,10 @@ mod tests { let chunks: Vec<_> = inlay_snapshot .chunks( InlayOffset(MultiBufferOffset(0))..inlay_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, highlights, ) .collect(); @@ -2408,7 +2424,10 @@ mod tests { let chunks: Vec<_> = inlay_snapshot .chunks( InlayOffset(MultiBufferOffset(0))..inlay_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, highlights, ) .collect(); diff --git a/crates/editor/src/display_map/tab_map.rs b/crates/editor/src/display_map/tab_map.rs index 187ed8614e01ddb8dcdae930fd484de9594cf63f..bb0e642df380e04fcfa9b9533f027be7171b4975 100644 --- a/crates/editor/src/display_map/tab_map.rs +++ b/crates/editor/src/display_map/tab_map.rs @@ -3,7 +3,7 @@ use super::{ fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot}, }; -use language::Point; +use language::{LanguageAwareStyling, Point}; use multi_buffer::MultiBufferSnapshot; use std::{cmp, num::NonZeroU32, ops::Range}; use sum_tree::Bias; @@ -101,7 +101,10 @@ impl TabMap { let mut last_tab_with_changed_expansion_offset = None; 'outer: for chunk in old_snapshot.fold_snapshot.chunks( fold_edit.old.end..old_end_row_successor_offset, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) { let mut remaining_tabs = chunk.tabs; @@ -244,7 +247,14 @@ impl TabSnapshot { self.max_point() }; let first_line_chars = self - .chunks(range.start..line_end, false, Highlights::default()) + .chunks( + range.start..line_end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) .flat_map(|chunk| chunk.text.chars()) .take_while(|&c| c != '\n') .count() as u32; @@ -254,7 +264,10 @@ impl TabSnapshot { } else { self.chunks( TabPoint::new(range.end.row(), 0)..range.end, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .flat_map(|chunk| chunk.text.chars()) @@ -274,7 +287,7 @@ impl TabSnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> TabChunks<'a> { let (input_start, expanded_char_column, to_next_stop) = @@ -324,7 +337,10 @@ impl TabSnapshot { pub fn text(&self) -> String { self.chunks( TabPoint::zero()..self.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|chunk| chunk.text) @@ -1170,7 +1186,10 @@ mod tests { tab_snapshot .chunks( TabPoint::new(0, ix as u32)..tab_snapshot.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|c| c.text) @@ -1246,8 +1265,14 @@ mod tests { let mut chunks = Vec::new(); let mut was_tab = false; let mut text = String::new(); - for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default()) - { + for chunk in snapshot.chunks( + start..snapshot.max_point(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) { if chunk.is_tab != was_tab { if !text.is_empty() { chunks.push((mem::take(&mut text), was_tab)); @@ -1296,7 +1321,14 @@ mod tests { // This should not panic. let result: String = tab_snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) .map(|c| c.text) .collect(); assert!(!result.is_empty()); @@ -1354,7 +1386,14 @@ mod tests { let expected_summary = TextSummary::from(expected_text.as_str()); assert_eq!( tabs_snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default() + ) .map(|c| c.text) .collect::(), expected_text, @@ -1436,7 +1475,10 @@ mod tests { let (_, fold_snapshot) = FoldMap::new(inlay_snapshot); let chunks = fold_snapshot.chunks( FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Default::default(), ); let mut cursor = TabStopCursor::new(chunks); @@ -1598,7 +1640,10 @@ mod tests { let (_, fold_snapshot) = FoldMap::new(inlay_snapshot); let chunks = fold_snapshot.chunks( FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Default::default(), ); let mut cursor = TabStopCursor::new(chunks); diff --git a/crates/editor/src/display_map/wrap_map.rs b/crates/editor/src/display_map/wrap_map.rs index d21642977ed923e15a583dfe767fd566e78c5de9..4ff11b1ef67971c5159a81278a5afaaaea171a28 100644 --- a/crates/editor/src/display_map/wrap_map.rs +++ b/crates/editor/src/display_map/wrap_map.rs @@ -5,7 +5,7 @@ use super::{ tab_map::{self, TabEdit, TabPoint, TabSnapshot}, }; use gpui::{App, AppContext as _, Context, Entity, Font, LineWrapper, Pixels, Task}; -use language::Point; +use language::{LanguageAwareStyling, Point}; use multi_buffer::{MultiBufferSnapshot, RowInfo}; use smol::future::yield_now; use std::{cmp, collections::VecDeque, mem, ops::Range, sync::LazyLock, time::Duration}; @@ -513,7 +513,10 @@ impl WrapSnapshot { let mut remaining = None; let mut chunks = new_tab_snapshot.chunks( TabPoint::new(edit.new_rows.start, 0)..new_tab_snapshot.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); let mut edit_transforms = Vec::::new(); @@ -656,7 +659,7 @@ impl WrapSnapshot { pub(crate) fn chunks<'a>( &'a self, rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> WrapChunks<'a> { let output_start = WrapPoint::new(rows.start, 0); @@ -960,7 +963,10 @@ impl WrapSnapshot { pub fn text_chunks(&self, wrap_row: WrapRow) -> impl Iterator { self.chunks( wrap_row..self.max_point().row() + WrapRow(1), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|h| h.text) @@ -1719,7 +1725,10 @@ mod tests { let actual_text = self .chunks( WrapRow(start_row)..WrapRow(end_row), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, Highlights::default(), ) .map(|c| c.text) diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 6550d79c9f73799d37ccf6433db38f2719636ee6..ae852b1055b33f151b402ee999ce50ba064788a4 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -132,9 +132,9 @@ use language::{ AutoindentMode, BlockCommentConfig, BracketMatch, BracketPair, Buffer, BufferRow, BufferSnapshot, Capability, CharClassifier, CharKind, CharScopeContext, CodeLabel, CursorShape, DiagnosticEntryRef, DiffOptions, EditPredictionsMode, EditPreview, HighlightedText, IndentKind, - IndentSize, Language, LanguageName, LanguageRegistry, LanguageScope, LocalFile, OffsetRangeExt, - OutlineItem, Point, Selection, SelectionGoal, TextObject, TransactionId, TreeSitterOptions, - WordsQuery, + IndentSize, Language, LanguageAwareStyling, LanguageName, LanguageRegistry, LanguageScope, + LocalFile, OffsetRangeExt, OutlineItem, Point, Selection, SelectionGoal, TextObject, + TransactionId, TreeSitterOptions, WordsQuery, language_settings::{ self, AllLanguageSettings, LanguageSettings, LspInsertMode, RewrapBehavior, WordsCompletionMode, all_language_settings, @@ -19147,7 +19147,13 @@ impl Editor { let range = buffer.anchor_before(rename_start)..buffer.anchor_after(rename_end); let mut old_highlight_id = None; let old_name: Arc = buffer - .chunks(rename_start..rename_end, true) + .chunks( + rename_start..rename_end, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) .map(|chunk| { if old_highlight_id.is_none() { old_highlight_id = chunk.syntax_highlight_id; @@ -25005,7 +25011,13 @@ impl Editor { selection.range() }; - let chunks = snapshot.chunks(range, true); + let chunks = snapshot.chunks( + range, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ); let mut lines = Vec::new(); let mut line: VecDeque = VecDeque::new(); diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index 7a532dc7a75ea3583456be6611ef072cd7692bc7..512fbb8855aa11d8c540065a55eb296919012821 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -51,7 +51,10 @@ use gpui::{ pattern_slash, point, px, quad, relative, size, solid_background, transparent_black, }; use itertools::Itertools; -use language::{HighlightedText, IndentGuideSettings, language_settings::ShowWhitespaceSetting}; +use language::{ + HighlightedText, IndentGuideSettings, LanguageAwareStyling, + language_settings::ShowWhitespaceSetting, +}; use markdown::Markdown; use multi_buffer::{ Anchor, ExcerptBoundaryInfo, ExpandExcerptDirection, ExpandInfo, MultiBufferPoint, @@ -3819,7 +3822,11 @@ impl EditorElement { } else { let use_tree_sitter = !snapshot.semantic_tokens_enabled || snapshot.use_tree_sitter_for_syntax(rows.start, cx); - let chunks = snapshot.highlighted_chunks(rows.clone(), use_tree_sitter, style); + let language_aware = LanguageAwareStyling { + tree_sitter: use_tree_sitter, + diagnostics: true, + }; + let chunks = snapshot.highlighted_chunks(rows.clone(), language_aware, style); LineWithInvisibles::from_chunks( chunks, style, @@ -11999,7 +12006,11 @@ pub fn layout_line( ) -> LineWithInvisibles { let use_tree_sitter = !snapshot.semantic_tokens_enabled || snapshot.use_tree_sitter_for_syntax(row, cx); - let chunks = snapshot.highlighted_chunks(row..row + DisplayRow(1), use_tree_sitter, style); + let language_aware = LanguageAwareStyling { + tree_sitter: use_tree_sitter, + diagnostics: true, + }; + let chunks = snapshot.highlighted_chunks(row..row + DisplayRow(1), language_aware, style); LineWithInvisibles::from_chunks( chunks, style, diff --git a/crates/editor/src/semantic_tokens.rs b/crates/editor/src/semantic_tokens.rs index 5e78be70d5627bd4f484a3efd44b13519b31b400..d485cfa70237fed542a240f202a8dc47b07467c4 100644 --- a/crates/editor/src/semantic_tokens.rs +++ b/crates/editor/src/semantic_tokens.rs @@ -475,13 +475,17 @@ mod tests { use gpui::{ AppContext as _, Entity, Focusable as _, HighlightStyle, TestAppContext, UpdateGlobal as _, }; - use language::{Language, LanguageConfig, LanguageMatcher}; + use language::{ + Diagnostic, DiagnosticEntry, DiagnosticSet, Language, LanguageAwareStyling, LanguageConfig, + LanguageMatcher, + }; use languages::FakeLspAdapter; + use lsp::LanguageServerId; use multi_buffer::{ AnchorRangeExt, ExpandExcerptDirection, MultiBuffer, MultiBufferOffset, PathKey, }; use project::Project; - use rope::Point; + use rope::{Point, PointUtf16}; use serde_json::json; use settings::{ GlobalLspSettingsContent, LanguageSettingsContent, SemanticTokenRule, SemanticTokenRules, @@ -2088,6 +2092,130 @@ mod tests { ); } + #[gpui::test] + async fn test_diagnostics_visible_when_semantic_token_set_to_full(cx: &mut TestAppContext) { + init_test(cx, |_| {}); + + update_test_language_settings(cx, &|language_settings| { + language_settings.languages.0.insert( + "Rust".into(), + LanguageSettingsContent { + semantic_tokens: Some(SemanticTokens::Full), + ..LanguageSettingsContent::default() + }, + ); + }); + + let mut cx = EditorLspTestContext::new_rust( + lsp::ServerCapabilities { + semantic_tokens_provider: Some( + lsp::SemanticTokensServerCapabilities::SemanticTokensOptions( + lsp::SemanticTokensOptions { + legend: lsp::SemanticTokensLegend { + token_types: vec!["function".into()], + token_modifiers: Vec::new(), + }, + full: Some(lsp::SemanticTokensFullOptions::Delta { delta: None }), + ..lsp::SemanticTokensOptions::default() + }, + ), + ), + ..lsp::ServerCapabilities::default() + }, + cx, + ) + .await; + + let mut full_request = cx + .set_request_handler::( + move |_, _, _| { + async move { + Ok(Some(lsp::SemanticTokensResult::Tokens( + lsp::SemanticTokens { + data: vec![ + 0, // delta_line + 3, // delta_start + 4, // length + 0, // token_type + 0, // token_modifiers_bitset + ], + result_id: Some("a".into()), + }, + ))) + } + }, + ); + + cx.set_state("ˇfn main() {}"); + assert!(full_request.next().await.is_some()); + + let task = cx.update_editor(|e, _, _| e.semantic_token_state.take_update_task()); + task.await; + + cx.update_buffer(|buffer, cx| { + buffer.update_diagnostics( + LanguageServerId(0), + DiagnosticSet::new( + [DiagnosticEntry { + range: PointUtf16::new(0, 3)..PointUtf16::new(0, 7), + diagnostic: Diagnostic { + severity: lsp::DiagnosticSeverity::ERROR, + group_id: 1, + message: "unused function".into(), + ..Default::default() + }, + }], + buffer, + ), + cx, + ) + }); + + cx.run_until_parked(); + let chunks = cx.update_editor(|editor, window, cx| { + editor + .snapshot(window, cx) + .display_snapshot + .chunks( + crate::display_map::DisplayRow(0)..crate::display_map::DisplayRow(1), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: true, + }, + crate::HighlightStyles::default(), + ) + .map(|chunk| { + ( + chunk.text.to_string(), + chunk.diagnostic_severity, + chunk.highlight_style, + ) + }) + .collect::>() + }); + + assert_eq!( + extract_semantic_highlights(&cx.editor, &cx), + vec![MultiBufferOffset(3)..MultiBufferOffset(7)] + ); + + assert!( + chunks.iter().any( + |(text, severity, style): &( + String, + Option, + Option + )| { + text == "main" + && *severity == Some(lsp::DiagnosticSeverity::ERROR) + && style.is_some() + } + ), + "expected 'main' chunk to have both diagnostic and semantic styling: {:?}", + chunks + ); + } + fn extract_semantic_highlight_styles( editor: &Entity, cx: &TestAppContext, diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index a467cd789555d39a32ad4e1d7b21da7b14df9c25..1e54134efcab4f0074a73b241f8e0d04cfbcbcdd 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -3733,16 +3733,24 @@ impl BufferSnapshot { /// returned in chunks where each chunk has a single syntax highlighting style and /// diagnostic status. #[ztracing::instrument(skip_all)] - pub fn chunks(&self, range: Range, language_aware: bool) -> BufferChunks<'_> { + pub fn chunks( + &self, + range: Range, + language_aware: LanguageAwareStyling, + ) -> BufferChunks<'_> { let range = range.start.to_offset(self)..range.end.to_offset(self); let mut syntax = None; - if language_aware { + if language_aware.tree_sitter { syntax = Some(self.get_highlights(range.clone())); } - // We want to look at diagnostic spans only when iterating over language-annotated chunks. - let diagnostics = language_aware; - BufferChunks::new(self.text.as_rope(), range, syntax, diagnostics, Some(self)) + BufferChunks::new( + self.text.as_rope(), + range, + syntax, + language_aware.diagnostics, + Some(self), + ) } pub fn highlighted_text_for_range( @@ -4477,7 +4485,13 @@ impl BufferSnapshot { let mut text = String::new(); let mut highlight_ranges = Vec::new(); let mut name_ranges = Vec::new(); - let mut chunks = self.chunks(source_range_for_text.clone(), true); + let mut chunks = self.chunks( + source_range_for_text.clone(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ); let mut last_buffer_range_end = 0; for (buffer_range, is_name) in buffer_ranges { let space_added = !text.is_empty() && buffer_range.start > last_buffer_range_end; @@ -5402,7 +5416,13 @@ impl BufferSnapshot { let mut words = BTreeMap::default(); let mut current_word_start_ix = None; let mut chunk_ix = query.range.start; - for chunk in self.chunks(query.range, false) { + for chunk in self.chunks( + query.range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) { for (i, c) in chunk.text.char_indices() { let ix = chunk_ix + i; if classifier.is_word(c) { @@ -5441,6 +5461,15 @@ impl BufferSnapshot { } } +/// A configuration to use when producing styled text chunks. +#[derive(Clone, Copy)] +pub struct LanguageAwareStyling { + /// Whether to highlight text chunks using tree-sitter. + pub tree_sitter: bool, + /// Whether to highlight text chunks based on the diagnostics data. + pub diagnostics: bool, +} + pub struct WordsQuery<'a> { /// Only returns words with all chars from the fuzzy string in them. pub fuzzy_contents: Option<&'a str>, diff --git a/crates/language/src/buffer_tests.rs b/crates/language/src/buffer_tests.rs index 9308ee6f0a0ee207b30be9e6fafa73ba9452d94c..9f4562bf547f389c5ecc5ca29470ac4e49da0e04 100644 --- a/crates/language/src/buffer_tests.rs +++ b/crates/language/src/buffer_tests.rs @@ -4102,7 +4102,13 @@ fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) { let snapshot = buffer.read(cx).snapshot(); // Get all chunks and verify their bitmaps - let chunks = snapshot.chunks(0..snapshot.len(), false); + let chunks = snapshot.chunks( + 0..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; diff --git a/crates/multi_buffer/src/multi_buffer.rs b/crates/multi_buffer/src/multi_buffer.rs index a54ff64af028f44adced1758933f794e9a002c5a..47c1288c8f9baeebf4afd54dd0597bfe5a41d15f 100644 --- a/crates/multi_buffer/src/multi_buffer.rs +++ b/crates/multi_buffer/src/multi_buffer.rs @@ -21,9 +21,9 @@ use itertools::Itertools; use language::{ AutoindentMode, Buffer, BufferChunks, BufferRow, BufferSnapshot, Capability, CharClassifier, CharKind, CharScopeContext, Chunk, CursorShape, DiagnosticEntryRef, File, IndentGuideSettings, - IndentSize, Language, LanguageScope, OffsetRangeExt, OffsetUtf16, Outline, OutlineItem, Point, - PointUtf16, Selection, TextDimension, TextObject, ToOffset as _, ToPoint as _, TransactionId, - TreeSitterOptions, Unclipped, + IndentSize, Language, LanguageAwareStyling, LanguageScope, OffsetRangeExt, OffsetUtf16, + Outline, OutlineItem, Point, PointUtf16, Selection, TextDimension, TextObject, ToOffset as _, + ToPoint as _, TransactionId, TreeSitterOptions, Unclipped, language_settings::{AllLanguageSettings, LanguageSettings}, }; @@ -1072,7 +1072,7 @@ pub struct MultiBufferChunks<'a> { range: Range, excerpt_offset_range: Range, excerpt_chunks: Option>, - language_aware: bool, + language_aware: LanguageAwareStyling, snapshot: &'a MultiBufferSnapshot, } @@ -3340,9 +3340,15 @@ impl EventEmitter for MultiBuffer {} impl MultiBufferSnapshot { pub fn text(&self) -> String { - self.chunks(MultiBufferOffset::ZERO..self.len(), false) - .map(|chunk| chunk.text) - .collect() + self.chunks( + MultiBufferOffset::ZERO..self.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) + .map(|chunk| chunk.text) + .collect() } pub fn reversed_chars_at(&self, position: T) -> impl Iterator + '_ { @@ -3378,7 +3384,14 @@ impl MultiBufferSnapshot { } pub fn text_for_range(&self, range: Range) -> impl Iterator + '_ { - self.chunks(range, false).map(|chunk| chunk.text) + self.chunks( + range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) + .map(|chunk| chunk.text) } pub fn is_line_blank(&self, row: MultiBufferRow) -> bool { @@ -4178,7 +4191,7 @@ impl MultiBufferSnapshot { pub fn chunks( &self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, ) -> MultiBufferChunks<'_> { let mut chunks = MultiBufferChunks { excerpt_offset_range: ExcerptDimension(MultiBufferOffset::ZERO) @@ -7227,7 +7240,7 @@ impl Excerpt { fn chunks_in_range<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, snapshot: &'a MultiBufferSnapshot, ) -> ExcerptChunks<'a> { let buffer = self.buffer_snapshot(snapshot); diff --git a/crates/multi_buffer/src/multi_buffer_tests.rs b/crates/multi_buffer/src/multi_buffer_tests.rs index bc904d1a05488ee365ebddf36c3b30accdfb9301..cebc9073e9d87a3c6eaf71d78e181d3e833ad56a 100644 --- a/crates/multi_buffer/src/multi_buffer_tests.rs +++ b/crates/multi_buffer/src/multi_buffer_tests.rs @@ -5039,7 +5039,13 @@ fn check_edits( fn assert_chunks_in_ranges(snapshot: &MultiBufferSnapshot) { let full_text = snapshot.text(); for ix in 0..full_text.len() { - let mut chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let mut chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); chunks.seek(MultiBufferOffset(ix)..snapshot.len()); let tail = chunks.map(|chunk| chunk.text).collect::(); assert_eq!(tail, &full_text[ix..], "seek to range: {:?}", ix..); @@ -5300,7 +5306,13 @@ fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) { let snapshot = multibuffer.read(cx).snapshot(cx); - let chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; @@ -5466,7 +5478,13 @@ fn test_random_chunk_bitmaps_with_diffs(cx: &mut App, mut rng: StdRng) { let snapshot = multibuffer.read(cx).snapshot(cx); - let chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; diff --git a/crates/outline_panel/src/outline_panel.rs b/crates/outline_panel/src/outline_panel.rs index b7d5afcb687c017fdf253717a9dae2c95c55b53b..fa23b805cd48461dabaddbb7670155cdfe1ba8b0 100644 --- a/crates/outline_panel/src/outline_panel.rs +++ b/crates/outline_panel/src/outline_panel.rs @@ -23,8 +23,8 @@ use gpui::{ uniform_list, }; use itertools::Itertools; -use language::language_settings::LanguageSettings; use language::{Anchor, BufferId, BufferSnapshot, OffsetRangeExt, OutlineItem}; +use language::{LanguageAwareStyling, language_settings::LanguageSettings}; use menu::{Cancel, SelectFirst, SelectLast, SelectNext, SelectPrevious}; use std::{ @@ -217,10 +217,13 @@ impl SearchState { let mut offset = context_offset_range.start; let mut context_text = String::new(); let mut highlight_ranges = Vec::new(); - for mut chunk in highlight_arguments - .multi_buffer_snapshot - .chunks(context_offset_range.start..context_offset_range.end, true) - { + for mut chunk in highlight_arguments.multi_buffer_snapshot.chunks( + context_offset_range.start..context_offset_range.end, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { if !non_whitespace_symbol_occurred { for c in chunk.text.chars() { if c.is_whitespace() { diff --git a/crates/project/src/lsp_store.rs b/crates/project/src/lsp_store.rs index 2f579f5a724db143bbd4b0f9853a217bd6b14655..9ea50fdc8f12b68147c1073219625c4fd257afd3 100644 --- a/crates/project/src/lsp_store.rs +++ b/crates/project/src/lsp_store.rs @@ -72,9 +72,10 @@ use itertools::Itertools as _; use language::{ Bias, BinaryStatus, Buffer, BufferRow, BufferSnapshot, CachedLspAdapter, Capability, CodeLabel, CodeLabelExt, Diagnostic, DiagnosticEntry, DiagnosticSet, DiagnosticSourceKind, Diff, - File as _, Language, LanguageName, LanguageRegistry, LocalFile, LspAdapter, LspAdapterDelegate, - LspInstaller, ManifestDelegate, ManifestName, ModelineSettings, OffsetUtf16, Patch, PointUtf16, - TextBufferSnapshot, ToOffset, ToOffsetUtf16, ToPointUtf16, Toolchain, Transaction, Unclipped, + File as _, Language, LanguageAwareStyling, LanguageName, LanguageRegistry, LocalFile, + LspAdapter, LspAdapterDelegate, LspInstaller, ManifestDelegate, ManifestName, ModelineSettings, + OffsetUtf16, Patch, PointUtf16, TextBufferSnapshot, ToOffset, ToOffsetUtf16, ToPointUtf16, + Toolchain, Transaction, Unclipped, language_settings::{ AllLanguageSettings, FormatOnSave, Formatter, LanguageSettings, all_language_settings, }, @@ -13527,7 +13528,13 @@ fn resolve_word_completion(snapshot: &BufferSnapshot, completion: &mut Completio } let mut offset = 0; - for chunk in snapshot.chunks(word_range.clone(), true) { + for chunk in snapshot.chunks( + word_range.clone(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { let end_offset = offset + chunk.text.len(); if let Some(highlight_id) = chunk.syntax_highlight_id { completion diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs index d6c2ce37c9e60e17bd43c3f6c3ad10cde52b4bec..f680ccee78e997064af2647f68d8aa3631fa4bd3 100644 --- a/crates/project/tests/integration/project_tests.rs +++ b/crates/project/tests/integration/project_tests.rs @@ -41,9 +41,10 @@ use gpui::{ use itertools::Itertools; use language::{ Buffer, BufferEvent, Diagnostic, DiagnosticEntry, DiagnosticEntryRef, DiagnosticSet, - DiagnosticSourceKind, DiskState, FakeLspAdapter, Language, LanguageConfig, LanguageMatcher, - LanguageName, LineEnding, ManifestName, ManifestProvider, ManifestQuery, OffsetRangeExt, Point, - ToPoint, Toolchain, ToolchainList, ToolchainLister, ToolchainMetadata, + DiagnosticSourceKind, DiskState, FakeLspAdapter, Language, LanguageAwareStyling, + LanguageConfig, LanguageMatcher, LanguageName, LineEnding, ManifestName, ManifestProvider, + ManifestQuery, OffsetRangeExt, Point, ToPoint, Toolchain, ToolchainList, ToolchainLister, + ToolchainMetadata, language_settings::{LanguageSettings, LanguageSettingsContent}, markdown_lang, rust_lang, tree_sitter_typescript, }; @@ -4382,7 +4383,13 @@ fn chunks_with_diagnostics( range: Range, ) -> Vec<(String, Option)> { let mut chunks: Vec<(String, Option)> = Vec::new(); - for chunk in buffer.snapshot().chunks(range, true) { + for chunk in buffer.snapshot().chunks( + range, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { if chunks .last() .is_some_and(|prev_chunk| prev_chunk.1 == chunk.diagnostic_severity) diff --git a/crates/vim/src/state.rs b/crates/vim/src/state.rs index 4dd557199ab9aebe0a2b26438bdaa0e321a956b2..9e9b42d31900e0ceb160df4ad4dd3ce3a530e155 100644 --- a/crates/vim/src/state.rs +++ b/crates/vim/src/state.rs @@ -17,7 +17,7 @@ use gpui::{ Action, App, AppContext, BorrowAppContext, ClipboardEntry, ClipboardItem, DismissEvent, Entity, EntityId, Global, HighlightStyle, StyledText, Subscription, Task, TextStyle, WeakEntity, }; -use language::{Buffer, BufferEvent, BufferId, Chunk, Point}; +use language::{Buffer, BufferEvent, BufferId, Chunk, LanguageAwareStyling, Point}; use multi_buffer::MultiBufferRow; use picker::{Picker, PickerDelegate}; @@ -1504,7 +1504,10 @@ impl PickerDelegate for MarksViewDelegate { position.row, snapshot.line_len(MultiBufferRow(position.row)), ), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, ); matches.push(MarksMatch { name: name.clone(), @@ -1530,7 +1533,10 @@ impl PickerDelegate for MarksViewDelegate { let chunks = snapshot.chunks( Point::new(position.row, 0) ..Point::new(position.row, snapshot.line_len(position.row)), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, ); matches.push(MarksMatch {