From a801a4aeef93e24d235da51a9a1fdbe9f5d1d6d1 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Wed, 27 Sep 2023 13:16:32 -0600 Subject: [PATCH 01/41] Remove some unnecessary Eqs --- .cargo/config.toml | 2 +- crates/language/src/buffer.rs | 4 ++-- crates/text/src/selection.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 9da6b3be080072d89d16a199e2d60d527eeacd07..e22bdb0f2c70a1ffda714674253cc533e9e7c1d1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,4 +3,4 @@ xtask = "run --package xtask --" [build] # v0 mangling scheme provides more detailed backtraces around closures -rustflags = ["-C", "symbol-mangling-version=v0"] +rustflags = ["-C", "symbol-mangling-version=v0", "-C", "link-arg=-fuse-ld=/opt/homebrew/Cellar/llvm/16.0.6/bin/ld64.lld"] diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 207c41e7cdf21a8ec4d61232e071296184b52a5f..19e5e290b9ce444cc3901840fca09449b730341b 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -159,7 +159,7 @@ pub struct CodeAction { pub lsp_action: lsp::CodeAction, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub enum Operation { Buffer(text::Operation), @@ -182,7 +182,7 @@ pub enum Operation { }, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub enum Event { Operation(Operation), Edited, diff --git a/crates/text/src/selection.rs b/crates/text/src/selection.rs index 205c27239d90699257816b5edfe3f6e38fa34dec..60d5e2f1c437086c246f7e4c94ba6536e0f80840 100644 --- a/crates/text/src/selection.rs +++ b/crates/text/src/selection.rs @@ -2,14 +2,14 @@ use crate::{Anchor, BufferSnapshot, TextDimension}; use std::cmp::Ordering; use std::ops::Range; -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq)] pub enum SelectionGoal { None, Column(u32), ColumnRange { start: u32, end: u32 }, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Selection { pub id: usize, pub start: T, From dacc8cb5f47ae8272afaf560979c7eb6d67e3354 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Wed, 27 Sep 2023 15:10:50 -0600 Subject: [PATCH 02/41] Begin to use pixels for column selection For zed-industries/community#759 For zed-industries/community#1966 Co-Authored-By: Julia --- crates/editor/src/display_map.rs | 381 ++++++++++++++++++++++++------- crates/editor/src/editor.rs | 51 ++++- crates/editor/src/element.rs | 59 +---- crates/editor/src/movement.rs | 50 +++- crates/text/src/selection.rs | 2 + 5 files changed, 382 insertions(+), 161 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index d97db9695ac4052f647a58d90fa1f23b4188004d..3d13447fc23749b3daea4049c9b76ef11fb0ff2c 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -5,22 +5,24 @@ mod tab_map; mod wrap_map; use crate::{ - link_go_to_definition::InlayHighlight, Anchor, AnchorRangeExt, InlayId, MultiBuffer, - MultiBufferSnapshot, ToOffset, ToPoint, + link_go_to_definition::InlayHighlight, Anchor, AnchorRangeExt, EditorStyle, InlayId, + MultiBuffer, MultiBufferSnapshot, ToOffset, ToPoint, }; pub use block_map::{BlockMap, BlockPoint}; use collections::{BTreeMap, HashMap, HashSet}; use fold_map::FoldMap; use gpui::{ color::Color, - fonts::{FontId, HighlightStyle}, - Entity, ModelContext, ModelHandle, + fonts::{FontId, HighlightStyle, Underline}, + text_layout::{Line, RunStyle}, + AppContext, Entity, FontCache, ModelContext, ModelHandle, TextLayoutCache, }; use inlay_map::InlayMap; use language::{ language_settings::language_settings, OffsetUtf16, Point, Subscription as BufferSubscription, }; -use std::{any::TypeId, fmt::Debug, num::NonZeroU32, ops::Range, sync::Arc}; +use lsp::DiagnosticSeverity; +use std::{any::TypeId, borrow::Cow, fmt::Debug, num::NonZeroU32, ops::Range, sync::Arc}; use sum_tree::{Bias, TreeMap}; use tab_map::TabMap; use wrap_map::WrapMap; @@ -316,6 +318,12 @@ pub struct Highlights<'a> { pub suggestion_highlight_style: Option, } +pub struct HighlightedChunk<'a> { + pub chunk: &'a str, + pub style: Option, + pub is_tab: bool, +} + pub struct DisplaySnapshot { pub buffer_snapshot: MultiBufferSnapshot, pub fold_snapshot: fold_map::FoldSnapshot, @@ -485,7 +493,7 @@ impl DisplaySnapshot { language_aware: bool, inlay_highlight_style: Option, suggestion_highlight_style: Option, - ) -> DisplayChunks<'_> { + ) -> DisplayChunks<'a> { self.block_snapshot.chunks( display_rows, language_aware, @@ -498,6 +506,174 @@ impl DisplaySnapshot { ) } + pub fn highlighted_chunks<'a>( + &'a self, + display_rows: Range, + style: &'a EditorStyle, + ) -> impl Iterator> { + self.chunks( + display_rows, + true, + Some(style.theme.hint), + Some(style.theme.suggestion), + ) + .map(|chunk| { + let mut highlight_style = chunk + .syntax_highlight_id + .and_then(|id| id.style(&style.syntax)); + + if let Some(chunk_highlight) = chunk.highlight_style { + if let Some(highlight_style) = highlight_style.as_mut() { + highlight_style.highlight(chunk_highlight); + } else { + highlight_style = Some(chunk_highlight); + } + } + + let mut diagnostic_highlight = HighlightStyle::default(); + + if chunk.is_unnecessary { + diagnostic_highlight.fade_out = Some(style.unnecessary_code_fade); + } + + if let Some(severity) = chunk.diagnostic_severity { + // Omit underlines for HINT/INFO diagnostics on 'unnecessary' code. + if severity <= DiagnosticSeverity::WARNING || !chunk.is_unnecessary { + let diagnostic_style = super::diagnostic_style(severity, true, style); + diagnostic_highlight.underline = Some(Underline { + color: Some(diagnostic_style.message.text.color), + thickness: 1.0.into(), + squiggly: true, + }); + } + } + + if let Some(highlight_style) = highlight_style.as_mut() { + highlight_style.highlight(diagnostic_highlight); + } else { + highlight_style = Some(diagnostic_highlight); + } + + HighlightedChunk { + chunk: chunk.text, + style: highlight_style, + is_tab: chunk.is_tab, + } + }) + } + + fn layout_line_for_row( + &self, + display_row: u32, + font_cache: &FontCache, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, + ) -> Line { + let mut styles = Vec::new(); + let mut line = String::new(); + + let range = display_row..display_row + 1; + for chunk in self.highlighted_chunks(range, editor_style) { + dbg!(chunk.chunk); + line.push_str(chunk.chunk); + + let text_style = if let Some(style) = chunk.style { + editor_style + .text + .clone() + .highlight(style, font_cache) + .map(Cow::Owned) + .unwrap_or_else(|_| Cow::Borrowed(&editor_style.text)) + } else { + Cow::Borrowed(&editor_style.text) + }; + + styles.push(( + chunk.chunk.len(), + RunStyle { + font_id: text_style.font_id, + color: text_style.color, + underline: text_style.underline, + }, + )); + } + + dbg!(&line, &editor_style.text.font_size, &styles); + text_layout_cache.layout_str(&line, editor_style.text.font_size, &styles) + } + + pub fn x_for_point( + &self, + display_point: DisplayPoint, + font_cache: &FontCache, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, + ) -> f32 { + let layout_line = self.layout_line_for_row( + display_point.row(), + font_cache, + text_layout_cache, + editor_style, + ); + layout_line.x_for_index(display_point.column() as usize) + } + + pub fn column_for_x( + &self, + display_row: u32, + x_coordinate: f32, + font_cache: &FontCache, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, + ) -> Option { + let layout_line = + self.layout_line_for_row(display_row, font_cache, text_layout_cache, editor_style); + layout_line.index_for_x(x_coordinate).map(|c| c as u32) + } + + // column_for_x(row, x) + + fn point( + &self, + display_point: DisplayPoint, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, + cx: &AppContext, + ) -> f32 { + let mut styles = Vec::new(); + let mut line = String::new(); + + let range = display_point.row()..display_point.row() + 1; + for chunk in self.highlighted_chunks(range, editor_style) { + dbg!(chunk.chunk); + line.push_str(chunk.chunk); + + let text_style = if let Some(style) = chunk.style { + editor_style + .text + .clone() + .highlight(style, cx.font_cache()) + .map(Cow::Owned) + .unwrap_or_else(|_| Cow::Borrowed(&editor_style.text)) + } else { + Cow::Borrowed(&editor_style.text) + }; + + styles.push(( + chunk.chunk.len(), + RunStyle { + font_id: text_style.font_id, + color: text_style.color, + underline: text_style.underline, + }, + )); + } + + dbg!(&line, &editor_style.text.font_size, &styles); + let layout_line = text_layout_cache.layout_str(&line, editor_style.text.font_size, &styles); + layout_line.x_for_index(display_point.column() as usize) + } + pub fn chars_at( &self, mut point: DisplayPoint, @@ -869,17 +1045,21 @@ pub fn next_rows(display_row: u32, display_map: &DisplaySnapshot) -> impl Iterat #[cfg(test)] pub mod tests { use super::*; - use crate::{movement, test::marked_display_snapshot}; + use crate::{ + movement, + test::{editor_test_context::EditorTestContext, marked_display_snapshot}, + }; use gpui::{color::Color, elements::*, test::observe, AppContext}; use language::{ language_settings::{AllLanguageSettings, AllLanguageSettingsContent}, Buffer, Language, LanguageConfig, SelectionGoal, }; + use project::Project; use rand::{prelude::*, Rng}; use settings::SettingsStore; use smol::stream::StreamExt; use std::{env, sync::Arc}; - use theme::SyntaxTheme; + use theme::{SyntaxTheme, Theme}; use util::test::{marked_text_ranges, sample_text}; use Bias::*; @@ -1148,95 +1328,119 @@ pub mod tests { } #[gpui::test(retries = 5)] - fn test_soft_wraps(cx: &mut AppContext) { + async fn test_soft_wraps(cx: &mut gpui::TestAppContext) { cx.foreground().set_block_on_ticks(usize::MAX..=usize::MAX); - init_test(cx, |_| {}); + cx.update(|cx| { + init_test(cx, |_| {}); + }); - let font_cache = cx.font_cache(); + let mut cx = EditorTestContext::new(cx).await; + let editor = cx.editor.clone(); + let window = cx.window.clone(); - let family_id = font_cache - .load_family(&["Helvetica"], &Default::default()) - .unwrap(); - let font_id = font_cache - .select_font(family_id, &Default::default()) - .unwrap(); - let font_size = 12.0; - let wrap_width = Some(64.); + cx.update_window(window, |cx| { + let editor_style = editor.read(&cx).style(cx); - let text = "one two three four five\nsix seven eight"; - let buffer = MultiBuffer::build_simple(text, cx); - let map = cx.add_model(|cx| { - DisplayMap::new(buffer.clone(), font_id, font_size, wrap_width, 1, 1, cx) - }); + let font_cache = cx.font_cache().clone(); - let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); - assert_eq!( - snapshot.text_chunks(0).collect::(), - "one two \nthree four \nfive\nsix seven \neight" - ); - assert_eq!( - snapshot.clip_point(DisplayPoint::new(0, 8), Bias::Left), - DisplayPoint::new(0, 7) - ); - assert_eq!( - snapshot.clip_point(DisplayPoint::new(0, 8), Bias::Right), - DisplayPoint::new(1, 0) - ); - assert_eq!( - movement::right(&snapshot, DisplayPoint::new(0, 7)), - DisplayPoint::new(1, 0) - ); - assert_eq!( - movement::left(&snapshot, DisplayPoint::new(1, 0)), - DisplayPoint::new(0, 7) - ); - assert_eq!( - movement::up( - &snapshot, - DisplayPoint::new(1, 10), - SelectionGoal::None, - false - ), - (DisplayPoint::new(0, 7), SelectionGoal::Column(10)) - ); - assert_eq!( - movement::down( - &snapshot, - DisplayPoint::new(0, 7), - SelectionGoal::Column(10), - false - ), - (DisplayPoint::new(1, 10), SelectionGoal::Column(10)) - ); - assert_eq!( - movement::down( - &snapshot, + let family_id = font_cache + .load_family(&["Helvetica"], &Default::default()) + .unwrap(); + let font_id = font_cache + .select_font(family_id, &Default::default()) + .unwrap(); + let font_size = 12.0; + let wrap_width = Some(64.); + + let text = "one two three four five\nsix seven eight"; + let buffer = MultiBuffer::build_simple(text, cx); + let map = cx.add_model(|cx| { + DisplayMap::new(buffer.clone(), font_id, font_size, wrap_width, 1, 1, cx) + }); + + let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); + assert_eq!( + snapshot.text_chunks(0).collect::(), + "one two \nthree four \nfive\nsix seven \neight" + ); + assert_eq!( + snapshot.clip_point(DisplayPoint::new(0, 8), Bias::Left), + DisplayPoint::new(0, 7) + ); + assert_eq!( + snapshot.clip_point(DisplayPoint::new(0, 8), Bias::Right), + DisplayPoint::new(1, 0) + ); + assert_eq!( + movement::right(&snapshot, DisplayPoint::new(0, 7)), + DisplayPoint::new(1, 0) + ); + assert_eq!( + movement::left(&snapshot, DisplayPoint::new(1, 0)), + DisplayPoint::new(0, 7) + ); + + let x = snapshot.x_for_point( DisplayPoint::new(1, 10), - SelectionGoal::Column(10), - false - ), - (DisplayPoint::new(2, 4), SelectionGoal::Column(10)) - ); + cx.font_cache(), + cx.text_layout_cache(), + &editor_style, + ); + dbg!(x); + assert_eq!( + movement::up( + &snapshot, + DisplayPoint::new(1, 10), + SelectionGoal::None, + false, + cx.font_cache(), + cx.text_layout_cache(), + &editor_style, + ), + ( + DisplayPoint::new(0, 7), + SelectionGoal::HorizontalPosition(x) + ) + ); + assert_eq!( + movement::down( + &snapshot, + DisplayPoint::new(0, 7), + SelectionGoal::Column(10), + false + ), + (DisplayPoint::new(1, 10), SelectionGoal::Column(10)) + ); + assert_eq!( + movement::down( + &snapshot, + DisplayPoint::new(1, 10), + SelectionGoal::Column(10), + false + ), + (DisplayPoint::new(2, 4), SelectionGoal::Column(10)) + ); - let ix = snapshot.buffer_snapshot.text().find("seven").unwrap(); - buffer.update(cx, |buffer, cx| { - buffer.edit([(ix..ix, "and ")], None, cx); - }); + let ix = snapshot.buffer_snapshot.text().find("seven").unwrap(); + buffer.update(cx, |buffer, cx| { + buffer.edit([(ix..ix, "and ")], None, cx); + }); - let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); - assert_eq!( - snapshot.text_chunks(1).collect::(), - "three four \nfive\nsix and \nseven eight" - ); + let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); + assert_eq!( + snapshot.text_chunks(1).collect::(), + "three four \nfive\nsix and \nseven eight" + ); - // Re-wrap on font size changes - map.update(cx, |map, cx| map.set_font(font_id, font_size + 3., cx)); + // Re-wrap on font size changes + map.update(cx, |map, cx| map.set_font(font_id, font_size + 3., cx)); - let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); - assert_eq!( - snapshot.text_chunks(1).collect::(), - "three \nfour five\nsix and \nseven \neight" - ) + let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); + assert_eq!( + snapshot.text_chunks(1).collect::(), + "three \nfour five\nsix and \nseven \neight" + ) + }); } #[gpui::test] @@ -1731,6 +1935,9 @@ pub mod tests { cx.foreground().forbid_parking(); cx.set_global(SettingsStore::test(cx)); language::init(cx); + crate::init(cx); + Project::init_settings(cx); + theme::init((), cx); cx.update_global::(|store, cx| { store.update_user_settings::(cx, f); }); diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 24ffa64a6af73aef827cea17757e5c0e6a4e94c2..bf1aa2e6b5784c4ff4d684d4a403e0bbc622de53 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -48,9 +48,9 @@ use gpui::{ impl_actions, keymap_matcher::KeymapContext, platform::{CursorStyle, MouseButton}, - serde_json, AnyElement, AnyViewHandle, AppContext, AsyncAppContext, ClipboardItem, Element, - Entity, ModelHandle, Subscription, Task, View, ViewContext, ViewHandle, WeakViewHandle, - WindowContext, + serde_json, text_layout, AnyElement, AnyViewHandle, AppContext, AsyncAppContext, ClipboardItem, + Element, Entity, ModelHandle, Subscription, Task, View, ViewContext, ViewHandle, + WeakViewHandle, WindowContext, }; use highlight_matching_bracket::refresh_matching_bracket_highlights; use hover_popover::{hide_hover, HoverState}; @@ -5274,13 +5274,25 @@ impl Editor { return; } + let font_cache = cx.font_cache().clone(); + let text_layout_cache = cx.text_layout_cache().clone(); + let editor_style = self.style(cx); + self.change_selections(Some(Autoscroll::fit()), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { if !selection.is_empty() && !line_mode { selection.goal = SelectionGoal::None; } - let (cursor, goal) = movement::up(map, selection.start, selection.goal, false); + let (cursor, goal) = movement::up( + map, + selection.start, + selection.goal, + false, + &font_cache, + &text_layout_cache, + &editor_style, + ); selection.collapse_to(cursor, goal); }); }) @@ -5308,22 +5320,47 @@ impl Editor { Autoscroll::fit() }; + let font_cache = cx.font_cache().clone(); + let text_layout = cx.text_layout_cache().clone(); + let editor_style = self.style(cx); + self.change_selections(Some(autoscroll), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { if !selection.is_empty() && !line_mode { selection.goal = SelectionGoal::None; } - let (cursor, goal) = - movement::up_by_rows(map, selection.end, row_count, selection.goal, false); + let (cursor, goal) = movement::up_by_rows( + map, + selection.end, + row_count, + selection.goal, + false, + &font_cache, + &text_layout, + &editor_style, + ); selection.collapse_to(cursor, goal); }); }); } pub fn select_up(&mut self, _: &SelectUp, cx: &mut ViewContext) { + let font_cache = cx.font_cache().clone(); + let text_layout = cx.text_layout_cache().clone(); + let editor_style = self.style(cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { - s.move_heads_with(|map, head, goal| movement::up(map, head, goal, false)) + s.move_heads_with(|map, head, goal| { + movement::up( + map, + head, + goal, + false, + &font_cache, + &text_layout, + &editor_style, + ) + }) }) } diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index 924d66c21c5532fa79a2abb0af5ac1116cde463e..24cbadfd37927266e0395a4a8c9589bb9d0ab554 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -4,7 +4,7 @@ use super::{ MAX_LINE_LEN, }; use crate::{ - display_map::{BlockStyle, DisplaySnapshot, FoldStatus, TransformBlock}, + display_map::{BlockStyle, DisplaySnapshot, FoldStatus, HighlightedChunk, TransformBlock}, editor_settings::ShowScrollbar, git::{diff_hunk_to_display, DisplayDiffHunk}, hover_popover::{ @@ -1584,56 +1584,7 @@ impl EditorElement { .collect() } else { let style = &self.style; - let chunks = snapshot - .chunks( - rows.clone(), - true, - Some(style.theme.hint), - Some(style.theme.suggestion), - ) - .map(|chunk| { - let mut highlight_style = chunk - .syntax_highlight_id - .and_then(|id| id.style(&style.syntax)); - - if let Some(chunk_highlight) = chunk.highlight_style { - if let Some(highlight_style) = highlight_style.as_mut() { - highlight_style.highlight(chunk_highlight); - } else { - highlight_style = Some(chunk_highlight); - } - } - - let mut diagnostic_highlight = HighlightStyle::default(); - - if chunk.is_unnecessary { - diagnostic_highlight.fade_out = Some(style.unnecessary_code_fade); - } - - if let Some(severity) = chunk.diagnostic_severity { - // Omit underlines for HINT/INFO diagnostics on 'unnecessary' code. - if severity <= DiagnosticSeverity::WARNING || !chunk.is_unnecessary { - let diagnostic_style = super::diagnostic_style(severity, true, style); - diagnostic_highlight.underline = Some(Underline { - color: Some(diagnostic_style.message.text.color), - thickness: 1.0.into(), - squiggly: true, - }); - } - } - - if let Some(highlight_style) = highlight_style.as_mut() { - highlight_style.highlight(diagnostic_highlight); - } else { - highlight_style = Some(diagnostic_highlight); - } - - HighlightedChunk { - chunk: chunk.text, - style: highlight_style, - is_tab: chunk.is_tab, - } - }); + let chunks = snapshot.highlighted_chunks(rows.clone(), style); LineWithInvisibles::from_chunks( chunks, @@ -1870,12 +1821,6 @@ impl EditorElement { } } -struct HighlightedChunk<'a> { - chunk: &'a str, - style: Option, - is_tab: bool, -} - #[derive(Debug)] pub struct LineWithInvisibles { pub line: Line, diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 974af4bc24070ffb46870badbcb57915a802c1c7..3403790681dbc24ba71a69d45b3972a97157254b 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -1,5 +1,6 @@ use super::{Bias, DisplayPoint, DisplaySnapshot, SelectionGoal, ToDisplayPoint}; -use crate::{char_kind, CharKind, ToOffset, ToPoint}; +use crate::{char_kind, CharKind, EditorStyle, ToOffset, ToPoint}; +use gpui::{FontCache, TextLayoutCache, WindowContext}; use language::Point; use std::ops::Range; @@ -47,8 +48,20 @@ pub fn up( start: DisplayPoint, goal: SelectionGoal, preserve_column_at_start: bool, + font_cache: &FontCache, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, ) -> (DisplayPoint, SelectionGoal) { - up_by_rows(map, start, 1, goal, preserve_column_at_start) + up_by_rows( + map, + start, + 1, + goal, + preserve_column_at_start, + font_cache, + text_layout_cache, + editor_style, + ) } pub fn down( @@ -66,11 +79,14 @@ pub fn up_by_rows( row_count: u32, goal: SelectionGoal, preserve_column_at_start: bool, + font_cache: &FontCache, + text_layout_cache: &TextLayoutCache, + editor_style: &EditorStyle, ) -> (DisplayPoint, SelectionGoal) { - let mut goal_column = match goal { - SelectionGoal::Column(column) => column, - SelectionGoal::ColumnRange { end, .. } => end, - _ => map.column_to_chars(start.row(), start.column()), + let mut goal_x = match goal { + SelectionGoal::HorizontalPosition(x) => x, + SelectionGoal::HorizontalRange { end, .. } => end, + _ => map.x_for_point(start, font_cache, text_layout_cache, editor_style), }; let prev_row = start.row().saturating_sub(row_count); @@ -79,19 +95,27 @@ pub fn up_by_rows( Bias::Left, ); if point.row() < start.row() { - *point.column_mut() = map.column_from_chars(point.row(), goal_column); + *point.column_mut() = map + .column_for_x( + point.row(), + goal_x, + font_cache, + text_layout_cache, + editor_style, + ) + .unwrap_or(point.column()); } else if preserve_column_at_start { return (start, goal); } else { point = DisplayPoint::new(0, 0); - goal_column = 0; + goal_x = 0.0; } let mut clipped_point = map.clip_point(point, Bias::Left); if clipped_point.row() < point.row() { clipped_point = map.clip_point(point, Bias::Right); } - (clipped_point, SelectionGoal::Column(goal_column)) + (clipped_point, SelectionGoal::HorizontalPosition(goal_x)) } pub fn down_by_rows( @@ -692,6 +716,7 @@ mod tests { #[gpui::test] fn test_move_up_and_down_with_excerpts(cx: &mut gpui::AppContext) { + /* init_test(cx); let family_id = cx @@ -727,6 +752,7 @@ mod tests { cx.add_model(|cx| DisplayMap::new(multibuffer, font_id, 14.0, None, 2, 2, cx)); let snapshot = display_map.update(cx, |map, cx| map.snapshot(cx)); + assert_eq!(snapshot.text(), "\n\nabc\ndefg\n\n\nhijkl\nmn"); // Can't move up into the first excerpt's header @@ -737,7 +763,10 @@ mod tests { SelectionGoal::Column(2), false ), - (DisplayPoint::new(2, 0), SelectionGoal::Column(0)), + ( + DisplayPoint::new(2, 0), + SelectionGoal::HorizontalPosition(0.0) + ), ); assert_eq!( up( @@ -808,6 +837,7 @@ mod tests { ), (DisplayPoint::new(7, 2), SelectionGoal::Column(2)), ); + */ } fn init_test(cx: &mut gpui::AppContext) { diff --git a/crates/text/src/selection.rs b/crates/text/src/selection.rs index 60d5e2f1c437086c246f7e4c94ba6536e0f80840..38831f92c238f22c7f257763cc557856fdc3e604 100644 --- a/crates/text/src/selection.rs +++ b/crates/text/src/selection.rs @@ -5,6 +5,8 @@ use std::ops::Range; #[derive(Copy, Clone, Debug, PartialEq)] pub enum SelectionGoal { None, + HorizontalPosition(f32), + HorizontalRange { start: f32, end: f32 }, Column(u32), ColumnRange { start: u32, end: u32 }, } From e7badb38e96ffedbf9c24780d671b2d6c8cc7cdd Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Thu, 28 Sep 2023 20:35:06 -0600 Subject: [PATCH 03/41] Refactor to pass a TextLayoutDetails around --- crates/editor/src/display_map.rs | 44 +++++++++++------------------- crates/editor/src/editor.rs | 31 +++++---------------- crates/editor/src/movement.rs | 46 ++++++++++++++++++-------------- 3 files changed, 48 insertions(+), 73 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 3d13447fc23749b3daea4049c9b76ef11fb0ff2c..45b4c0abede8e3cfd592771a935de0d4512db96b 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -5,8 +5,8 @@ mod tab_map; mod wrap_map; use crate::{ - link_go_to_definition::InlayHighlight, Anchor, AnchorRangeExt, EditorStyle, InlayId, - MultiBuffer, MultiBufferSnapshot, ToOffset, ToPoint, + link_go_to_definition::InlayHighlight, movement::TextLayoutDetails, Anchor, AnchorRangeExt, + EditorStyle, InlayId, MultiBuffer, MultiBufferSnapshot, ToOffset, ToPoint, }; pub use block_map::{BlockMap, BlockPoint}; use collections::{BTreeMap, HashMap, HashSet}; @@ -565,9 +565,11 @@ impl DisplaySnapshot { fn layout_line_for_row( &self, display_row: u32, - font_cache: &FontCache, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, + TextLayoutDetails { + font_cache, + text_layout_cache, + editor_style, + }: &TextLayoutDetails, ) -> Line { let mut styles = Vec::new(); let mut line = String::new(); @@ -605,16 +607,9 @@ impl DisplaySnapshot { pub fn x_for_point( &self, display_point: DisplayPoint, - font_cache: &FontCache, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, + text_layout_details: &TextLayoutDetails, ) -> f32 { - let layout_line = self.layout_line_for_row( - display_point.row(), - font_cache, - text_layout_cache, - editor_style, - ); + let layout_line = self.layout_line_for_row(display_point.row(), text_layout_details); layout_line.x_for_index(display_point.column() as usize) } @@ -622,12 +617,9 @@ impl DisplaySnapshot { &self, display_row: u32, x_coordinate: f32, - font_cache: &FontCache, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, + text_layout_details: &TextLayoutDetails, ) -> Option { - let layout_line = - self.layout_line_for_row(display_row, font_cache, text_layout_cache, editor_style); + let layout_line = self.layout_line_for_row(display_row, text_layout_details); layout_line.index_for_x(x_coordinate).map(|c| c as u32) } @@ -1339,7 +1331,8 @@ pub mod tests { let window = cx.window.clone(); cx.update_window(window, |cx| { - let editor_style = editor.read(&cx).style(cx); + let text_layout_details = + editor.read_with(cx, |editor, cx| TextLayoutDetails::new(editor, cx)); let font_cache = cx.font_cache().clone(); @@ -1380,12 +1373,7 @@ pub mod tests { DisplayPoint::new(0, 7) ); - let x = snapshot.x_for_point( - DisplayPoint::new(1, 10), - cx.font_cache(), - cx.text_layout_cache(), - &editor_style, - ); + let x = snapshot.x_for_point(DisplayPoint::new(1, 10), &text_layout_details); dbg!(x); assert_eq!( movement::up( @@ -1393,9 +1381,7 @@ pub mod tests { DisplayPoint::new(1, 10), SelectionGoal::None, false, - cx.font_cache(), - cx.text_layout_cache(), - &editor_style, + &text_layout_details, ), ( DisplayPoint::new(0, 7), diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index bf1aa2e6b5784c4ff4d684d4a403e0bbc622de53..081d33c8a0bfde27cdd8e94712fc14cb0bb9bd35 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -71,6 +71,7 @@ use link_go_to_definition::{ }; use log::error; use lsp::LanguageServerId; +use movement::TextLayoutDetails; use multi_buffer::ToOffsetUtf16; pub use multi_buffer::{ Anchor, AnchorRangeExt, ExcerptId, ExcerptRange, MultiBuffer, MultiBufferSnapshot, ToOffset, @@ -5274,9 +5275,7 @@ impl Editor { return; } - let font_cache = cx.font_cache().clone(); - let text_layout_cache = cx.text_layout_cache().clone(); - let editor_style = self.style(cx); + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { let line_mode = s.line_mode; @@ -5289,9 +5288,7 @@ impl Editor { selection.start, selection.goal, false, - &font_cache, - &text_layout_cache, - &editor_style, + &text_layout_details, ); selection.collapse_to(cursor, goal); }); @@ -5320,9 +5317,7 @@ impl Editor { Autoscroll::fit() }; - let font_cache = cx.font_cache().clone(); - let text_layout = cx.text_layout_cache().clone(); - let editor_style = self.style(cx); + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(autoscroll), cx, |s| { let line_mode = s.line_mode; @@ -5336,9 +5331,7 @@ impl Editor { row_count, selection.goal, false, - &font_cache, - &text_layout, - &editor_style, + &text_layout_details, ); selection.collapse_to(cursor, goal); }); @@ -5346,20 +5339,10 @@ impl Editor { } pub fn select_up(&mut self, _: &SelectUp, cx: &mut ViewContext) { - let font_cache = cx.font_cache().clone(); - let text_layout = cx.text_layout_cache().clone(); - let editor_style = self.style(cx); + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { s.move_heads_with(|map, head, goal| { - movement::up( - map, - head, - goal, - false, - &font_cache, - &text_layout, - &editor_style, - ) + movement::up(map, head, goal, false, &text_layout_details) }) }) } diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 3403790681dbc24ba71a69d45b3972a97157254b..836d5dda2fc454432acb34fe1e3e3c3f61f9abc3 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -1,8 +1,8 @@ use super::{Bias, DisplayPoint, DisplaySnapshot, SelectionGoal, ToDisplayPoint}; -use crate::{char_kind, CharKind, EditorStyle, ToOffset, ToPoint}; -use gpui::{FontCache, TextLayoutCache, WindowContext}; +use crate::{char_kind, CharKind, Editor, EditorStyle, ToOffset, ToPoint}; +use gpui::{text_layout, FontCache, TextLayoutCache, WindowContext}; use language::Point; -use std::ops::Range; +use std::{ops::Range, sync::Arc}; #[derive(Debug, PartialEq)] pub enum FindRange { @@ -10,6 +10,24 @@ pub enum FindRange { MultiLine, } +/// TextLayoutDetails encompasses everything we need to move vertically +/// taking into account variable width characters. +pub struct TextLayoutDetails { + pub font_cache: Arc, + pub text_layout_cache: Arc, + pub editor_style: EditorStyle, +} + +impl TextLayoutDetails { + pub fn new(editor: &Editor, cx: &WindowContext) -> TextLayoutDetails { + TextLayoutDetails { + font_cache: cx.font_cache().clone(), + text_layout_cache: cx.text_layout_cache().clone(), + editor_style: editor.style(cx), + } + } +} + pub fn left(map: &DisplaySnapshot, mut point: DisplayPoint) -> DisplayPoint { if point.column() > 0 { *point.column_mut() -= 1; @@ -48,9 +66,7 @@ pub fn up( start: DisplayPoint, goal: SelectionGoal, preserve_column_at_start: bool, - font_cache: &FontCache, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { up_by_rows( map, @@ -58,9 +74,7 @@ pub fn up( 1, goal, preserve_column_at_start, - font_cache, - text_layout_cache, - editor_style, + text_layout_details, ) } @@ -79,14 +93,12 @@ pub fn up_by_rows( row_count: u32, goal: SelectionGoal, preserve_column_at_start: bool, - font_cache: &FontCache, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { let mut goal_x = match goal { SelectionGoal::HorizontalPosition(x) => x, SelectionGoal::HorizontalRange { end, .. } => end, - _ => map.x_for_point(start, font_cache, text_layout_cache, editor_style), + _ => map.x_for_point(start, text_layout_details), }; let prev_row = start.row().saturating_sub(row_count); @@ -96,13 +108,7 @@ pub fn up_by_rows( ); if point.row() < start.row() { *point.column_mut() = map - .column_for_x( - point.row(), - goal_x, - font_cache, - text_layout_cache, - editor_style, - ) + .column_for_x(point.row(), goal_x, text_layout_details) .unwrap_or(point.column()); } else if preserve_column_at_start { return (start, goal); From ef7e2c5d86208a8ecf738ae16c355188fa5d357a Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Thu, 28 Sep 2023 21:39:24 -0600 Subject: [PATCH 04/41] Get the project running! --- crates/editor/src/display_map.rs | 21 +- crates/editor/src/editor.rs | 37 +++- crates/editor/src/editor_tests.rs | 1 + crates/editor/src/movement.rs | 309 +++++++++++++++++----------- crates/vim/src/motion.rs | 32 ++- crates/vim/src/normal.rs | 34 +-- crates/vim/src/normal/change.rs | 31 ++- crates/vim/src/normal/delete.rs | 7 +- crates/vim/src/normal/paste.rs | 17 +- crates/vim/src/normal/substitute.rs | 26 ++- crates/vim/src/normal/yank.rs | 4 +- crates/vim/src/visual.rs | 15 +- 12 files changed, 352 insertions(+), 182 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 45b4c0abede8e3cfd592771a935de0d4512db96b..cebafbd651c17e7a80f0cac820f995cda0ed91c5 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -1392,19 +1392,28 @@ pub mod tests { movement::down( &snapshot, DisplayPoint::new(0, 7), - SelectionGoal::Column(10), - false + SelectionGoal::HorizontalPosition(x), + false, + &text_layout_details ), - (DisplayPoint::new(1, 10), SelectionGoal::Column(10)) + ( + DisplayPoint::new(1, 10), + SelectionGoal::HorizontalPosition(x) + ) ); + dbg!("starting down..."); assert_eq!( movement::down( &snapshot, DisplayPoint::new(1, 10), - SelectionGoal::Column(10), - false + SelectionGoal::HorizontalPosition(x), + false, + &text_layout_details ), - (DisplayPoint::new(2, 4), SelectionGoal::Column(10)) + ( + DisplayPoint::new(2, 4), + SelectionGoal::HorizontalPosition(x) + ) ); let ix = snapshot.buffer_snapshot.text().find("seven").unwrap(); diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 081d33c8a0bfde27cdd8e94712fc14cb0bb9bd35..e68b1f008f0526a4a7ce09bc195fa3db8e932fea 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -4988,6 +4988,7 @@ impl Editor { } pub fn transpose(&mut self, _: &Transpose, cx: &mut ViewContext) { + let text_layout_details = TextLayoutDetails::new(&self, cx); self.transact(cx, |this, cx| { let edits = this.change_selections(Some(Autoscroll::fit()), cx, |s| { let mut edits: Vec<(Range, String)> = Default::default(); @@ -5011,7 +5012,10 @@ impl Editor { *head.column_mut() += 1; head = display_map.clip_point(head, Bias::Right); - selection.collapse_to(head, SelectionGoal::Column(head.column())); + let goal = SelectionGoal::HorizontalPosition( + display_map.x_for_point(head, &text_layout_details), + ); + selection.collapse_to(head, goal); let transpose_start = display_map .buffer_snapshot @@ -5355,13 +5359,20 @@ impl Editor { return; } + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { if !selection.is_empty() && !line_mode { selection.goal = SelectionGoal::None; } - let (cursor, goal) = movement::down(map, selection.end, selection.goal, false); + let (cursor, goal) = movement::down( + map, + selection.end, + selection.goal, + false, + &text_layout_details, + ); selection.collapse_to(cursor, goal); }); }); @@ -5398,22 +5409,32 @@ impl Editor { Autoscroll::fit() }; + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(autoscroll), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { if !selection.is_empty() && !line_mode { selection.goal = SelectionGoal::None; } - let (cursor, goal) = - movement::down_by_rows(map, selection.end, row_count, selection.goal, false); + let (cursor, goal) = movement::down_by_rows( + map, + selection.end, + row_count, + selection.goal, + false, + &text_layout_details, + ); selection.collapse_to(cursor, goal); }); }); } pub fn select_down(&mut self, _: &SelectDown, cx: &mut ViewContext) { + let text_layout_details = TextLayoutDetails::new(&self, cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { - s.move_heads_with(|map, head, goal| movement::down(map, head, goal, false)) + s.move_heads_with(|map, head, goal| { + movement::down(map, head, goal, false, &text_layout_details) + }) }); } @@ -6286,6 +6307,7 @@ impl Editor { } pub fn toggle_comments(&mut self, action: &ToggleComments, cx: &mut ViewContext) { + let text_layout_details = TextLayoutDetails::new(&self, cx); self.transact(cx, |this, cx| { let mut selections = this.selections.all::(cx); let mut edits = Vec::new(); @@ -6528,7 +6550,10 @@ impl Editor { point.row += 1; point = snapshot.clip_point(point, Bias::Left); let display_point = point.to_display_point(display_snapshot); - (display_point, SelectionGoal::Column(display_point.column())) + let goal = SelectionGoal::HorizontalPosition( + display_snapshot.x_for_point(display_point, &text_layout_details), + ); + (display_point, goal) }) }); } diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs index dee27e0121256edaa38a90bb175d436eba768f96..affe9f60a2930167949d45c8762661358704fb9f 100644 --- a/crates/editor/src/editor_tests.rs +++ b/crates/editor/src/editor_tests.rs @@ -847,6 +847,7 @@ fn test_move_cursor(cx: &mut TestAppContext) { #[gpui::test] fn test_move_cursor_multibyte(cx: &mut TestAppContext) { + todo!(); init_test(cx, |_| {}); let view = cx diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 836d5dda2fc454432acb34fe1e3e3c3f61f9abc3..e2306a1b2daaa589896c07dbcffac034a46aea4f 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -83,8 +83,16 @@ pub fn down( start: DisplayPoint, goal: SelectionGoal, preserve_column_at_end: bool, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { - down_by_rows(map, start, 1, goal, preserve_column_at_end) + down_by_rows( + map, + start, + 1, + goal, + preserve_column_at_end, + text_layout_details, + ) } pub fn up_by_rows( @@ -130,29 +138,32 @@ pub fn down_by_rows( row_count: u32, goal: SelectionGoal, preserve_column_at_end: bool, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { - let mut goal_column = match goal { - SelectionGoal::Column(column) => column, - SelectionGoal::ColumnRange { end, .. } => end, - _ => map.column_to_chars(start.row(), start.column()), + let mut goal_x = match goal { + SelectionGoal::HorizontalPosition(x) => x, + SelectionGoal::HorizontalRange { end, .. } => end, + _ => map.x_for_point(start, text_layout_details), }; let new_row = start.row() + row_count; let mut point = map.clip_point(DisplayPoint::new(new_row, 0), Bias::Right); if point.row() > start.row() { - *point.column_mut() = map.column_from_chars(point.row(), goal_column); + *point.column_mut() = map + .column_for_x(point.row(), goal_x, text_layout_details) + .unwrap_or(map.line_len(point.row())); } else if preserve_column_at_end { return (start, goal); } else { point = map.max_point(); - goal_column = map.column_to_chars(point.row(), point.column()) + goal_x = map.x_for_point(point, text_layout_details) } let mut clipped_point = map.clip_point(point, Bias::Right); if clipped_point.row() > point.row() { clipped_point = map.clip_point(point, Bias::Left); } - (clipped_point, SelectionGoal::Column(goal_column)) + (clipped_point, SelectionGoal::HorizontalPosition(goal_x)) } pub fn line_beginning( @@ -426,9 +437,12 @@ pub fn split_display_range_by_lines( mod tests { use super::*; use crate::{ - display_map::Inlay, test::marked_display_snapshot, Buffer, DisplayMap, ExcerptRange, - InlayId, MultiBuffer, + display_map::Inlay, + test::{editor_test_context::EditorTestContext, marked_display_snapshot}, + Buffer, DisplayMap, ExcerptRange, InlayId, MultiBuffer, }; + use language::language_settings::AllLanguageSettings; + use project::Project; use settings::SettingsStore; use util::post_inc; @@ -721,129 +735,173 @@ mod tests { } #[gpui::test] - fn test_move_up_and_down_with_excerpts(cx: &mut gpui::AppContext) { - /* - init_test(cx); - - let family_id = cx - .font_cache() - .load_family(&["Helvetica"], &Default::default()) - .unwrap(); - let font_id = cx - .font_cache() - .select_font(family_id, &Default::default()) - .unwrap(); + async fn test_move_up_and_down_with_excerpts(cx: &mut gpui::TestAppContext) { + cx.update(|cx| { + init_test(cx); + }); - let buffer = - cx.add_model(|cx| Buffer::new(0, cx.model_id() as u64, "abc\ndefg\nhijkl\nmn")); - let multibuffer = cx.add_model(|cx| { - let mut multibuffer = MultiBuffer::new(0); - multibuffer.push_excerpts( - buffer.clone(), - [ - ExcerptRange { - context: Point::new(0, 0)..Point::new(1, 4), - primary: None, - }, - ExcerptRange { - context: Point::new(2, 0)..Point::new(3, 2), - primary: None, - }, - ], - cx, + let mut cx = EditorTestContext::new(cx).await; + let editor = cx.editor.clone(); + let window = cx.window.clone(); + cx.update_window(window, |cx| { + let text_layout_details = + editor.read_with(cx, |editor, cx| TextLayoutDetails::new(editor, cx)); + + let family_id = cx + .font_cache() + .load_family(&["Helvetica"], &Default::default()) + .unwrap(); + let font_id = cx + .font_cache() + .select_font(family_id, &Default::default()) + .unwrap(); + + let buffer = + cx.add_model(|cx| Buffer::new(0, cx.model_id() as u64, "abc\ndefg\nhijkl\nmn")); + let multibuffer = cx.add_model(|cx| { + let mut multibuffer = MultiBuffer::new(0); + multibuffer.push_excerpts( + buffer.clone(), + [ + ExcerptRange { + context: Point::new(0, 0)..Point::new(1, 4), + primary: None, + }, + ExcerptRange { + context: Point::new(2, 0)..Point::new(3, 2), + primary: None, + }, + ], + cx, + ); + multibuffer + }); + let display_map = + cx.add_model(|cx| DisplayMap::new(multibuffer, font_id, 14.0, None, 2, 2, cx)); + let snapshot = display_map.update(cx, |map, cx| map.snapshot(cx)); + + assert_eq!(snapshot.text(), "\n\nabc\ndefg\n\n\nhijkl\nmn"); + + let col_2_x = snapshot.x_for_point(DisplayPoint::new(2, 2), &text_layout_details); + + // Can't move up into the first excerpt's header + assert_eq!( + up( + &snapshot, + DisplayPoint::new(2, 2), + SelectionGoal::HorizontalPosition(col_2_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(2, 0), + SelectionGoal::HorizontalPosition(0.0) + ), + ); + assert_eq!( + up( + &snapshot, + DisplayPoint::new(2, 0), + SelectionGoal::None, + false, + &text_layout_details + ), + ( + DisplayPoint::new(2, 0), + SelectionGoal::HorizontalPosition(0.0) + ), ); - multibuffer - }); - let display_map = - cx.add_model(|cx| DisplayMap::new(multibuffer, font_id, 14.0, None, 2, 2, cx)); - let snapshot = display_map.update(cx, |map, cx| map.snapshot(cx)); + let col_4_x = snapshot.x_for_point(DisplayPoint::new(3, 4), &text_layout_details); - assert_eq!(snapshot.text(), "\n\nabc\ndefg\n\n\nhijkl\nmn"); + // Move up and down within first excerpt + assert_eq!( + up( + &snapshot, + DisplayPoint::new(3, 4), + SelectionGoal::HorizontalPosition(col_4_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(2, 3), + SelectionGoal::HorizontalPosition(col_4_x) + ), + ); + assert_eq!( + down( + &snapshot, + DisplayPoint::new(2, 3), + SelectionGoal::HorizontalPosition(col_4_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(3, 4), + SelectionGoal::HorizontalPosition(col_4_x) + ), + ); - // Can't move up into the first excerpt's header - assert_eq!( - up( - &snapshot, - DisplayPoint::new(2, 2), - SelectionGoal::Column(2), - false - ), - ( - DisplayPoint::new(2, 0), - SelectionGoal::HorizontalPosition(0.0) - ), - ); - assert_eq!( - up( - &snapshot, - DisplayPoint::new(2, 0), - SelectionGoal::None, - false - ), - (DisplayPoint::new(2, 0), SelectionGoal::Column(0)), - ); + let col_5_x = snapshot.x_for_point(DisplayPoint::new(6, 5), &text_layout_details); - // Move up and down within first excerpt - assert_eq!( - up( - &snapshot, - DisplayPoint::new(3, 4), - SelectionGoal::Column(4), - false - ), - (DisplayPoint::new(2, 3), SelectionGoal::Column(4)), - ); - assert_eq!( - down( - &snapshot, - DisplayPoint::new(2, 3), - SelectionGoal::Column(4), - false - ), - (DisplayPoint::new(3, 4), SelectionGoal::Column(4)), - ); + // Move up and down across second excerpt's header + assert_eq!( + up( + &snapshot, + DisplayPoint::new(6, 5), + SelectionGoal::HorizontalPosition(col_5_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(3, 4), + SelectionGoal::HorizontalPosition(col_5_x) + ), + ); + assert_eq!( + down( + &snapshot, + DisplayPoint::new(3, 4), + SelectionGoal::HorizontalPosition(col_5_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(6, 5), + SelectionGoal::HorizontalPosition(col_5_x) + ), + ); - // Move up and down across second excerpt's header - assert_eq!( - up( - &snapshot, - DisplayPoint::new(6, 5), - SelectionGoal::Column(5), - false - ), - (DisplayPoint::new(3, 4), SelectionGoal::Column(5)), - ); - assert_eq!( - down( - &snapshot, - DisplayPoint::new(3, 4), - SelectionGoal::Column(5), - false - ), - (DisplayPoint::new(6, 5), SelectionGoal::Column(5)), - ); + let max_point_x = snapshot.x_for_point(DisplayPoint::new(7, 2), &text_layout_details); - // Can't move down off the end - assert_eq!( - down( - &snapshot, - DisplayPoint::new(7, 0), - SelectionGoal::Column(0), - false - ), - (DisplayPoint::new(7, 2), SelectionGoal::Column(2)), - ); - assert_eq!( - down( - &snapshot, - DisplayPoint::new(7, 2), - SelectionGoal::Column(2), - false - ), - (DisplayPoint::new(7, 2), SelectionGoal::Column(2)), - ); - */ + // Can't move down off the end + assert_eq!( + down( + &snapshot, + DisplayPoint::new(7, 0), + SelectionGoal::HorizontalPosition(0.0), + false, + &text_layout_details + ), + ( + DisplayPoint::new(7, 2), + SelectionGoal::HorizontalPosition(max_point_x) + ), + ); + assert_eq!( + down( + &snapshot, + DisplayPoint::new(7, 2), + SelectionGoal::HorizontalPosition(max_point_x), + false, + &text_layout_details + ), + ( + DisplayPoint::new(7, 2), + SelectionGoal::HorizontalPosition(max_point_x) + ), + ); + }); } fn init_test(cx: &mut gpui::AppContext) { @@ -851,5 +909,6 @@ mod tests { theme::init((), cx); language::init(cx); crate::init(cx); + Project::init_settings(cx); } } diff --git a/crates/vim/src/motion.rs b/crates/vim/src/motion.rs index a197121626860c75b8e3b9101c03b30920960ed1..2f1e376c3e1e9e32eb60ae3a741caa50db833516 100644 --- a/crates/vim/src/motion.rs +++ b/crates/vim/src/motion.rs @@ -3,7 +3,7 @@ use std::cmp; use editor::{ char_kind, display_map::{DisplaySnapshot, FoldPoint, ToDisplayPoint}, - movement::{self, find_boundary, find_preceding_boundary, FindRange}, + movement::{self, find_boundary, find_preceding_boundary, FindRange, TextLayoutDetails}, Bias, CharKind, DisplayPoint, ToOffset, }; use gpui::{actions, impl_actions, AppContext, WindowContext}; @@ -361,6 +361,7 @@ impl Motion { point: DisplayPoint, goal: SelectionGoal, maybe_times: Option, + text_layout_details: &TextLayoutDetails, ) -> Option<(DisplayPoint, SelectionGoal)> { let times = maybe_times.unwrap_or(1); use Motion::*; @@ -373,13 +374,13 @@ impl Motion { } => down(map, point, goal, times), Down { display_lines: true, - } => down_display(map, point, goal, times), + } => down_display(map, point, goal, times, &text_layout_details), Up { display_lines: false, } => up(map, point, goal, times), Up { display_lines: true, - } => up_display(map, point, goal, times), + } => up_display(map, point, goal, times, &text_layout_details), Right => (right(map, point, times), SelectionGoal::None), NextWordStart { ignore_punctuation } => ( next_word_start(map, point, *ignore_punctuation, times), @@ -442,10 +443,15 @@ impl Motion { selection: &mut Selection, times: Option, expand_to_surrounding_newline: bool, + text_layout_details: &TextLayoutDetails, ) -> bool { - if let Some((new_head, goal)) = - self.move_point(map, selection.head(), selection.goal, times) - { + if let Some((new_head, goal)) = self.move_point( + map, + selection.head(), + selection.goal, + times, + &text_layout_details, + ) { selection.set_head(new_head, goal); if self.linewise() { @@ -566,9 +572,10 @@ fn down_display( mut point: DisplayPoint, mut goal: SelectionGoal, times: usize, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { for _ in 0..times { - (point, goal) = movement::down(map, point, goal, true); + (point, goal) = movement::down(map, point, goal, true, text_layout_details); } (point, goal) @@ -606,9 +613,10 @@ fn up_display( mut point: DisplayPoint, mut goal: SelectionGoal, times: usize, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { for _ in 0..times { - (point, goal) = movement::up(map, point, goal, true); + (point, goal) = movement::up(map, point, goal, true, &text_layout_details); } (point, goal) @@ -707,7 +715,7 @@ fn previous_word_start( point } -fn first_non_whitespace( +pub(crate) fn first_non_whitespace( map: &DisplaySnapshot, display_lines: bool, from: DisplayPoint, @@ -890,7 +898,11 @@ fn next_line_start(map: &DisplaySnapshot, point: DisplayPoint, times: usize) -> first_non_whitespace(map, false, correct_line) } -fn next_line_end(map: &DisplaySnapshot, mut point: DisplayPoint, times: usize) -> DisplayPoint { +pub(crate) fn next_line_end( + map: &DisplaySnapshot, + mut point: DisplayPoint, + times: usize, +) -> DisplayPoint { if times > 1 { point = down(map, point, SelectionGoal::None, times - 1).0; } diff --git a/crates/vim/src/normal.rs b/crates/vim/src/normal.rs index 36eab2c4c0be91eda40868a8b773237ce008ae87..9c93f19fc7e33512b621bece2c957e00240c799d 100644 --- a/crates/vim/src/normal.rs +++ b/crates/vim/src/normal.rs @@ -12,13 +12,13 @@ mod yank; use std::sync::Arc; use crate::{ - motion::{self, Motion}, + motion::{self, first_non_whitespace, next_line_end, right, Motion}, object::Object, state::{Mode, Operator}, Vim, }; use collections::HashSet; -use editor::scroll::autoscroll::Autoscroll; +use editor::{movement::TextLayoutDetails, scroll::autoscroll::Autoscroll}; use editor::{Bias, DisplayPoint}; use gpui::{actions, AppContext, ViewContext, WindowContext}; use language::SelectionGoal; @@ -177,10 +177,11 @@ pub(crate) fn move_cursor( cx: &mut WindowContext, ) { vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.change_selections(Some(Autoscroll::fit()), cx, |s| { s.move_cursors_with(|map, cursor, goal| { motion - .move_point(map, cursor, goal, times) + .move_point(map, cursor, goal, times, &text_layout_details) .unwrap_or((cursor, goal)) }) }) @@ -193,8 +194,8 @@ fn insert_after(_: &mut Workspace, _: &InsertAfter, cx: &mut ViewContext, cx: &m | Motion::StartOfLine { .. } ); vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.transact(cx, |editor, cx| { // We are swapping to insert mode anyway. Just set the line end clipping behavior now editor.set_clip_at_line_ends(false, cx); @@ -27,9 +28,15 @@ pub fn change_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &m s.move_with(|map, selection| { motion_succeeded |= if let Motion::NextWordStart { ignore_punctuation } = motion { - expand_changed_word_selection(map, selection, times, ignore_punctuation) + expand_changed_word_selection( + map, + selection, + times, + ignore_punctuation, + &text_layout_details, + ) } else { - motion.expand_selection(map, selection, times, false) + motion.expand_selection(map, selection, times, false, &text_layout_details) }; }); }); @@ -81,6 +88,7 @@ fn expand_changed_word_selection( selection: &mut Selection, times: Option, ignore_punctuation: bool, + text_layout_details: &TextLayoutDetails, ) -> bool { if times.is_none() || times.unwrap() == 1 { let scope = map @@ -103,11 +111,22 @@ fn expand_changed_word_selection( }); true } else { - Motion::NextWordStart { ignore_punctuation } - .expand_selection(map, selection, None, false) + Motion::NextWordStart { ignore_punctuation }.expand_selection( + map, + selection, + None, + false, + &text_layout_details, + ) } } else { - Motion::NextWordStart { ignore_punctuation }.expand_selection(map, selection, times, false) + Motion::NextWordStart { ignore_punctuation }.expand_selection( + map, + selection, + times, + false, + &text_layout_details, + ) } } diff --git a/crates/vim/src/normal/delete.rs b/crates/vim/src/normal/delete.rs index 517ece803365bef5734feac48013861add2cf1c2..1ad91ff308376aeecd01deee2eea89171aa701ea 100644 --- a/crates/vim/src/normal/delete.rs +++ b/crates/vim/src/normal/delete.rs @@ -1,12 +1,15 @@ use crate::{motion::Motion, object::Object, utils::copy_selections_content, Vim}; use collections::{HashMap, HashSet}; -use editor::{display_map::ToDisplayPoint, scroll::autoscroll::Autoscroll, Bias}; +use editor::{ + display_map::ToDisplayPoint, movement::TextLayoutDetails, scroll::autoscroll::Autoscroll, Bias, +}; use gpui::WindowContext; use language::Point; pub fn delete_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &mut WindowContext) { vim.stop_recording(); vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); let mut original_columns: HashMap<_, _> = Default::default(); @@ -14,7 +17,7 @@ pub fn delete_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &m s.move_with(|map, selection| { let original_head = selection.head(); original_columns.insert(selection.id, original_head.column()); - motion.expand_selection(map, selection, times, true); + motion.expand_selection(map, selection, times, true, &text_layout_details); // Motion::NextWordStart on an empty line should delete it. if let Motion::NextWordStart { diff --git a/crates/vim/src/normal/paste.rs b/crates/vim/src/normal/paste.rs index dda8dea1e480fcbf07a7df7f66b7846b27ee3d32..7cb5261c493eefb0b5bfea9b4ad12cd57eae0e51 100644 --- a/crates/vim/src/normal/paste.rs +++ b/crates/vim/src/normal/paste.rs @@ -1,8 +1,10 @@ use std::{borrow::Cow, cmp}; use editor::{ - display_map::ToDisplayPoint, movement, scroll::autoscroll::Autoscroll, ClipboardSelection, - DisplayPoint, + display_map::ToDisplayPoint, + movement::{self, TextLayoutDetails}, + scroll::autoscroll::Autoscroll, + ClipboardSelection, DisplayPoint, }; use gpui::{impl_actions, AppContext, ViewContext}; use language::{Bias, SelectionGoal}; @@ -30,6 +32,7 @@ fn paste(_: &mut Workspace, action: &Paste, cx: &mut ViewContext) { Vim::update(cx, |vim, cx| { vim.record_current_action(cx); vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); @@ -168,8 +171,14 @@ fn paste(_: &mut Workspace, action: &Paste, cx: &mut ViewContext) { let mut cursor = anchor.to_display_point(map); if *line_mode { if !before { - cursor = - movement::down(map, cursor, SelectionGoal::None, false).0; + cursor = movement::down( + map, + cursor, + SelectionGoal::None, + false, + &text_layout_details, + ) + .0; } cursor = movement::indented_line_beginning(map, cursor, true); } else if !is_multiline { diff --git a/crates/vim/src/normal/substitute.rs b/crates/vim/src/normal/substitute.rs index bb6e1abf92a0a687e32c7ee6a1e92787a1a82ba9..ddc937d03fd7f8af9e46081460004c930864d946 100644 --- a/crates/vim/src/normal/substitute.rs +++ b/crates/vim/src/normal/substitute.rs @@ -1,9 +1,13 @@ -use editor::movement; +use editor::movement::{self, TextLayoutDetails}; use gpui::{actions, AppContext, WindowContext}; use language::Point; use workspace::Workspace; -use crate::{motion::Motion, utils::copy_selections_content, Mode, Vim}; +use crate::{ + motion::{right, Motion}, + utils::copy_selections_content, + Mode, Vim, +}; actions!(vim, [Substitute, SubstituteLine]); @@ -32,10 +36,17 @@ pub fn substitute(vim: &mut Vim, count: Option, line_mode: bool, cx: &mut vim.update_active_editor(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); editor.transact(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.change_selections(None, cx, |s| { s.move_with(|map, selection| { if selection.start == selection.end { - Motion::Right.expand_selection(map, selection, count, true); + Motion::Right.expand_selection( + map, + selection, + count, + true, + &text_layout_details, + ); } if line_mode { // in Visual mode when the selection contains the newline at the end @@ -43,7 +54,13 @@ pub fn substitute(vim: &mut Vim, count: Option, line_mode: bool, cx: &mut if !selection.is_empty() && selection.end.column() == 0 { selection.end = movement::left(map, selection.end); } - Motion::CurrentLine.expand_selection(map, selection, None, false); + Motion::CurrentLine.expand_selection( + map, + selection, + None, + false, + &text_layout_details, + ); if let Some((point, _)) = (Motion::FirstNonWhitespace { display_lines: false, }) @@ -52,6 +69,7 @@ pub fn substitute(vim: &mut Vim, count: Option, line_mode: bool, cx: &mut selection.start, selection.goal, None, + &text_layout_details, ) { selection.start = point; } diff --git a/crates/vim/src/normal/yank.rs b/crates/vim/src/normal/yank.rs index 7212a865bd56a8eea01db7efd020d714b26a17ee..b50fcdf7ecbc9691b3e230fbf9d4ba2419c7bfd8 100644 --- a/crates/vim/src/normal/yank.rs +++ b/crates/vim/src/normal/yank.rs @@ -1,9 +1,11 @@ use crate::{motion::Motion, object::Object, utils::copy_selections_content, Vim}; use collections::HashMap; +use editor::movement::TextLayoutDetails; use gpui::WindowContext; pub fn yank_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &mut WindowContext) { vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); let mut original_positions: HashMap<_, _> = Default::default(); @@ -11,7 +13,7 @@ pub fn yank_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &mut s.move_with(|map, selection| { let original_position = (selection.head(), selection.goal); original_positions.insert(selection.id, original_position); - motion.expand_selection(map, selection, times, true); + motion.expand_selection(map, selection, times, true, &text_layout_details); }); }); copy_selections_content(editor, motion.linewise(), cx); diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index eac823de610280c24bb83003e007a28c29e81bf5..bec91007e3f7c8818076a5a5ee1cda2c41f81ab2 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -4,7 +4,7 @@ use std::{cmp, sync::Arc}; use collections::HashMap; use editor::{ display_map::{DisplaySnapshot, ToDisplayPoint}, - movement, + movement::{self, TextLayoutDetails}, scroll::autoscroll::Autoscroll, Bias, DisplayPoint, Editor, }; @@ -57,6 +57,7 @@ pub fn init(cx: &mut AppContext) { pub fn visual_motion(motion: Motion, times: Option, cx: &mut WindowContext) { Vim::update(cx, |vim, cx| { vim.update_active_editor(cx, |editor, cx| { + let text_layout_details = TextLayoutDetails::new(editor, cx); if vim.state().mode == Mode::VisualBlock && !matches!( motion, @@ -67,7 +68,7 @@ pub fn visual_motion(motion: Motion, times: Option, cx: &mut WindowContex { let is_up_or_down = matches!(motion, Motion::Up { .. } | Motion::Down { .. }); visual_block_motion(is_up_or_down, editor, cx, |map, point, goal| { - motion.move_point(map, point, goal, times) + motion.move_point(map, point, goal, times, &text_layout_details) }) } else { editor.change_selections(Some(Autoscroll::fit()), cx, |s| { @@ -89,9 +90,13 @@ pub fn visual_motion(motion: Motion, times: Option, cx: &mut WindowContex current_head = movement::left(map, selection.end) } - let Some((new_head, goal)) = - motion.move_point(map, current_head, selection.goal, times) - else { + let Some((new_head, goal)) = motion.move_point( + map, + current_head, + selection.goal, + times, + &text_layout_details, + ) else { return; }; From 002e2cc42c41f4fcb847061ce0b25721fa1895d5 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Thu, 28 Sep 2023 22:40:27 -0600 Subject: [PATCH 05/41] Round better for up/down --- crates/editor/src/display_map.rs | 4 ++-- crates/editor/src/movement.rs | 8 ++------ crates/gpui/src/text_layout.rs | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index cebafbd651c17e7a80f0cac820f995cda0ed91c5..424ff1518a7812be7be40f06fe8e413befe07657 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -618,9 +618,9 @@ impl DisplaySnapshot { display_row: u32, x_coordinate: f32, text_layout_details: &TextLayoutDetails, - ) -> Option { + ) -> u32 { let layout_line = self.layout_line_for_row(display_row, text_layout_details); - layout_line.index_for_x(x_coordinate).map(|c| c as u32) + layout_line.closest_index_for_x(x_coordinate) as u32 } // column_for_x(row, x) diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index e2306a1b2daaa589896c07dbcffac034a46aea4f..38cf5cd6c1a1eac49a6fbed2ba0b00312688fad0 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -115,9 +115,7 @@ pub fn up_by_rows( Bias::Left, ); if point.row() < start.row() { - *point.column_mut() = map - .column_for_x(point.row(), goal_x, text_layout_details) - .unwrap_or(point.column()); + *point.column_mut() = map.column_for_x(point.row(), goal_x, text_layout_details) } else if preserve_column_at_start { return (start, goal); } else { @@ -149,9 +147,7 @@ pub fn down_by_rows( let new_row = start.row() + row_count; let mut point = map.clip_point(DisplayPoint::new(new_row, 0), Bias::Right); if point.row() > start.row() { - *point.column_mut() = map - .column_for_x(point.row(), goal_x, text_layout_details) - .unwrap_or(map.line_len(point.row())); + *point.column_mut() = map.column_for_x(point.row(), goal_x, text_layout_details) } else if preserve_column_at_end { return (start, goal); } else { diff --git a/crates/gpui/src/text_layout.rs b/crates/gpui/src/text_layout.rs index 97f4b7a12d7a2aa95159dc049e57c6b5a4eb2e21..7fb87b10df2ce3baf822fbe5a6fddb4955e5f134 100644 --- a/crates/gpui/src/text_layout.rs +++ b/crates/gpui/src/text_layout.rs @@ -266,6 +266,8 @@ impl Line { self.layout.len == 0 } + /// index_for_x returns the character containing the given x coordinate. + /// (e.g. to handle a mouse-click) pub fn index_for_x(&self, x: f32) -> Option { if x >= self.layout.width { None @@ -281,6 +283,28 @@ impl Line { } } + /// closest_index_for_x returns the character boundary closest to the given x coordinate + /// (e.g. to handle aligning up/down arrow keys) + pub fn closest_index_for_x(&self, x: f32) -> usize { + let mut prev_index = 0; + let mut prev_x = 0.0; + + for run in self.layout.runs.iter() { + for glyph in run.glyphs.iter() { + if glyph.position.x() >= x { + if glyph.position.x() - x < x - prev_x { + return glyph.index; + } else { + return prev_index; + } + } + prev_index = glyph.index; + prev_x = glyph.position.x(); + } + } + prev_index + } + pub fn paint( &self, origin: Vector2F, From ab050d18901e47d41fdcd11da3e950688ae2e665 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Tue, 3 Oct 2023 19:10:01 -0600 Subject: [PATCH 06/41] Use Horizontal ranges everywhere --- crates/editor/src/display_map.rs | 49 +-------- crates/editor/src/editor.rs | 28 +++-- crates/editor/src/element.rs | 5 +- crates/editor/src/movement.rs | 6 +- crates/editor/src/selections_collection.rs | 23 ++-- crates/text/src/selection.rs | 4 +- crates/vim/src/motion.rs | 120 ++++++++++++--------- crates/vim/src/normal.rs | 32 ++++-- crates/vim/src/normal/substitute.rs | 6 +- crates/vim/src/test.rs | 56 ++++++++++ crates/vim/src/vim.rs | 2 +- crates/vim/src/visual.rs | 44 ++++++-- crates/vim/test_data/test_j.json | 3 + 13 files changed, 229 insertions(+), 149 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 424ff1518a7812be7be40f06fe8e413befe07657..0f2b5665c6eff88a631e630b4e6751d5e91b4431 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -15,7 +15,7 @@ use gpui::{ color::Color, fonts::{FontId, HighlightStyle, Underline}, text_layout::{Line, RunStyle}, - AppContext, Entity, FontCache, ModelContext, ModelHandle, TextLayoutCache, + Entity, ModelContext, ModelHandle, }; use inlay_map::InlayMap; use language::{ @@ -576,7 +576,6 @@ impl DisplaySnapshot { let range = display_row..display_row + 1; for chunk in self.highlighted_chunks(range, editor_style) { - dbg!(chunk.chunk); line.push_str(chunk.chunk); let text_style = if let Some(style) = chunk.style { @@ -600,7 +599,6 @@ impl DisplaySnapshot { )); } - dbg!(&line, &editor_style.text.font_size, &styles); text_layout_cache.layout_str(&line, editor_style.text.font_size, &styles) } @@ -623,49 +621,6 @@ impl DisplaySnapshot { layout_line.closest_index_for_x(x_coordinate) as u32 } - // column_for_x(row, x) - - fn point( - &self, - display_point: DisplayPoint, - text_layout_cache: &TextLayoutCache, - editor_style: &EditorStyle, - cx: &AppContext, - ) -> f32 { - let mut styles = Vec::new(); - let mut line = String::new(); - - let range = display_point.row()..display_point.row() + 1; - for chunk in self.highlighted_chunks(range, editor_style) { - dbg!(chunk.chunk); - line.push_str(chunk.chunk); - - let text_style = if let Some(style) = chunk.style { - editor_style - .text - .clone() - .highlight(style, cx.font_cache()) - .map(Cow::Owned) - .unwrap_or_else(|_| Cow::Borrowed(&editor_style.text)) - } else { - Cow::Borrowed(&editor_style.text) - }; - - styles.push(( - chunk.chunk.len(), - RunStyle { - font_id: text_style.font_id, - color: text_style.color, - underline: text_style.underline, - }, - )); - } - - dbg!(&line, &editor_style.text.font_size, &styles); - let layout_line = text_layout_cache.layout_str(&line, editor_style.text.font_size, &styles); - layout_line.x_for_index(display_point.column() as usize) - } - pub fn chars_at( &self, mut point: DisplayPoint, @@ -1374,7 +1329,6 @@ pub mod tests { ); let x = snapshot.x_for_point(DisplayPoint::new(1, 10), &text_layout_details); - dbg!(x); assert_eq!( movement::up( &snapshot, @@ -1401,7 +1355,6 @@ pub mod tests { SelectionGoal::HorizontalPosition(x) ) ); - dbg!("starting down..."); assert_eq!( movement::down( &snapshot, diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index e68b1f008f0526a4a7ce09bc195fa3db8e932fea..88db2f1dfe12ac069943c3675ee6d95197e7d186 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -48,9 +48,9 @@ use gpui::{ impl_actions, keymap_matcher::KeymapContext, platform::{CursorStyle, MouseButton}, - serde_json, text_layout, AnyElement, AnyViewHandle, AppContext, AsyncAppContext, ClipboardItem, - Element, Entity, ModelHandle, Subscription, Task, View, ViewContext, ViewHandle, - WeakViewHandle, WindowContext, + serde_json, AnyElement, AnyViewHandle, AppContext, AsyncAppContext, ClipboardItem, Element, + Entity, ModelHandle, Subscription, Task, View, ViewContext, ViewHandle, WeakViewHandle, + WindowContext, }; use highlight_matching_bracket::refresh_matching_bracket_highlights; use hover_popover::{hide_hover, HoverState}; @@ -5953,11 +5953,14 @@ impl Editor { fn add_selection(&mut self, above: bool, cx: &mut ViewContext) { let display_map = self.display_map.update(cx, |map, cx| map.snapshot(cx)); let mut selections = self.selections.all::(cx); + let text_layout_details = TextLayoutDetails::new(self, cx); let mut state = self.add_selections_state.take().unwrap_or_else(|| { let oldest_selection = selections.iter().min_by_key(|s| s.id).unwrap().clone(); let range = oldest_selection.display_range(&display_map).sorted(); - let columns = cmp::min(range.start.column(), range.end.column()) - ..cmp::max(range.start.column(), range.end.column()); + + let start_x = display_map.x_for_point(range.start, &text_layout_details); + let end_x = display_map.x_for_point(range.end, &text_layout_details); + let positions = start_x.min(end_x)..start_x.max(end_x); selections.clear(); let mut stack = Vec::new(); @@ -5965,8 +5968,9 @@ impl Editor { if let Some(selection) = self.selections.build_columnar_selection( &display_map, row, - &columns, + &positions, oldest_selection.reversed, + &text_layout_details, ) { stack.push(selection.id); selections.push(selection); @@ -5994,12 +5998,15 @@ impl Editor { let range = selection.display_range(&display_map).sorted(); debug_assert_eq!(range.start.row(), range.end.row()); let mut row = range.start.row(); - let columns = if let SelectionGoal::ColumnRange { start, end } = selection.goal + let positions = if let SelectionGoal::HorizontalRange { start, end } = + selection.goal { start..end } else { - cmp::min(range.start.column(), range.end.column()) - ..cmp::max(range.start.column(), range.end.column()) + let start_x = display_map.x_for_point(range.start, &text_layout_details); + let end_x = display_map.x_for_point(range.end, &text_layout_details); + + start_x.min(end_x)..start_x.max(end_x) }; while row != end_row { @@ -6012,8 +6019,9 @@ impl Editor { if let Some(new_selection) = self.selections.build_columnar_selection( &display_map, row, - &columns, + &positions, selection.reversed, + &text_layout_details, ) { state.stack.push(new_selection.id); if above { diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index 24cbadfd37927266e0395a4a8c9589bb9d0ab554..30015eb760d98474d420a41ca13bc71c691d95fd 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -22,7 +22,7 @@ use git::diff::DiffHunkStatus; use gpui::{ color::Color, elements::*, - fonts::{HighlightStyle, TextStyle, Underline}, + fonts::TextStyle, geometry::{ rect::RectF, vector::{vec2f, Vector2F}, @@ -37,8 +37,7 @@ use gpui::{ use itertools::Itertools; use json::json; use language::{ - language_settings::ShowWhitespaceSetting, Bias, CursorShape, DiagnosticSeverity, OffsetUtf16, - Selection, + language_settings::ShowWhitespaceSetting, Bias, CursorShape, OffsetUtf16, Selection, }; use project::{ project_settings::{GitGutterSetting, ProjectSettings}, diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 38cf5cd6c1a1eac49a6fbed2ba0b00312688fad0..7e75ae5e5d22c3f114c936c649d8be367d247f55 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -1,6 +1,6 @@ use super::{Bias, DisplayPoint, DisplaySnapshot, SelectionGoal, ToDisplayPoint}; use crate::{char_kind, CharKind, Editor, EditorStyle, ToOffset, ToPoint}; -use gpui::{text_layout, FontCache, TextLayoutCache, WindowContext}; +use gpui::{FontCache, TextLayoutCache, WindowContext}; use language::Point; use std::{ops::Range, sync::Arc}; @@ -105,7 +105,9 @@ pub fn up_by_rows( ) -> (DisplayPoint, SelectionGoal) { let mut goal_x = match goal { SelectionGoal::HorizontalPosition(x) => x, + SelectionGoal::WrappedHorizontalPosition((_, x)) => x, SelectionGoal::HorizontalRange { end, .. } => end, + SelectionGoal::WrappedHorizontalRange { end: (_, end), .. } => end, _ => map.x_for_point(start, text_layout_details), }; @@ -140,7 +142,9 @@ pub fn down_by_rows( ) -> (DisplayPoint, SelectionGoal) { let mut goal_x = match goal { SelectionGoal::HorizontalPosition(x) => x, + SelectionGoal::WrappedHorizontalPosition((_, x)) => x, SelectionGoal::HorizontalRange { end, .. } => end, + SelectionGoal::WrappedHorizontalRange { end: (_, end), .. } => end, _ => map.x_for_point(start, text_layout_details), }; diff --git a/crates/editor/src/selections_collection.rs b/crates/editor/src/selections_collection.rs index 6a21c898ef3617fc37bfe159be220cfe4360f884..2fa8ffe408742f6e1a3d9c80dd8d124b2ddaa1ec 100644 --- a/crates/editor/src/selections_collection.rs +++ b/crates/editor/src/selections_collection.rs @@ -1,6 +1,6 @@ use std::{ cell::Ref, - cmp, iter, mem, + iter, mem, ops::{Deref, DerefMut, Range, Sub}, sync::Arc, }; @@ -13,6 +13,7 @@ use util::post_inc; use crate::{ display_map::{DisplayMap, DisplaySnapshot, ToDisplayPoint}, + movement::TextLayoutDetails, Anchor, DisplayPoint, ExcerptId, MultiBuffer, MultiBufferSnapshot, SelectMode, ToOffset, }; @@ -305,23 +306,27 @@ impl SelectionsCollection { &mut self, display_map: &DisplaySnapshot, row: u32, - columns: &Range, + positions: &Range, reversed: bool, + text_layout_details: &TextLayoutDetails, ) -> Option> { - let is_empty = columns.start == columns.end; + let is_empty = positions.start == positions.end; let line_len = display_map.line_len(row); - if columns.start < line_len || (is_empty && columns.start == line_len) { - let start = DisplayPoint::new(row, columns.start); - let end = DisplayPoint::new(row, cmp::min(columns.end, line_len)); + + let start_col = display_map.column_for_x(row, positions.start, text_layout_details); + if start_col < line_len || (is_empty && start_col == line_len) { + let start = DisplayPoint::new(row, start_col); + let end_col = display_map.column_for_x(row, positions.end, text_layout_details); + let end = DisplayPoint::new(row, end_col); Some(Selection { id: post_inc(&mut self.next_selection_id), start: start.to_point(display_map), end: end.to_point(display_map), reversed, - goal: SelectionGoal::ColumnRange { - start: columns.start, - end: columns.end, + goal: SelectionGoal::HorizontalRange { + start: positions.start, + end: positions.end, }, }) } else { diff --git a/crates/text/src/selection.rs b/crates/text/src/selection.rs index 38831f92c238f22c7f257763cc557856fdc3e604..e127083112caef9c8762179fe2c8a2cac276388b 100644 --- a/crates/text/src/selection.rs +++ b/crates/text/src/selection.rs @@ -7,8 +7,8 @@ pub enum SelectionGoal { None, HorizontalPosition(f32), HorizontalRange { start: f32, end: f32 }, - Column(u32), - ColumnRange { start: u32, end: u32 }, + WrappedHorizontalPosition((u32, f32)), + WrappedHorizontalRange { start: (u32, f32), end: (u32, f32) }, } #[derive(Clone, Debug, PartialEq)] diff --git a/crates/vim/src/motion.rs b/crates/vim/src/motion.rs index 2f1e376c3e1e9e32eb60ae3a741caa50db833516..36514f8cc46e046934529a73a7aa185d7d108148 100644 --- a/crates/vim/src/motion.rs +++ b/crates/vim/src/motion.rs @@ -1,5 +1,3 @@ -use std::cmp; - use editor::{ char_kind, display_map::{DisplaySnapshot, FoldPoint, ToDisplayPoint}, @@ -371,13 +369,13 @@ impl Motion { Backspace => (backspace(map, point, times), SelectionGoal::None), Down { display_lines: false, - } => down(map, point, goal, times), + } => up_down_buffer_rows(map, point, goal, times as isize, &text_layout_details), Down { display_lines: true, } => down_display(map, point, goal, times, &text_layout_details), Up { display_lines: false, - } => up(map, point, goal, times), + } => up_down_buffer_rows(map, point, goal, 0 - times as isize, &text_layout_details), Up { display_lines: true, } => up_display(map, point, goal, times, &text_layout_details), @@ -536,35 +534,86 @@ fn backspace(map: &DisplaySnapshot, mut point: DisplayPoint, times: usize) -> Di point } -fn down( +pub(crate) fn start_of_relative_buffer_row( + map: &DisplaySnapshot, + point: DisplayPoint, + times: isize, +) -> DisplayPoint { + let start = map.display_point_to_fold_point(point, Bias::Left); + let target = start.row() as isize + times; + let new_row = (target.max(0) as u32).min(map.fold_snapshot.max_point().row()); + + map.clip_point( + map.fold_point_to_display_point( + map.fold_snapshot + .clip_point(FoldPoint::new(new_row, 0), Bias::Right), + ), + Bias::Right, + ) +} + +fn up_down_buffer_rows( map: &DisplaySnapshot, point: DisplayPoint, mut goal: SelectionGoal, - times: usize, + times: isize, + text_layout_details: &TextLayoutDetails, ) -> (DisplayPoint, SelectionGoal) { let start = map.display_point_to_fold_point(point, Bias::Left); + let begin_folded_line = map.fold_point_to_display_point( + map.fold_snapshot + .clip_point(FoldPoint::new(start.row(), 0), Bias::Left), + ); + let select_nth_wrapped_row = point.row() - begin_folded_line.row(); - let goal_column = match goal { - SelectionGoal::Column(column) => column, - SelectionGoal::ColumnRange { end, .. } => end, + let (goal_wrap, goal_x) = match goal { + SelectionGoal::WrappedHorizontalPosition((row, x)) => (row, x), + SelectionGoal::WrappedHorizontalRange { end: (row, x), .. } => (row, x), + SelectionGoal::HorizontalRange { end, .. } => (select_nth_wrapped_row, end), + SelectionGoal::HorizontalPosition(x) => (select_nth_wrapped_row, x), _ => { - goal = SelectionGoal::Column(start.column()); - start.column() + let x = map.x_for_point(point, text_layout_details); + goal = SelectionGoal::WrappedHorizontalPosition((select_nth_wrapped_row, x)); + (select_nth_wrapped_row, x) } }; - let new_row = cmp::min( - start.row() + times as u32, - map.fold_snapshot.max_point().row(), - ); - let new_col = cmp::min(goal_column, map.fold_snapshot.line_len(new_row)); - let point = map.fold_point_to_display_point( + let target = start.row() as isize + times; + let new_row = (target.max(0) as u32).min(map.fold_snapshot.max_point().row()); + + let mut begin_folded_line = map.fold_point_to_display_point( map.fold_snapshot - .clip_point(FoldPoint::new(new_row, new_col), Bias::Left), + .clip_point(FoldPoint::new(new_row, 0), Bias::Left), ); - // clip twice to "clip at end of line" - (map.clip_point(point, Bias::Left), goal) + let mut i = 0; + while i < goal_wrap && begin_folded_line.row() < map.max_point().row() { + let next_folded_line = DisplayPoint::new(begin_folded_line.row() + 1, 0); + if map + .display_point_to_fold_point(next_folded_line, Bias::Right) + .row() + == new_row + { + i += 1; + begin_folded_line = next_folded_line; + } else { + break; + } + } + + let new_col = if i == goal_wrap { + map.column_for_x(begin_folded_line.row(), goal_x, text_layout_details) + } else { + map.line_len(begin_folded_line.row()) + }; + + ( + map.clip_point( + DisplayPoint::new(begin_folded_line.row(), new_col), + Bias::Left, + ), + goal, + ) } fn down_display( @@ -581,33 +630,6 @@ fn down_display( (point, goal) } -pub(crate) fn up( - map: &DisplaySnapshot, - point: DisplayPoint, - mut goal: SelectionGoal, - times: usize, -) -> (DisplayPoint, SelectionGoal) { - let start = map.display_point_to_fold_point(point, Bias::Left); - - let goal_column = match goal { - SelectionGoal::Column(column) => column, - SelectionGoal::ColumnRange { end, .. } => end, - _ => { - goal = SelectionGoal::Column(start.column()); - start.column() - } - }; - - let new_row = start.row().saturating_sub(times as u32); - let new_col = cmp::min(goal_column, map.fold_snapshot.line_len(new_row)); - let point = map.fold_point_to_display_point( - map.fold_snapshot - .clip_point(FoldPoint::new(new_row, new_col), Bias::Left), - ); - - (map.clip_point(point, Bias::Left), goal) -} - fn up_display( map: &DisplaySnapshot, mut point: DisplayPoint, @@ -894,7 +916,7 @@ fn find_backward( } fn next_line_start(map: &DisplaySnapshot, point: DisplayPoint, times: usize) -> DisplayPoint { - let correct_line = down(map, point, SelectionGoal::None, times).0; + let correct_line = start_of_relative_buffer_row(map, point, times as isize); first_non_whitespace(map, false, correct_line) } @@ -904,7 +926,7 @@ pub(crate) fn next_line_end( times: usize, ) -> DisplayPoint { if times > 1 { - point = down(map, point, SelectionGoal::None, times - 1).0; + point = start_of_relative_buffer_row(map, point, times as isize - 1); } end_of_line(map, false, point) } diff --git a/crates/vim/src/normal.rs b/crates/vim/src/normal.rs index 9c93f19fc7e33512b621bece2c957e00240c799d..0e883cd758472a3395571521a8169efd6bae8270 100644 --- a/crates/vim/src/normal.rs +++ b/crates/vim/src/normal.rs @@ -194,9 +194,7 @@ fn insert_after(_: &mut Workspace, _: &InsertAfter, cx: &mut ViewContext, cx: &mut WindowContext) { Vim::update(cx, |vim, cx| { if vim.enabled && vim.state().mode == Mode::Normal && !newest.is_empty() { - if matches!(newest.goal, SelectionGoal::ColumnRange { .. }) { + if matches!(newest.goal, SelectionGoal::HorizontalRange { .. }) { vim.switch_mode(Mode::VisualBlock, false, cx); } else { vim.switch_mode(Mode::Visual, false, cx) diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index bec91007e3f7c8818076a5a5ee1cda2c41f81ab2..ac4c5478a1d924def738fbc711aee3c3bb12c509 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -140,17 +140,21 @@ pub fn visual_block_motion( SelectionGoal, ) -> Option<(DisplayPoint, SelectionGoal)>, ) { + let text_layout_details = TextLayoutDetails::new(editor, cx); editor.change_selections(Some(Autoscroll::fit()), cx, |s| { let map = &s.display_map(); let mut head = s.newest_anchor().head().to_display_point(map); let mut tail = s.oldest_anchor().tail().to_display_point(map); let (start, end) = match s.newest_anchor().goal { - SelectionGoal::ColumnRange { start, end } if preserve_goal => (start, end), - SelectionGoal::Column(start) if preserve_goal => (start, start + 1), - _ => (tail.column(), head.column()), + SelectionGoal::HorizontalRange { start, end } if preserve_goal => (start, end), + SelectionGoal::HorizontalPosition(start) if preserve_goal => (start, start + 10.0), + _ => ( + map.x_for_point(tail, &text_layout_details), + map.x_for_point(head, &text_layout_details), + ), }; - let goal = SelectionGoal::ColumnRange { start, end }; + let goal = SelectionGoal::HorizontalRange { start, end }; let was_reversed = tail.column() > head.column(); if !was_reversed && !preserve_goal { @@ -172,21 +176,39 @@ pub fn visual_block_motion( head = movement::saturating_right(map, head) } - let columns = if is_reversed { - head.column()..tail.column() + let positions = if is_reversed { + map.x_for_point(head, &text_layout_details)..map.x_for_point(tail, &text_layout_details) } else if head.column() == tail.column() { - head.column()..(head.column() + 1) + map.x_for_point(head, &text_layout_details) + ..map.x_for_point(head, &text_layout_details) + 10.0 } else { - tail.column()..head.column() + map.x_for_point(tail, &text_layout_details)..map.x_for_point(head, &text_layout_details) }; let mut selections = Vec::new(); let mut row = tail.row(); loop { - let start = map.clip_point(DisplayPoint::new(row, columns.start), Bias::Left); - let end = map.clip_point(DisplayPoint::new(row, columns.end), Bias::Left); - if columns.start <= map.line_len(row) { + let start = map.clip_point( + DisplayPoint::new( + row, + map.column_for_x(row, positions.start, &text_layout_details), + ), + Bias::Left, + ); + let end = map.clip_point( + DisplayPoint::new( + row, + map.column_for_x(row, positions.end, &text_layout_details), + ), + Bias::Left, + ); + if positions.start + <= map.x_for_point( + DisplayPoint::new(row, map.line_len(row)), + &text_layout_details, + ) + { let selection = Selection { id: s.new_selection_id(), start: start.to_point(map), diff --git a/crates/vim/test_data/test_j.json b/crates/vim/test_data/test_j.json index 64aaf65ef8960f66253d788fa97f4b4306bbee70..703f69d22c4c780d76ec9cd049aca0164b9cd624 100644 --- a/crates/vim/test_data/test_j.json +++ b/crates/vim/test_data/test_j.json @@ -1,3 +1,6 @@ +{"Put":{"state":"aaˇaa\n😃😃"}} +{"Key":"j"} +{"Get":{"state":"aaaa\n😃ˇ😃","mode":"Normal"}} {"Put":{"state":"ˇThe quick brown\nfox jumps"}} {"Key":"j"} {"Get":{"state":"The quick brown\nˇfox jumps","mode":"Normal"}} From 40755961ea0d0f3e252e2248b027fdbf21a2f659 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Mon, 16 Oct 2023 11:54:32 -0400 Subject: [PATCH 07/41] added initial template outline --- crates/ai/src/ai.rs | 1 + crates/ai/src/templates.rs | 76 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 crates/ai/src/templates.rs diff --git a/crates/ai/src/ai.rs b/crates/ai/src/ai.rs index 5256a6a6432907dd22c30d6a03e492a46fef77df..04e9e14536c16d80de133940db6723349e8d2371 100644 --- a/crates/ai/src/ai.rs +++ b/crates/ai/src/ai.rs @@ -1,2 +1,3 @@ pub mod completion; pub mod embedding; +pub mod templates; diff --git a/crates/ai/src/templates.rs b/crates/ai/src/templates.rs new file mode 100644 index 0000000000000000000000000000000000000000..d9771ce56964dcc782eb4c3aaa8a5ec6c8a76cd3 --- /dev/null +++ b/crates/ai/src/templates.rs @@ -0,0 +1,76 @@ +use std::fmt::Write; + +pub struct PromptCodeSnippet { + path: Option, + language_name: Option, + content: String, +} + +enum PromptFileType { + Text, + Code, +} + +#[derive(Default)] +struct PromptArguments { + pub language_name: Option, + pub project_name: Option, + pub snippets: Vec, +} + +impl PromptArguments { + pub fn get_file_type(&self) -> PromptFileType { + if self + .language_name + .as_ref() + .and_then(|name| Some(!["Markdown", "Plain Text"].contains(&name.as_str()))) + .unwrap_or(true) + { + PromptFileType::Code + } else { + PromptFileType::Text + } + } +} + +trait PromptTemplate { + fn generate(args: PromptArguments) -> String; +} + +struct EngineerPreamble {} + +impl PromptTemplate for EngineerPreamble { + fn generate(args: PromptArguments) -> String { + let mut prompt = String::new(); + + match args.get_file_type() { + PromptFileType::Code => { + writeln!( + prompt, + "You are an expert {} engineer.", + args.language_name.unwrap_or("".to_string()) + ) + .unwrap(); + } + PromptFileType::Text => { + writeln!(prompt, "You are an expert engineer.").unwrap(); + } + } + + if let Some(project_name) = args.project_name { + writeln!( + prompt, + "You are currently working inside the '{project_name}' in Zed the code editor." + ) + .unwrap(); + } + + prompt + } +} + +struct RepositorySnippets {} + +impl PromptTemplate for RepositorySnippets { + fn generate(args: PromptArguments) -> String {} +} From 500af6d7754adf1a60f245200271e4dd40d7fb8f Mon Sep 17 00:00:00 2001 From: KCaverly Date: Mon, 16 Oct 2023 18:47:10 -0400 Subject: [PATCH 08/41] progress on prompt chains --- Cargo.lock | 1 + crates/ai/Cargo.toml | 1 + crates/ai/src/prompts.rs | 149 ++++++++++++++++++ crates/ai/src/templates.rs | 76 --------- crates/ai/src/templates/base.rs | 112 +++++++++++++ crates/ai/src/templates/mod.rs | 3 + crates/ai/src/templates/preamble.rs | 34 ++++ crates/ai/src/templates/repository_context.rs | 49 ++++++ 8 files changed, 349 insertions(+), 76 deletions(-) create mode 100644 crates/ai/src/prompts.rs delete mode 100644 crates/ai/src/templates.rs create mode 100644 crates/ai/src/templates/base.rs create mode 100644 crates/ai/src/templates/mod.rs create mode 100644 crates/ai/src/templates/preamble.rs create mode 100644 crates/ai/src/templates/repository_context.rs diff --git a/Cargo.lock b/Cargo.lock index cd9dee0bda70dd5180b1f59201dd69feeebba1b6..9938c5d2fa328fb9834db781e32f02eb3b39e5fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,7 @@ dependencies = [ "futures 0.3.28", "gpui", "isahc", + "language", "lazy_static", "log", "matrixmultiply", diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml index 542d7f422fe8c1eaec7d10bf59cb5ccaa2d65ca3..b24c4e5ece5b02eac003a6c18f186faa1eaef7ef 100644 --- a/crates/ai/Cargo.toml +++ b/crates/ai/Cargo.toml @@ -11,6 +11,7 @@ doctest = false [dependencies] gpui = { path = "../gpui" } util = { path = "../util" } +language = { path = "../language" } async-trait.workspace = true anyhow.workspace = true futures.workspace = true diff --git a/crates/ai/src/prompts.rs b/crates/ai/src/prompts.rs new file mode 100644 index 0000000000000000000000000000000000000000..6d2c0629fa08e2d464adc5bf1d48c44659da8545 --- /dev/null +++ b/crates/ai/src/prompts.rs @@ -0,0 +1,149 @@ +use gpui::{AsyncAppContext, ModelHandle}; +use language::{Anchor, Buffer}; +use std::{fmt::Write, ops::Range, path::PathBuf}; + +pub struct PromptCodeSnippet { + path: Option, + language_name: Option, + content: String, +} + +impl PromptCodeSnippet { + pub fn new(buffer: ModelHandle, range: Range, cx: &AsyncAppContext) -> Self { + let (content, language_name, file_path) = buffer.read_with(cx, |buffer, _| { + let snapshot = buffer.snapshot(); + let content = snapshot.text_for_range(range.clone()).collect::(); + + let language_name = buffer + .language() + .and_then(|language| Some(language.name().to_string())); + + let file_path = buffer + .file() + .and_then(|file| Some(file.path().to_path_buf())); + + (content, language_name, file_path) + }); + + PromptCodeSnippet { + path: file_path, + language_name, + content, + } + } +} + +impl ToString for PromptCodeSnippet { + fn to_string(&self) -> String { + let path = self + .path + .as_ref() + .and_then(|path| Some(path.to_string_lossy().to_string())) + .unwrap_or("".to_string()); + let language_name = self.language_name.clone().unwrap_or("".to_string()); + let content = self.content.clone(); + + format!("The below code snippet may be relevant from file: {path}\n```{language_name}\n{content}\n```") + } +} + +enum PromptFileType { + Text, + Code, +} + +#[derive(Default)] +struct PromptArguments { + pub language_name: Option, + pub project_name: Option, + pub snippets: Vec, + pub model_name: String, +} + +impl PromptArguments { + pub fn get_file_type(&self) -> PromptFileType { + if self + .language_name + .as_ref() + .and_then(|name| Some(!["Markdown", "Plain Text"].contains(&name.as_str()))) + .unwrap_or(true) + { + PromptFileType::Code + } else { + PromptFileType::Text + } + } +} + +trait PromptTemplate { + fn generate(args: PromptArguments, max_token_length: Option) -> String; +} + +struct EngineerPreamble {} + +impl PromptTemplate for EngineerPreamble { + fn generate(args: PromptArguments, max_token_length: Option) -> String { + let mut prompt = String::new(); + + match args.get_file_type() { + PromptFileType::Code => { + writeln!( + prompt, + "You are an expert {} engineer.", + args.language_name.unwrap_or("".to_string()) + ) + .unwrap(); + } + PromptFileType::Text => { + writeln!(prompt, "You are an expert engineer.").unwrap(); + } + } + + if let Some(project_name) = args.project_name { + writeln!( + prompt, + "You are currently working inside the '{project_name}' in Zed the code editor." + ) + .unwrap(); + } + + prompt + } +} + +struct RepositorySnippets {} + +impl PromptTemplate for RepositorySnippets { + fn generate(args: PromptArguments, max_token_length: Option) -> String { + const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; + let mut template = "You are working inside a large repository, here are a few code snippets that may be useful"; + let mut prompt = String::new(); + + if let Ok(encoding) = tiktoken_rs::get_bpe_from_model(args.model_name.as_str()) { + let default_token_count = + tiktoken_rs::model::get_context_size(args.model_name.as_str()); + let mut remaining_token_count = max_token_length.unwrap_or(default_token_count); + + for snippet in args.snippets { + let mut snippet_prompt = template.to_string(); + let content = snippet.to_string(); + writeln!(snippet_prompt, "{content}").unwrap(); + + let token_count = encoding + .encode_with_special_tokens(snippet_prompt.as_str()) + .len(); + if token_count <= remaining_token_count { + if token_count < MAXIMUM_SNIPPET_TOKEN_COUNT { + writeln!(prompt, "{snippet_prompt}").unwrap(); + remaining_token_count -= token_count; + template = ""; + } + } else { + break; + } + } + } + + prompt + } +} diff --git a/crates/ai/src/templates.rs b/crates/ai/src/templates.rs deleted file mode 100644 index d9771ce56964dcc782eb4c3aaa8a5ec6c8a76cd3..0000000000000000000000000000000000000000 --- a/crates/ai/src/templates.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::fmt::Write; - -pub struct PromptCodeSnippet { - path: Option, - language_name: Option, - content: String, -} - -enum PromptFileType { - Text, - Code, -} - -#[derive(Default)] -struct PromptArguments { - pub language_name: Option, - pub project_name: Option, - pub snippets: Vec, -} - -impl PromptArguments { - pub fn get_file_type(&self) -> PromptFileType { - if self - .language_name - .as_ref() - .and_then(|name| Some(!["Markdown", "Plain Text"].contains(&name.as_str()))) - .unwrap_or(true) - { - PromptFileType::Code - } else { - PromptFileType::Text - } - } -} - -trait PromptTemplate { - fn generate(args: PromptArguments) -> String; -} - -struct EngineerPreamble {} - -impl PromptTemplate for EngineerPreamble { - fn generate(args: PromptArguments) -> String { - let mut prompt = String::new(); - - match args.get_file_type() { - PromptFileType::Code => { - writeln!( - prompt, - "You are an expert {} engineer.", - args.language_name.unwrap_or("".to_string()) - ) - .unwrap(); - } - PromptFileType::Text => { - writeln!(prompt, "You are an expert engineer.").unwrap(); - } - } - - if let Some(project_name) = args.project_name { - writeln!( - prompt, - "You are currently working inside the '{project_name}' in Zed the code editor." - ) - .unwrap(); - } - - prompt - } -} - -struct RepositorySnippets {} - -impl PromptTemplate for RepositorySnippets { - fn generate(args: PromptArguments) -> String {} -} diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs new file mode 100644 index 0000000000000000000000000000000000000000..3d8479e51253f8aa7f8157104fb9ed2220cfe3f2 --- /dev/null +++ b/crates/ai/src/templates/base.rs @@ -0,0 +1,112 @@ +use std::cmp::Reverse; + +use crate::templates::repository_context::PromptCodeSnippet; + +pub(crate) enum PromptFileType { + Text, + Code, +} + +#[derive(Default)] +pub struct PromptArguments { + pub model_name: String, + pub language_name: Option, + pub project_name: Option, + pub snippets: Vec, + pub reserved_tokens: usize, +} + +impl PromptArguments { + pub(crate) fn get_file_type(&self) -> PromptFileType { + if self + .language_name + .as_ref() + .and_then(|name| Some(!["Markdown", "Plain Text"].contains(&name.as_str()))) + .unwrap_or(true) + { + PromptFileType::Code + } else { + PromptFileType::Text + } + } +} + +pub trait PromptTemplate { + fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String; +} + +#[repr(i8)] +#[derive(PartialEq, Eq, PartialOrd, Ord)] +pub enum PromptPriority { + Low, + Medium, + High, +} + +pub struct PromptChain { + args: PromptArguments, + templates: Vec<(PromptPriority, Box)>, +} + +impl PromptChain { + pub fn new( + args: PromptArguments, + templates: Vec<(PromptPriority, Box)>, + ) -> Self { + // templates.sort_by(|a, b| a.0.cmp(&b.0)); + + PromptChain { args, templates } + } + + pub fn generate(&self, truncate: bool) -> anyhow::Result { + // Argsort based on Prompt Priority + let mut sorted_indices = (0..self.templates.len()).collect::>(); + sorted_indices.sort_by_key(|&i| Reverse(&self.templates[i].0)); + + println!("{:?}", sorted_indices); + + let mut prompts = Vec::new(); + for (_, template) in &self.templates { + prompts.push(template.generate(&self.args, None)); + } + + anyhow::Ok(prompts.join("\n")) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + #[test] + pub fn test_prompt_chain() { + struct TestPromptTemplate {} + impl PromptTemplate for TestPromptTemplate { + fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { + "This is a test prompt template".to_string() + } + } + + struct TestLowPriorityTemplate {} + impl PromptTemplate for TestLowPriorityTemplate { + fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { + "This is a low priority test prompt template".to_string() + } + } + + let args = PromptArguments { + model_name: "gpt-4".to_string(), + ..Default::default() + }; + + let templates: Vec<(PromptPriority, Box)> = vec![ + (PromptPriority::High, Box::new(TestPromptTemplate {})), + (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), + ]; + let chain = PromptChain::new(args, templates); + + let prompt = chain.generate(false); + println!("{:?}", prompt); + panic!(); + } +} diff --git a/crates/ai/src/templates/mod.rs b/crates/ai/src/templates/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..62cb600eca4fb641265a3937ec5bf8f1e8c2d9c2 --- /dev/null +++ b/crates/ai/src/templates/mod.rs @@ -0,0 +1,3 @@ +pub mod base; +pub mod preamble; +pub mod repository_context; diff --git a/crates/ai/src/templates/preamble.rs b/crates/ai/src/templates/preamble.rs new file mode 100644 index 0000000000000000000000000000000000000000..b1d33f885ea493f9488894154fe262e7ce177edc --- /dev/null +++ b/crates/ai/src/templates/preamble.rs @@ -0,0 +1,34 @@ +use crate::templates::base::{PromptArguments, PromptFileType, PromptTemplate}; +use std::fmt::Write; + +struct EngineerPreamble {} + +impl PromptTemplate for EngineerPreamble { + fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { + let mut prompt = String::new(); + + match args.get_file_type() { + PromptFileType::Code => { + writeln!( + prompt, + "You are an expert {} engineer.", + args.language_name.clone().unwrap_or("".to_string()) + ) + .unwrap(); + } + PromptFileType::Text => { + writeln!(prompt, "You are an expert engineer.").unwrap(); + } + } + + if let Some(project_name) = args.project_name.clone() { + writeln!( + prompt, + "You are currently working inside the '{project_name}' in Zed the code editor." + ) + .unwrap(); + } + + prompt + } +} diff --git a/crates/ai/src/templates/repository_context.rs b/crates/ai/src/templates/repository_context.rs new file mode 100644 index 0000000000000000000000000000000000000000..f9c2253c654de8da59ffa99ec07a12233b121d01 --- /dev/null +++ b/crates/ai/src/templates/repository_context.rs @@ -0,0 +1,49 @@ +use std::{ops::Range, path::PathBuf}; + +use gpui::{AsyncAppContext, ModelHandle}; +use language::{Anchor, Buffer}; + +pub struct PromptCodeSnippet { + path: Option, + language_name: Option, + content: String, +} + +impl PromptCodeSnippet { + pub fn new(buffer: ModelHandle, range: Range, cx: &AsyncAppContext) -> Self { + let (content, language_name, file_path) = buffer.read_with(cx, |buffer, _| { + let snapshot = buffer.snapshot(); + let content = snapshot.text_for_range(range.clone()).collect::(); + + let language_name = buffer + .language() + .and_then(|language| Some(language.name().to_string())); + + let file_path = buffer + .file() + .and_then(|file| Some(file.path().to_path_buf())); + + (content, language_name, file_path) + }); + + PromptCodeSnippet { + path: file_path, + language_name, + content, + } + } +} + +impl ToString for PromptCodeSnippet { + fn to_string(&self) -> String { + let path = self + .path + .as_ref() + .and_then(|path| Some(path.to_string_lossy().to_string())) + .unwrap_or("".to_string()); + let language_name = self.language_name.clone().unwrap_or("".to_string()); + let content = self.content.clone(); + + format!("The below code snippet may be relevant from file: {path}\n```{language_name}\n{content}\n```") + } +} From ad92fe49c7deeb098dcd442bc996602630f4f056 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Tue, 17 Oct 2023 11:58:45 -0400 Subject: [PATCH 09/41] implement initial concept of prompt chain --- crates/ai/src/templates/base.rs | 229 +++++++++++++++++++++++++++++--- 1 file changed, 208 insertions(+), 21 deletions(-) diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index 3d8479e51253f8aa7f8157104fb9ed2220cfe3f2..74a4c424ae93b46da34d3f5493f6e2363b31c2f5 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -1,15 +1,25 @@ -use std::cmp::Reverse; +use std::fmt::Write; +use std::{cmp::Reverse, sync::Arc}; + +use util::ResultExt; use crate::templates::repository_context::PromptCodeSnippet; +pub trait LanguageModel { + fn name(&self) -> String; + fn count_tokens(&self, content: &str) -> usize; + fn truncate(&self, content: &str, length: usize) -> String; + fn capacity(&self) -> usize; +} + pub(crate) enum PromptFileType { Text, Code, } -#[derive(Default)] +// TODO: Set this up to manage for defaults well pub struct PromptArguments { - pub model_name: String, + pub model: Arc, pub language_name: Option, pub project_name: Option, pub snippets: Vec, @@ -32,7 +42,11 @@ impl PromptArguments { } pub trait PromptTemplate { - fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String; + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)>; } #[repr(i8)] @@ -53,24 +67,52 @@ impl PromptChain { args: PromptArguments, templates: Vec<(PromptPriority, Box)>, ) -> Self { - // templates.sort_by(|a, b| a.0.cmp(&b.0)); - PromptChain { args, templates } } - pub fn generate(&self, truncate: bool) -> anyhow::Result { + pub fn generate(&self, truncate: bool) -> anyhow::Result<(String, usize)> { // Argsort based on Prompt Priority + let seperator = "\n"; + let seperator_tokens = self.args.model.count_tokens(seperator); let mut sorted_indices = (0..self.templates.len()).collect::>(); sorted_indices.sort_by_key(|&i| Reverse(&self.templates[i].0)); - println!("{:?}", sorted_indices); - let mut prompts = Vec::new(); - for (_, template) in &self.templates { - prompts.push(template.generate(&self.args, None)); + + // If Truncate + let mut tokens_outstanding = if truncate { + Some(self.args.model.capacity() - self.args.reserved_tokens) + } else { + None + }; + + for idx in sorted_indices { + let (_, template) = &self.templates[idx]; + if let Some((template_prompt, prompt_token_count)) = + template.generate(&self.args, tokens_outstanding).log_err() + { + println!( + "GENERATED PROMPT ({:?}): {:?}", + &prompt_token_count, &template_prompt + ); + if template_prompt != "" { + prompts.push(template_prompt); + + if let Some(remaining_tokens) = tokens_outstanding { + let new_tokens = prompt_token_count + seperator_tokens; + tokens_outstanding = if remaining_tokens > new_tokens { + Some(remaining_tokens - new_tokens) + } else { + Some(0) + }; + } + } + } } - anyhow::Ok(prompts.join("\n")) + let full_prompt = prompts.join(seperator); + let total_token_count = self.args.model.count_tokens(&full_prompt); + anyhow::Ok((prompts.join(seperator), total_token_count)) } } @@ -82,21 +124,81 @@ pub(crate) mod tests { pub fn test_prompt_chain() { struct TestPromptTemplate {} impl PromptTemplate for TestPromptTemplate { - fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { - "This is a test prompt template".to_string() + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + let mut content = "This is a test prompt template".to_string(); + + let mut token_count = args.model.count_tokens(&content); + if let Some(max_token_length) = max_token_length { + if token_count > max_token_length { + content = args.model.truncate(&content, max_token_length); + token_count = max_token_length; + } + } + + anyhow::Ok((content, token_count)) } } struct TestLowPriorityTemplate {} impl PromptTemplate for TestLowPriorityTemplate { - fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { - "This is a low priority test prompt template".to_string() + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + let mut content = "This is a low priority test prompt template".to_string(); + + let mut token_count = args.model.count_tokens(&content); + if let Some(max_token_length) = max_token_length { + if token_count > max_token_length { + content = args.model.truncate(&content, max_token_length); + token_count = max_token_length; + } + } + + anyhow::Ok((content, token_count)) } } + #[derive(Clone)] + struct DummyLanguageModel { + capacity: usize, + } + + impl DummyLanguageModel { + fn set_capacity(&mut self, capacity: usize) { + self.capacity = capacity + } + } + + impl LanguageModel for DummyLanguageModel { + fn name(&self) -> String { + "dummy".to_string() + } + fn count_tokens(&self, content: &str) -> usize { + content.chars().collect::>().len() + } + fn truncate(&self, content: &str, length: usize) -> String { + content.chars().collect::>()[..length] + .into_iter() + .collect::() + } + fn capacity(&self) -> usize { + self.capacity + } + } + + let model: Arc = Arc::new(DummyLanguageModel { capacity: 100 }); let args = PromptArguments { - model_name: "gpt-4".to_string(), - ..Default::default() + model: model.clone(), + language_name: None, + project_name: None, + snippets: Vec::new(), + reserved_tokens: 0, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -105,8 +207,93 @@ pub(crate) mod tests { ]; let chain = PromptChain::new(args, templates); - let prompt = chain.generate(false); - println!("{:?}", prompt); - panic!(); + let (prompt, token_count) = chain.generate(false).unwrap(); + + assert_eq!( + prompt, + "This is a test prompt template\nThis is a low priority test prompt template" + .to_string() + ); + + assert_eq!(model.count_tokens(&prompt), token_count); + + // Testing with Truncation Off + // Should ignore capacity and return all prompts + let model: Arc = Arc::new(DummyLanguageModel { capacity: 20 }); + let args = PromptArguments { + model: model.clone(), + language_name: None, + project_name: None, + snippets: Vec::new(), + reserved_tokens: 0, + }; + + let templates: Vec<(PromptPriority, Box)> = vec![ + (PromptPriority::High, Box::new(TestPromptTemplate {})), + (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), + ]; + let chain = PromptChain::new(args, templates); + + let (prompt, token_count) = chain.generate(false).unwrap(); + + assert_eq!( + prompt, + "This is a test prompt template\nThis is a low priority test prompt template" + .to_string() + ); + + assert_eq!(model.count_tokens(&prompt), token_count); + + // Testing with Truncation Off + // Should ignore capacity and return all prompts + let capacity = 20; + let model: Arc = Arc::new(DummyLanguageModel { capacity }); + let args = PromptArguments { + model: model.clone(), + language_name: None, + project_name: None, + snippets: Vec::new(), + reserved_tokens: 0, + }; + + let templates: Vec<(PromptPriority, Box)> = vec![ + (PromptPriority::High, Box::new(TestPromptTemplate {})), + (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), + (PromptPriority::Low, Box::new(TestLowPriorityTemplate {})), + ]; + let chain = PromptChain::new(args, templates); + + let (prompt, token_count) = chain.generate(true).unwrap(); + + assert_eq!(prompt, "This is a test promp".to_string()); + assert_eq!(token_count, capacity); + + // Change Ordering of Prompts Based on Priority + let capacity = 120; + let reserved_tokens = 10; + let model: Arc = Arc::new(DummyLanguageModel { capacity }); + let args = PromptArguments { + model: model.clone(), + language_name: None, + project_name: None, + snippets: Vec::new(), + reserved_tokens, + }; + let templates: Vec<(PromptPriority, Box)> = vec![ + (PromptPriority::Medium, Box::new(TestPromptTemplate {})), + (PromptPriority::High, Box::new(TestLowPriorityTemplate {})), + (PromptPriority::Low, Box::new(TestLowPriorityTemplate {})), + ]; + let chain = PromptChain::new(args, templates); + + let (prompt, token_count) = chain.generate(true).unwrap(); + println!("TOKEN COUNT: {:?}", token_count); + + assert_eq!( + prompt, + "This is a low priority test prompt template\nThis is a test prompt template\nThis is a low priority test prompt " + .to_string() + ); + assert_eq!(token_count, capacity - reserved_tokens); } } From a874a09b7e3b30696dad650bc997342fd8a53a61 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Tue, 17 Oct 2023 16:21:03 -0400 Subject: [PATCH 10/41] added openai language model tokenizer and LanguageModel trait --- crates/ai/src/ai.rs | 1 + crates/ai/src/models.rs | 49 ++++++++++++++++++++++++++ crates/ai/src/templates/base.rs | 54 ++++++++++++----------------- crates/ai/src/templates/preamble.rs | 42 +++++++++++++++------- 4 files changed, 102 insertions(+), 44 deletions(-) create mode 100644 crates/ai/src/models.rs diff --git a/crates/ai/src/ai.rs b/crates/ai/src/ai.rs index 04e9e14536c16d80de133940db6723349e8d2371..f168c157934f6b70be775f7e17e9ba27ef9b3103 100644 --- a/crates/ai/src/ai.rs +++ b/crates/ai/src/ai.rs @@ -1,3 +1,4 @@ pub mod completion; pub mod embedding; +pub mod models; pub mod templates; diff --git a/crates/ai/src/models.rs b/crates/ai/src/models.rs new file mode 100644 index 0000000000000000000000000000000000000000..4fe96d44f33f10ad1e6ee8572a8cceb02fca8fd4 --- /dev/null +++ b/crates/ai/src/models.rs @@ -0,0 +1,49 @@ +use anyhow::anyhow; +use tiktoken_rs::CoreBPE; +use util::ResultExt; + +pub trait LanguageModel { + fn name(&self) -> String; + fn count_tokens(&self, content: &str) -> anyhow::Result; + fn truncate(&self, content: &str, length: usize) -> anyhow::Result; + fn capacity(&self) -> anyhow::Result; +} + +struct OpenAILanguageModel { + name: String, + bpe: Option, +} + +impl OpenAILanguageModel { + pub fn load(model_name: String) -> Self { + let bpe = tiktoken_rs::get_bpe_from_model(&model_name).log_err(); + OpenAILanguageModel { + name: model_name, + bpe, + } + } +} + +impl LanguageModel for OpenAILanguageModel { + fn name(&self) -> String { + self.name.clone() + } + fn count_tokens(&self, content: &str) -> anyhow::Result { + if let Some(bpe) = &self.bpe { + anyhow::Ok(bpe.encode_with_special_tokens(content).len()) + } else { + Err(anyhow!("bpe for open ai model was not retrieved")) + } + } + fn truncate(&self, content: &str, length: usize) -> anyhow::Result { + if let Some(bpe) = &self.bpe { + let tokens = bpe.encode_with_special_tokens(content); + bpe.decode(tokens[..length].to_vec()) + } else { + Err(anyhow!("bpe for open ai model was not retrieved")) + } + } + fn capacity(&self) -> anyhow::Result { + anyhow::Ok(tiktoken_rs::model::get_context_size(&self.name)) + } +} diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index 74a4c424ae93b46da34d3f5493f6e2363b31c2f5..b5f9da3586f7793e601ca8f5bf7a3158da5949c8 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -1,17 +1,11 @@ -use std::fmt::Write; -use std::{cmp::Reverse, sync::Arc}; +use std::cmp::Reverse; +use std::sync::Arc; use util::ResultExt; +use crate::models::LanguageModel; use crate::templates::repository_context::PromptCodeSnippet; -pub trait LanguageModel { - fn name(&self) -> String; - fn count_tokens(&self, content: &str) -> usize; - fn truncate(&self, content: &str, length: usize) -> String; - fn capacity(&self) -> usize; -} - pub(crate) enum PromptFileType { Text, Code, @@ -73,7 +67,7 @@ impl PromptChain { pub fn generate(&self, truncate: bool) -> anyhow::Result<(String, usize)> { // Argsort based on Prompt Priority let seperator = "\n"; - let seperator_tokens = self.args.model.count_tokens(seperator); + let seperator_tokens = self.args.model.count_tokens(seperator)?; let mut sorted_indices = (0..self.templates.len()).collect::>(); sorted_indices.sort_by_key(|&i| Reverse(&self.templates[i].0)); @@ -81,7 +75,7 @@ impl PromptChain { // If Truncate let mut tokens_outstanding = if truncate { - Some(self.args.model.capacity() - self.args.reserved_tokens) + Some(self.args.model.capacity()? - self.args.reserved_tokens) } else { None }; @@ -111,7 +105,7 @@ impl PromptChain { } let full_prompt = prompts.join(seperator); - let total_token_count = self.args.model.count_tokens(&full_prompt); + let total_token_count = self.args.model.count_tokens(&full_prompt)?; anyhow::Ok((prompts.join(seperator), total_token_count)) } } @@ -131,10 +125,10 @@ pub(crate) mod tests { ) -> anyhow::Result<(String, usize)> { let mut content = "This is a test prompt template".to_string(); - let mut token_count = args.model.count_tokens(&content); + let mut token_count = args.model.count_tokens(&content)?; if let Some(max_token_length) = max_token_length { if token_count > max_token_length { - content = args.model.truncate(&content, max_token_length); + content = args.model.truncate(&content, max_token_length)?; token_count = max_token_length; } } @@ -152,10 +146,10 @@ pub(crate) mod tests { ) -> anyhow::Result<(String, usize)> { let mut content = "This is a low priority test prompt template".to_string(); - let mut token_count = args.model.count_tokens(&content); + let mut token_count = args.model.count_tokens(&content)?; if let Some(max_token_length) = max_token_length { if token_count > max_token_length { - content = args.model.truncate(&content, max_token_length); + content = args.model.truncate(&content, max_token_length)?; token_count = max_token_length; } } @@ -169,26 +163,22 @@ pub(crate) mod tests { capacity: usize, } - impl DummyLanguageModel { - fn set_capacity(&mut self, capacity: usize) { - self.capacity = capacity - } - } - impl LanguageModel for DummyLanguageModel { fn name(&self) -> String { "dummy".to_string() } - fn count_tokens(&self, content: &str) -> usize { - content.chars().collect::>().len() + fn count_tokens(&self, content: &str) -> anyhow::Result { + anyhow::Ok(content.chars().collect::>().len()) } - fn truncate(&self, content: &str, length: usize) -> String { - content.chars().collect::>()[..length] - .into_iter() - .collect::() + fn truncate(&self, content: &str, length: usize) -> anyhow::Result { + anyhow::Ok( + content.chars().collect::>()[..length] + .into_iter() + .collect::(), + ) } - fn capacity(&self) -> usize { - self.capacity + fn capacity(&self) -> anyhow::Result { + anyhow::Ok(self.capacity) } } @@ -215,7 +205,7 @@ pub(crate) mod tests { .to_string() ); - assert_eq!(model.count_tokens(&prompt), token_count); + assert_eq!(model.count_tokens(&prompt).unwrap(), token_count); // Testing with Truncation Off // Should ignore capacity and return all prompts @@ -242,7 +232,7 @@ pub(crate) mod tests { .to_string() ); - assert_eq!(model.count_tokens(&prompt), token_count); + assert_eq!(model.count_tokens(&prompt).unwrap(), token_count); // Testing with Truncation Off // Should ignore capacity and return all prompts diff --git a/crates/ai/src/templates/preamble.rs b/crates/ai/src/templates/preamble.rs index b1d33f885ea493f9488894154fe262e7ce177edc..f395dbf8beeabde2a703214cc0426900908be990 100644 --- a/crates/ai/src/templates/preamble.rs +++ b/crates/ai/src/templates/preamble.rs @@ -4,31 +4,49 @@ use std::fmt::Write; struct EngineerPreamble {} impl PromptTemplate for EngineerPreamble { - fn generate(&self, args: &PromptArguments, max_token_length: Option) -> String { - let mut prompt = String::new(); + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + let mut prompts = Vec::new(); match args.get_file_type() { PromptFileType::Code => { - writeln!( - prompt, + prompts.push(format!( "You are an expert {} engineer.", args.language_name.clone().unwrap_or("".to_string()) - ) - .unwrap(); + )); } PromptFileType::Text => { - writeln!(prompt, "You are an expert engineer.").unwrap(); + prompts.push("You are an expert engineer.".to_string()); } } if let Some(project_name) = args.project_name.clone() { - writeln!( - prompt, + prompts.push(format!( "You are currently working inside the '{project_name}' in Zed the code editor." - ) - .unwrap(); + )); } - prompt + if let Some(mut remaining_tokens) = max_token_length { + let mut prompt = String::new(); + let mut total_count = 0; + for prompt_piece in prompts { + let prompt_token_count = + args.model.count_tokens(&prompt_piece)? + args.model.count_tokens("\n")?; + if remaining_tokens > prompt_token_count { + writeln!(prompt, "{prompt_piece}").unwrap(); + remaining_tokens -= prompt_token_count; + total_count += prompt_token_count; + } + } + + anyhow::Ok((prompt, total_count)) + } else { + let prompt = prompts.join("\n"); + let token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count)) + } } } From 02853bbd606dc87a638bd2ca01a5232203069499 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Tue, 17 Oct 2023 17:29:07 -0400 Subject: [PATCH 11/41] added prompt template for repository context --- crates/ai/src/models.rs | 8 +- crates/ai/src/prompts.rs | 149 ------------------ crates/ai/src/templates/preamble.rs | 6 +- crates/ai/src/templates/repository_context.rs | 47 +++++- crates/assistant/src/assistant_panel.rs | 22 ++- crates/assistant/src/prompts.rs | 87 ++++------ 6 files changed, 96 insertions(+), 223 deletions(-) delete mode 100644 crates/ai/src/prompts.rs diff --git a/crates/ai/src/models.rs b/crates/ai/src/models.rs index 4fe96d44f33f10ad1e6ee8572a8cceb02fca8fd4..69e73e9b56ec7db7023983a67f5ee994c97c5725 100644 --- a/crates/ai/src/models.rs +++ b/crates/ai/src/models.rs @@ -9,16 +9,16 @@ pub trait LanguageModel { fn capacity(&self) -> anyhow::Result; } -struct OpenAILanguageModel { +pub struct OpenAILanguageModel { name: String, bpe: Option, } impl OpenAILanguageModel { - pub fn load(model_name: String) -> Self { - let bpe = tiktoken_rs::get_bpe_from_model(&model_name).log_err(); + pub fn load(model_name: &str) -> Self { + let bpe = tiktoken_rs::get_bpe_from_model(model_name).log_err(); OpenAILanguageModel { - name: model_name, + name: model_name.to_string(), bpe, } } diff --git a/crates/ai/src/prompts.rs b/crates/ai/src/prompts.rs deleted file mode 100644 index 6d2c0629fa08e2d464adc5bf1d48c44659da8545..0000000000000000000000000000000000000000 --- a/crates/ai/src/prompts.rs +++ /dev/null @@ -1,149 +0,0 @@ -use gpui::{AsyncAppContext, ModelHandle}; -use language::{Anchor, Buffer}; -use std::{fmt::Write, ops::Range, path::PathBuf}; - -pub struct PromptCodeSnippet { - path: Option, - language_name: Option, - content: String, -} - -impl PromptCodeSnippet { - pub fn new(buffer: ModelHandle, range: Range, cx: &AsyncAppContext) -> Self { - let (content, language_name, file_path) = buffer.read_with(cx, |buffer, _| { - let snapshot = buffer.snapshot(); - let content = snapshot.text_for_range(range.clone()).collect::(); - - let language_name = buffer - .language() - .and_then(|language| Some(language.name().to_string())); - - let file_path = buffer - .file() - .and_then(|file| Some(file.path().to_path_buf())); - - (content, language_name, file_path) - }); - - PromptCodeSnippet { - path: file_path, - language_name, - content, - } - } -} - -impl ToString for PromptCodeSnippet { - fn to_string(&self) -> String { - let path = self - .path - .as_ref() - .and_then(|path| Some(path.to_string_lossy().to_string())) - .unwrap_or("".to_string()); - let language_name = self.language_name.clone().unwrap_or("".to_string()); - let content = self.content.clone(); - - format!("The below code snippet may be relevant from file: {path}\n```{language_name}\n{content}\n```") - } -} - -enum PromptFileType { - Text, - Code, -} - -#[derive(Default)] -struct PromptArguments { - pub language_name: Option, - pub project_name: Option, - pub snippets: Vec, - pub model_name: String, -} - -impl PromptArguments { - pub fn get_file_type(&self) -> PromptFileType { - if self - .language_name - .as_ref() - .and_then(|name| Some(!["Markdown", "Plain Text"].contains(&name.as_str()))) - .unwrap_or(true) - { - PromptFileType::Code - } else { - PromptFileType::Text - } - } -} - -trait PromptTemplate { - fn generate(args: PromptArguments, max_token_length: Option) -> String; -} - -struct EngineerPreamble {} - -impl PromptTemplate for EngineerPreamble { - fn generate(args: PromptArguments, max_token_length: Option) -> String { - let mut prompt = String::new(); - - match args.get_file_type() { - PromptFileType::Code => { - writeln!( - prompt, - "You are an expert {} engineer.", - args.language_name.unwrap_or("".to_string()) - ) - .unwrap(); - } - PromptFileType::Text => { - writeln!(prompt, "You are an expert engineer.").unwrap(); - } - } - - if let Some(project_name) = args.project_name { - writeln!( - prompt, - "You are currently working inside the '{project_name}' in Zed the code editor." - ) - .unwrap(); - } - - prompt - } -} - -struct RepositorySnippets {} - -impl PromptTemplate for RepositorySnippets { - fn generate(args: PromptArguments, max_token_length: Option) -> String { - const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; - let mut template = "You are working inside a large repository, here are a few code snippets that may be useful"; - let mut prompt = String::new(); - - if let Ok(encoding) = tiktoken_rs::get_bpe_from_model(args.model_name.as_str()) { - let default_token_count = - tiktoken_rs::model::get_context_size(args.model_name.as_str()); - let mut remaining_token_count = max_token_length.unwrap_or(default_token_count); - - for snippet in args.snippets { - let mut snippet_prompt = template.to_string(); - let content = snippet.to_string(); - writeln!(snippet_prompt, "{content}").unwrap(); - - let token_count = encoding - .encode_with_special_tokens(snippet_prompt.as_str()) - .len(); - if token_count <= remaining_token_count { - if token_count < MAXIMUM_SNIPPET_TOKEN_COUNT { - writeln!(prompt, "{snippet_prompt}").unwrap(); - remaining_token_count -= token_count; - template = ""; - } - } else { - break; - } - } - } - - prompt - } -} diff --git a/crates/ai/src/templates/preamble.rs b/crates/ai/src/templates/preamble.rs index f395dbf8beeabde2a703214cc0426900908be990..5834fa1b21b2011fbbc82d781493c4e4e523b685 100644 --- a/crates/ai/src/templates/preamble.rs +++ b/crates/ai/src/templates/preamble.rs @@ -1,7 +1,7 @@ use crate::templates::base::{PromptArguments, PromptFileType, PromptTemplate}; use std::fmt::Write; -struct EngineerPreamble {} +pub struct EngineerPreamble {} impl PromptTemplate for EngineerPreamble { fn generate( @@ -14,8 +14,8 @@ impl PromptTemplate for EngineerPreamble { match args.get_file_type() { PromptFileType::Code => { prompts.push(format!( - "You are an expert {} engineer.", - args.language_name.clone().unwrap_or("".to_string()) + "You are an expert {}engineer.", + args.language_name.clone().unwrap_or("".to_string()) + " " )); } PromptFileType::Text => { diff --git a/crates/ai/src/templates/repository_context.rs b/crates/ai/src/templates/repository_context.rs index f9c2253c654de8da59ffa99ec07a12233b121d01..7dd1647c440a228b5fc2c8317fe35e0931d4c1a9 100644 --- a/crates/ai/src/templates/repository_context.rs +++ b/crates/ai/src/templates/repository_context.rs @@ -1,8 +1,11 @@ +use crate::templates::base::{PromptArguments, PromptTemplate}; +use std::fmt::Write; use std::{ops::Range, path::PathBuf}; use gpui::{AsyncAppContext, ModelHandle}; use language::{Anchor, Buffer}; +#[derive(Clone)] pub struct PromptCodeSnippet { path: Option, language_name: Option, @@ -17,7 +20,7 @@ impl PromptCodeSnippet { let language_name = buffer .language() - .and_then(|language| Some(language.name().to_string())); + .and_then(|language| Some(language.name().to_string().to_lowercase())); let file_path = buffer .file() @@ -47,3 +50,45 @@ impl ToString for PromptCodeSnippet { format!("The below code snippet may be relevant from file: {path}\n```{language_name}\n{content}\n```") } } + +pub struct RepositoryContext {} + +impl PromptTemplate for RepositoryContext { + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; + let mut template = "You are working inside a large repository, here are a few code snippets that may be useful."; + let mut prompt = String::new(); + + let mut remaining_tokens = max_token_length.clone(); + let seperator_token_length = args.model.count_tokens("\n")?; + for snippet in &args.snippets { + let mut snippet_prompt = template.to_string(); + let content = snippet.to_string(); + writeln!(snippet_prompt, "{content}").unwrap(); + + let token_count = args.model.count_tokens(&snippet_prompt)?; + if token_count <= MAXIMUM_SNIPPET_TOKEN_COUNT { + if let Some(tokens_left) = remaining_tokens { + if tokens_left >= token_count { + writeln!(prompt, "{snippet_prompt}").unwrap(); + remaining_tokens = if tokens_left >= (token_count + seperator_token_length) + { + Some(tokens_left - token_count - seperator_token_length) + } else { + Some(0) + }; + } + } else { + writeln!(prompt, "{snippet_prompt}").unwrap(); + } + } + } + + let total_token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, total_token_count)) + } +} diff --git a/crates/assistant/src/assistant_panel.rs b/crates/assistant/src/assistant_panel.rs index e8edf70498c14e7324073b732cae6b887f3131f9..06de5c135fdf535e4f253ec6f92ef2d449f769a2 100644 --- a/crates/assistant/src/assistant_panel.rs +++ b/crates/assistant/src/assistant_panel.rs @@ -1,12 +1,15 @@ use crate::{ assistant_settings::{AssistantDockPosition, AssistantSettings, OpenAIModel}, codegen::{self, Codegen, CodegenKind}, - prompts::{generate_content_prompt, PromptCodeSnippet}, + prompts::generate_content_prompt, MessageId, MessageMetadata, MessageStatus, Role, SavedConversation, SavedConversationMetadata, SavedMessage, }; -use ai::completion::{ - stream_completion, OpenAICompletionProvider, OpenAIRequest, RequestMessage, OPENAI_API_URL, +use ai::{ + completion::{ + stream_completion, OpenAICompletionProvider, OpenAIRequest, RequestMessage, OPENAI_API_URL, + }, + templates::repository_context::PromptCodeSnippet, }; use anyhow::{anyhow, Result}; use chrono::{DateTime, Local}; @@ -668,14 +671,7 @@ impl AssistantPanel { let snippets = cx.spawn(|_, cx| async move { let mut snippets = Vec::new(); for result in search_results.await { - snippets.push(PromptCodeSnippet::new(result, &cx)); - - // snippets.push(result.buffer.read_with(&cx, |buffer, _| { - // buffer - // .snapshot() - // .text_for_range(result.range) - // .collect::() - // })); + snippets.push(PromptCodeSnippet::new(result.buffer, result.range, &cx)); } snippets }); @@ -717,7 +713,8 @@ impl AssistantPanel { } cx.spawn(|_, mut cx| async move { - let prompt = prompt.await; + // I Don't know if we want to return a ? here. + let prompt = prompt.await?; messages.push(RequestMessage { role: Role::User, @@ -729,6 +726,7 @@ impl AssistantPanel { stream: true, }; codegen.update(&mut cx, |codegen, cx| codegen.start(request, cx)); + anyhow::Ok(()) }) .detach(); } diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index 7aafe75920b351e7244f14858036a4aa9af64f6f..e33a6e4022e87c99b899b0f492d0c25e1514cb4f 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -1,61 +1,15 @@ use crate::codegen::CodegenKind; -use gpui::AsyncAppContext; +use ai::models::{LanguageModel, OpenAILanguageModel}; +use ai::templates::base::{PromptArguments, PromptChain, PromptPriority, PromptTemplate}; +use ai::templates::preamble::EngineerPreamble; +use ai::templates::repository_context::{PromptCodeSnippet, RepositoryContext}; use language::{BufferSnapshot, OffsetRangeExt, ToOffset}; -use semantic_index::SearchResult; use std::cmp::{self, Reverse}; use std::fmt::Write; use std::ops::Range; -use std::path::PathBuf; +use std::sync::Arc; use tiktoken_rs::ChatCompletionRequestMessage; -pub struct PromptCodeSnippet { - path: Option, - language_name: Option, - content: String, -} - -impl PromptCodeSnippet { - pub fn new(search_result: SearchResult, cx: &AsyncAppContext) -> Self { - let (content, language_name, file_path) = - search_result.buffer.read_with(cx, |buffer, _| { - let snapshot = buffer.snapshot(); - let content = snapshot - .text_for_range(search_result.range.clone()) - .collect::(); - - let language_name = buffer - .language() - .and_then(|language| Some(language.name().to_string())); - - let file_path = buffer - .file() - .and_then(|file| Some(file.path().to_path_buf())); - - (content, language_name, file_path) - }); - - PromptCodeSnippet { - path: file_path, - language_name, - content, - } - } -} - -impl ToString for PromptCodeSnippet { - fn to_string(&self) -> String { - let path = self - .path - .as_ref() - .and_then(|path| Some(path.to_string_lossy().to_string())) - .unwrap_or("".to_string()); - let language_name = self.language_name.clone().unwrap_or("".to_string()); - let content = self.content.clone(); - - format!("The below code snippet may be relevant from file: {path}\n```{language_name}\n{content}\n```") - } -} - #[allow(dead_code)] fn summarize(buffer: &BufferSnapshot, selected_range: Range) -> String { #[derive(Debug)] @@ -175,7 +129,32 @@ pub fn generate_content_prompt( kind: CodegenKind, search_results: Vec, model: &str, -) -> String { +) -> anyhow::Result { + // Using new Prompt Templates + let openai_model: Arc = Arc::new(OpenAILanguageModel::load(model)); + let lang_name = if let Some(language_name) = language_name { + Some(language_name.to_string()) + } else { + None + }; + + let args = PromptArguments { + model: openai_model, + language_name: lang_name.clone(), + project_name: None, + snippets: search_results.clone(), + reserved_tokens: 1000, + }; + + let templates: Vec<(PromptPriority, Box)> = vec![ + (PromptPriority::High, Box::new(EngineerPreamble {})), + (PromptPriority::Low, Box::new(RepositoryContext {})), + ]; + let chain = PromptChain::new(args, templates); + + let prompt = chain.generate(true)?; + println!("{:?}", prompt); + const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; const RESERVED_TOKENS_FOR_GENERATION: usize = 1000; @@ -183,7 +162,7 @@ pub fn generate_content_prompt( let range = range.to_offset(buffer); // General Preamble - if let Some(language_name) = language_name { + if let Some(language_name) = language_name.clone() { prompts.push(format!("You're an expert {language_name} engineer.\n")); } else { prompts.push("You're an expert engineer.\n".to_string()); @@ -297,7 +276,7 @@ pub fn generate_content_prompt( } } - prompts.join("\n") + anyhow::Ok(prompts.join("\n")) } #[cfg(test)] From 178a79fc471c541cc6351f491fbf585a551a9bce Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 12:29:10 -0400 Subject: [PATCH 12/41] added prompt template for file context without truncation --- crates/ai/src/templates/base.rs | 13 ++++ crates/ai/src/templates/file_context.rs | 85 +++++++++++++++++++++++++ crates/ai/src/templates/mod.rs | 1 + 3 files changed, 99 insertions(+) create mode 100644 crates/ai/src/templates/file_context.rs diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index b5f9da3586f7793e601ca8f5bf7a3158da5949c8..0bf04f5ed17c607ba115446e455ca1ffd937d5bd 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -1,6 +1,9 @@ use std::cmp::Reverse; +use std::ops::Range; use std::sync::Arc; +use gpui::ModelHandle; +use language::{Anchor, Buffer, BufferSnapshot, ToOffset}; use util::ResultExt; use crate::models::LanguageModel; @@ -18,6 +21,8 @@ pub struct PromptArguments { pub project_name: Option, pub snippets: Vec, pub reserved_tokens: usize, + pub buffer: Option, + pub selected_range: Option>, } impl PromptArguments { @@ -189,6 +194,8 @@ pub(crate) mod tests { project_name: None, snippets: Vec::new(), reserved_tokens: 0, + buffer: None, + selected_range: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -216,6 +223,8 @@ pub(crate) mod tests { project_name: None, snippets: Vec::new(), reserved_tokens: 0, + buffer: None, + selected_range: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -244,6 +253,8 @@ pub(crate) mod tests { project_name: None, snippets: Vec::new(), reserved_tokens: 0, + buffer: None, + selected_range: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -268,6 +279,8 @@ pub(crate) mod tests { project_name: None, snippets: Vec::new(), reserved_tokens, + buffer: None, + selected_range: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ (PromptPriority::Medium, Box::new(TestPromptTemplate {})), diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs new file mode 100644 index 0000000000000000000000000000000000000000..68bf424db1ddb6c3cd11907688ee5080e8f41c5f --- /dev/null +++ b/crates/ai/src/templates/file_context.rs @@ -0,0 +1,85 @@ +use language::ToOffset; + +use crate::templates::base::PromptArguments; +use crate::templates::base::PromptTemplate; +use std::fmt::Write; + +pub struct FileContext {} + +impl PromptTemplate for FileContext { + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + let mut prompt = String::new(); + + // Add Initial Preamble + // TODO: Do we want to add the path in here? + writeln!( + prompt, + "The file you are currently working on has the following content:" + ) + .unwrap(); + + let language_name = args + .language_name + .clone() + .unwrap_or("".to_string()) + .to_lowercase(); + writeln!(prompt, "```{language_name}").unwrap(); + + if let Some(buffer) = &args.buffer { + let mut content = String::new(); + + if let Some(selected_range) = &args.selected_range { + let start = selected_range.start.to_offset(buffer); + let end = selected_range.end.to_offset(buffer); + + writeln!( + prompt, + "{}", + buffer.text_for_range(0..start).collect::() + ) + .unwrap(); + + if start == end { + writeln!(prompt, "<|START|>").unwrap(); + } else { + writeln!(prompt, "<|START|").unwrap(); + } + + writeln!( + prompt, + "{}", + buffer.text_for_range(start..end).collect::() + ) + .unwrap(); + if start != end { + writeln!(prompt, "|END|>").unwrap(); + } + + writeln!( + prompt, + "{}", + buffer.text_for_range(end..buffer.len()).collect::() + ) + .unwrap(); + + writeln!(prompt, "```").unwrap(); + + if start == end { + writeln!(prompt, "In particular, the user's cursor is currently on the '<|START|>' span in the above content, with no text selected.").unwrap(); + } else { + writeln!(prompt, "In particular, the user has selected a section of the text between the '<|START|' and '|END|>' spans.").unwrap(); + } + } else { + // If we dont have a selected range, include entire file. + writeln!(prompt, "{}", &buffer.text()).unwrap(); + } + } + + let token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count)) + } +} diff --git a/crates/ai/src/templates/mod.rs b/crates/ai/src/templates/mod.rs index 62cb600eca4fb641265a3937ec5bf8f1e8c2d9c2..886af86e91db4dada1a051f211c19e030c100ec7 100644 --- a/crates/ai/src/templates/mod.rs +++ b/crates/ai/src/templates/mod.rs @@ -1,3 +1,4 @@ pub mod base; +pub mod file_context; pub mod preamble; pub mod repository_context; From fa61c1b9c1751912436dc44508af8aaa475493f2 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 13:03:11 -0400 Subject: [PATCH 13/41] add prompt template for generate inline content --- crates/ai/src/templates/base.rs | 5 ++ crates/ai/src/templates/generate.rs | 88 +++++++++++++++++++++++++++++ crates/ai/src/templates/mod.rs | 1 + 3 files changed, 94 insertions(+) create mode 100644 crates/ai/src/templates/generate.rs diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index 0bf04f5ed17c607ba115446e455ca1ffd937d5bd..d4882bafc91d4a408558a8eafbf7ce5360132217 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -17,6 +17,7 @@ pub(crate) enum PromptFileType { // TODO: Set this up to manage for defaults well pub struct PromptArguments { pub model: Arc, + pub user_prompt: Option, pub language_name: Option, pub project_name: Option, pub snippets: Vec, @@ -196,6 +197,7 @@ pub(crate) mod tests { reserved_tokens: 0, buffer: None, selected_range: None, + user_prompt: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -225,6 +227,7 @@ pub(crate) mod tests { reserved_tokens: 0, buffer: None, selected_range: None, + user_prompt: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -255,6 +258,7 @@ pub(crate) mod tests { reserved_tokens: 0, buffer: None, selected_range: None, + user_prompt: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ @@ -281,6 +285,7 @@ pub(crate) mod tests { reserved_tokens, buffer: None, selected_range: None, + user_prompt: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ (PromptPriority::Medium, Box::new(TestPromptTemplate {})), diff --git a/crates/ai/src/templates/generate.rs b/crates/ai/src/templates/generate.rs new file mode 100644 index 0000000000000000000000000000000000000000..d8a1ff6cf142fe8a4a81079ed3ada3c4f803eb75 --- /dev/null +++ b/crates/ai/src/templates/generate.rs @@ -0,0 +1,88 @@ +use crate::templates::base::{PromptArguments, PromptFileType, PromptTemplate}; +use anyhow::anyhow; +use std::fmt::Write; + +pub fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + +pub struct GenerateInlineContent {} + +impl PromptTemplate for GenerateInlineContent { + fn generate( + &self, + args: &PromptArguments, + max_token_length: Option, + ) -> anyhow::Result<(String, usize)> { + let Some(user_prompt) = &args.user_prompt else { + return Err(anyhow!("user prompt not provided")); + }; + + let file_type = args.get_file_type(); + let content_type = match &file_type { + PromptFileType::Code => "code", + PromptFileType::Text => "text", + }; + + let mut prompt = String::new(); + + if let Some(selected_range) = &args.selected_range { + if selected_range.start == selected_range.end { + writeln!( + prompt, + "Assume the cursor is located where the `<|START|>` span is." + ) + .unwrap(); + writeln!( + prompt, + "{} can't be replaced, so assume your answer will be inserted at the cursor.", + capitalize(content_type) + ) + .unwrap(); + writeln!( + prompt, + "Generate {content_type} based on the users prompt: {user_prompt}", + ) + .unwrap(); + } else { + writeln!(prompt, "Modify the user's selected {content_type} based upon the users prompt: '{user_prompt}'").unwrap(); + writeln!(prompt, "You MUST reply with only the adjusted {content_type} (within the '<|START|' and '|END|>' spans), not the entire file.").unwrap(); + } + } else { + writeln!( + prompt, + "Generate {content_type} based on the users prompt: {user_prompt}" + ) + .unwrap(); + } + + if let Some(language_name) = &args.language_name { + writeln!( + prompt, + "Your answer MUST always and only be valid {}.", + language_name + ) + .unwrap(); + } + writeln!(prompt, "Never make remarks about the output.").unwrap(); + writeln!( + prompt, + "Do not return anything else, except the generated {content_type}." + ) + .unwrap(); + + match file_type { + PromptFileType::Code => { + writeln!(prompt, "Always wrap your code in a Markdown block.").unwrap(); + } + _ => {} + } + + let token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count)) + } +} diff --git a/crates/ai/src/templates/mod.rs b/crates/ai/src/templates/mod.rs index 886af86e91db4dada1a051f211c19e030c100ec7..0025269a440d1e6ead6a81615a64a3c28da62bb8 100644 --- a/crates/ai/src/templates/mod.rs +++ b/crates/ai/src/templates/mod.rs @@ -1,4 +1,5 @@ pub mod base; pub mod file_context; +pub mod generate; pub mod preamble; pub mod repository_context; From b9bb27512caf402727680fc3ad6926f9006adfce Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 13:10:31 -0400 Subject: [PATCH 14/41] fix template ordering during prompt chain generation --- crates/ai/src/templates/base.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index d4882bafc91d4a408558a8eafbf7ce5360132217..db437a029cd73ec620385362ed83061103d82078 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -77,8 +77,6 @@ impl PromptChain { let mut sorted_indices = (0..self.templates.len()).collect::>(); sorted_indices.sort_by_key(|&i| Reverse(&self.templates[i].0)); - let mut prompts = Vec::new(); - // If Truncate let mut tokens_outstanding = if truncate { Some(self.args.model.capacity()? - self.args.reserved_tokens) @@ -86,6 +84,7 @@ impl PromptChain { None }; + let mut prompts = vec!["".to_string(); sorted_indices.len()]; for idx in sorted_indices { let (_, template) = &self.templates[idx]; if let Some((template_prompt, prompt_token_count)) = @@ -96,7 +95,7 @@ impl PromptChain { &prompt_token_count, &template_prompt ); if template_prompt != "" { - prompts.push(template_prompt); + prompts[idx] = template_prompt; if let Some(remaining_tokens) = tokens_outstanding { let new_tokens = prompt_token_count + seperator_tokens; From aa1825681c60176d391ba497a9d28b0e5703fa60 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 14:20:12 -0400 Subject: [PATCH 15/41] update the assistant panel to use new prompt templates --- crates/ai/src/templates/base.rs | 4 - crates/ai/src/templates/file_context.rs | 10 +- crates/ai/src/templates/preamble.rs | 2 +- crates/assistant/src/assistant_panel.rs | 17 ++- crates/assistant/src/prompts.rs | 146 +++--------------------- 5 files changed, 33 insertions(+), 146 deletions(-) diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index db437a029cd73ec620385362ed83061103d82078..aaf08d755efb4746192bb75e64f0f7cc7e7a4e83 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -90,10 +90,6 @@ impl PromptChain { if let Some((template_prompt, prompt_token_count)) = template.generate(&self.args, tokens_outstanding).log_err() { - println!( - "GENERATED PROMPT ({:?}): {:?}", - &prompt_token_count, &template_prompt - ); if template_prompt != "" { prompts[idx] = template_prompt; diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 68bf424db1ddb6c3cd11907688ee5080e8f41c5f..6d0630504983fbe90597525ea8f49dd23e0a1036 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -44,22 +44,22 @@ impl PromptTemplate for FileContext { .unwrap(); if start == end { - writeln!(prompt, "<|START|>").unwrap(); + write!(prompt, "<|START|>").unwrap(); } else { - writeln!(prompt, "<|START|").unwrap(); + write!(prompt, "<|START|").unwrap(); } - writeln!( + write!( prompt, "{}", buffer.text_for_range(start..end).collect::() ) .unwrap(); if start != end { - writeln!(prompt, "|END|>").unwrap(); + write!(prompt, "|END|>").unwrap(); } - writeln!( + write!( prompt, "{}", buffer.text_for_range(end..buffer.len()).collect::() diff --git a/crates/ai/src/templates/preamble.rs b/crates/ai/src/templates/preamble.rs index 5834fa1b21b2011fbbc82d781493c4e4e523b685..9eabaaeb97fe4216c6bac44cf4eabfb7c129ecf2 100644 --- a/crates/ai/src/templates/preamble.rs +++ b/crates/ai/src/templates/preamble.rs @@ -25,7 +25,7 @@ impl PromptTemplate for EngineerPreamble { if let Some(project_name) = args.project_name.clone() { prompts.push(format!( - "You are currently working inside the '{project_name}' in Zed the code editor." + "You are currently working inside the '{project_name}' project in code editor Zed." )); } diff --git a/crates/assistant/src/assistant_panel.rs b/crates/assistant/src/assistant_panel.rs index 3a0f05379e1d06ce9900bcb2179ed1a347c96f70..4dd4e2a98315c042d74c7ef6bde78200287ab6ad 100644 --- a/crates/assistant/src/assistant_panel.rs +++ b/crates/assistant/src/assistant_panel.rs @@ -612,6 +612,18 @@ impl AssistantPanel { let project = pending_assist.project.clone(); + let project_name = if let Some(project) = project.upgrade(cx) { + Some( + project + .read(cx) + .worktree_root_names(cx) + .collect::>() + .join("/"), + ) + } else { + None + }; + self.inline_prompt_history .retain(|prompt| prompt != user_prompt); self.inline_prompt_history.push_back(user_prompt.into()); @@ -649,7 +661,6 @@ impl AssistantPanel { None }; - let codegen_kind = codegen.read(cx).kind().clone(); let user_prompt = user_prompt.to_string(); let snippets = if retrieve_context { @@ -692,11 +703,11 @@ impl AssistantPanel { generate_content_prompt( user_prompt, language_name, - &buffer, + buffer, range, - codegen_kind, snippets, model_name, + project_name, ) }); diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index 333742aa0525afe4d362523868ada9cb187cc363..1457d28fff22c83c29090dcded37aa9a915918bd 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -1,6 +1,8 @@ use crate::codegen::CodegenKind; use ai::models::{LanguageModel, OpenAILanguageModel}; use ai::templates::base::{PromptArguments, PromptChain, PromptPriority, PromptTemplate}; +use ai::templates::file_context::FileContext; +use ai::templates::generate::GenerateInlineContent; use ai::templates::preamble::EngineerPreamble; use ai::templates::repository_context::{PromptCodeSnippet, RepositoryContext}; use language::{BufferSnapshot, OffsetRangeExt, ToOffset}; @@ -124,11 +126,11 @@ fn summarize(buffer: &BufferSnapshot, selected_range: Range) -> S pub fn generate_content_prompt( user_prompt: String, language_name: Option<&str>, - buffer: &BufferSnapshot, - range: Range, - kind: CodegenKind, + buffer: BufferSnapshot, + range: Range, search_results: Vec, model: &str, + project_name: Option, ) -> anyhow::Result { // Using new Prompt Templates let openai_model: Arc = Arc::new(OpenAILanguageModel::load(model)); @@ -141,146 +143,24 @@ pub fn generate_content_prompt( let args = PromptArguments { model: openai_model, language_name: lang_name.clone(), - project_name: None, + project_name, snippets: search_results.clone(), reserved_tokens: 1000, + buffer: Some(buffer), + selected_range: Some(range), + user_prompt: Some(user_prompt.clone()), }; let templates: Vec<(PromptPriority, Box)> = vec![ (PromptPriority::High, Box::new(EngineerPreamble {})), (PromptPriority::Low, Box::new(RepositoryContext {})), + (PromptPriority::Medium, Box::new(FileContext {})), + (PromptPriority::High, Box::new(GenerateInlineContent {})), ]; let chain = PromptChain::new(args, templates); + let (prompt, _) = chain.generate(true)?; - let prompt = chain.generate(true)?; - println!("{:?}", prompt); - - const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; - const RESERVED_TOKENS_FOR_GENERATION: usize = 1000; - - let mut prompts = Vec::new(); - let range = range.to_offset(buffer); - - // General Preamble - if let Some(language_name) = language_name.clone() { - prompts.push(format!("You're an expert {language_name} engineer.\n")); - } else { - prompts.push("You're an expert engineer.\n".to_string()); - } - - // Snippets - let mut snippet_position = prompts.len() - 1; - - let mut content = String::new(); - content.extend(buffer.text_for_range(0..range.start)); - if range.start == range.end { - content.push_str("<|START|>"); - } else { - content.push_str("<|START|"); - } - content.extend(buffer.text_for_range(range.clone())); - if range.start != range.end { - content.push_str("|END|>"); - } - content.extend(buffer.text_for_range(range.end..buffer.len())); - - prompts.push("The file you are currently working on has the following content:\n".to_string()); - - if let Some(language_name) = language_name { - let language_name = language_name.to_lowercase(); - prompts.push(format!("```{language_name}\n{content}\n```")); - } else { - prompts.push(format!("```\n{content}\n```")); - } - - match kind { - CodegenKind::Generate { position: _ } => { - prompts.push("In particular, the user's cursor is currently on the '<|START|>' span in the above outline, with no text selected.".to_string()); - prompts - .push("Assume the cursor is located where the `<|START|` marker is.".to_string()); - prompts.push( - "Text can't be replaced, so assume your answer will be inserted at the cursor." - .to_string(), - ); - prompts.push(format!( - "Generate text based on the users prompt: {user_prompt}" - )); - } - CodegenKind::Transform { range: _ } => { - prompts.push("In particular, the user has selected a section of the text between the '<|START|' and '|END|>' spans.".to_string()); - prompts.push(format!( - "Modify the users code selected text based upon the users prompt: '{user_prompt}'" - )); - prompts.push("You MUST reply with only the adjusted code (within the '<|START|' and '|END|>' spans), not the entire file.".to_string()); - } - } - - if let Some(language_name) = language_name { - prompts.push(format!( - "Your answer MUST always and only be valid {language_name}" - )); - } - prompts.push("Never make remarks about the output.".to_string()); - prompts.push("Do not return any text, except the generated code.".to_string()); - prompts.push("Always wrap your code in a Markdown block".to_string()); - - let current_messages = [ChatCompletionRequestMessage { - role: "user".to_string(), - content: Some(prompts.join("\n")), - function_call: None, - name: None, - }]; - - let mut remaining_token_count = if let Ok(current_token_count) = - tiktoken_rs::num_tokens_from_messages(model, ¤t_messages) - { - let max_token_count = tiktoken_rs::model::get_context_size(model); - let intermediate_token_count = if max_token_count > current_token_count { - max_token_count - current_token_count - } else { - 0 - }; - - if intermediate_token_count < RESERVED_TOKENS_FOR_GENERATION { - 0 - } else { - intermediate_token_count - RESERVED_TOKENS_FOR_GENERATION - } - } else { - // If tiktoken fails to count token count, assume we have no space remaining. - 0 - }; - - // TODO: - // - add repository name to snippet - // - add file path - // - add language - if let Ok(encoding) = tiktoken_rs::get_bpe_from_model(model) { - let mut template = "You are working inside a large repository, here are a few code snippets that may be useful"; - - for search_result in search_results { - let mut snippet_prompt = template.to_string(); - let snippet = search_result.to_string(); - writeln!(snippet_prompt, "```\n{snippet}\n```").unwrap(); - - let token_count = encoding - .encode_with_special_tokens(snippet_prompt.as_str()) - .len(); - if token_count <= remaining_token_count { - if token_count < MAXIMUM_SNIPPET_TOKEN_COUNT { - prompts.insert(snippet_position, snippet_prompt); - snippet_position += 1; - remaining_token_count -= token_count; - // If you have already added the template to the prompt, remove the template. - template = ""; - } - } else { - break; - } - } - } - - anyhow::Ok(prompts.join("\n")) + anyhow::Ok(prompt) } #[cfg(test)] From 473067db3173f6e43666f1283c850cff8d2b8cd5 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 15:56:39 -0400 Subject: [PATCH 16/41] update PromptPriority to accomodate for both Mandatory and Ordered prompts --- crates/ai/src/templates/base.rs | 101 ++++++++++++++---- crates/ai/src/templates/file_context.rs | 2 - crates/ai/src/templates/repository_context.rs | 2 +- crates/assistant/src/prompts.rs | 20 ++-- 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index aaf08d755efb4746192bb75e64f0f7cc7e7a4e83..2afcc87ff5dc49072b558fffc4f22da1a34909e9 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -1,9 +1,9 @@ +use anyhow::anyhow; use std::cmp::Reverse; use std::ops::Range; use std::sync::Arc; -use gpui::ModelHandle; -use language::{Anchor, Buffer, BufferSnapshot, ToOffset}; +use language::BufferSnapshot; use util::ResultExt; use crate::models::LanguageModel; @@ -50,11 +50,21 @@ pub trait PromptTemplate { } #[repr(i8)] -#[derive(PartialEq, Eq, PartialOrd, Ord)] +#[derive(PartialEq, Eq, Ord)] pub enum PromptPriority { - Low, - Medium, - High, + Mandatory, // Ignores truncation + Ordered { order: usize }, // Truncates based on priority +} + +impl PartialOrd for PromptPriority { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Mandatory, Self::Mandatory) => Some(std::cmp::Ordering::Equal), + (Self::Mandatory, Self::Ordered { .. }) => Some(std::cmp::Ordering::Greater), + (Self::Ordered { .. }, Self::Mandatory) => Some(std::cmp::Ordering::Less), + (Self::Ordered { order: a }, Self::Ordered { order: b }) => b.partial_cmp(a), + } + } } pub struct PromptChain { @@ -86,14 +96,36 @@ impl PromptChain { let mut prompts = vec!["".to_string(); sorted_indices.len()]; for idx in sorted_indices { - let (_, template) = &self.templates[idx]; + let (priority, template) = &self.templates[idx]; + + // If PromptPriority is marked as mandatory, we ignore the tokens outstanding + // However, if a prompt is generated in excess of the available tokens, + // we raise an error outlining that a mandatory prompt has exceeded the available + // balance + let template_tokens = if let Some(template_tokens) = tokens_outstanding { + match priority { + &PromptPriority::Mandatory => None, + _ => Some(template_tokens), + } + } else { + None + }; + if let Some((template_prompt, prompt_token_count)) = - template.generate(&self.args, tokens_outstanding).log_err() + template.generate(&self.args, template_tokens).log_err() { if template_prompt != "" { prompts[idx] = template_prompt; if let Some(remaining_tokens) = tokens_outstanding { + if prompt_token_count > remaining_tokens + && priority == &PromptPriority::Mandatory + { + return Err(anyhow!( + "mandatory template added in excess of model capacity" + )); + } + let new_tokens = prompt_token_count + seperator_tokens; tokens_outstanding = if remaining_tokens > new_tokens { Some(remaining_tokens - new_tokens) @@ -105,6 +137,8 @@ impl PromptChain { } } + prompts.retain(|x| x != ""); + let full_prompt = prompts.join(seperator); let total_token_count = self.args.model.count_tokens(&full_prompt)?; anyhow::Ok((prompts.join(seperator), total_token_count)) @@ -196,8 +230,14 @@ pub(crate) mod tests { }; let templates: Vec<(PromptPriority, Box)> = vec![ - (PromptPriority::High, Box::new(TestPromptTemplate {})), - (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), + ( + PromptPriority::Ordered { order: 0 }, + Box::new(TestPromptTemplate {}), + ), + ( + PromptPriority::Ordered { order: 1 }, + Box::new(TestLowPriorityTemplate {}), + ), ]; let chain = PromptChain::new(args, templates); @@ -226,8 +266,14 @@ pub(crate) mod tests { }; let templates: Vec<(PromptPriority, Box)> = vec![ - (PromptPriority::High, Box::new(TestPromptTemplate {})), - (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), + ( + PromptPriority::Ordered { order: 0 }, + Box::new(TestPromptTemplate {}), + ), + ( + PromptPriority::Ordered { order: 1 }, + Box::new(TestLowPriorityTemplate {}), + ), ]; let chain = PromptChain::new(args, templates); @@ -257,9 +303,18 @@ pub(crate) mod tests { }; let templates: Vec<(PromptPriority, Box)> = vec![ - (PromptPriority::High, Box::new(TestPromptTemplate {})), - (PromptPriority::Medium, Box::new(TestLowPriorityTemplate {})), - (PromptPriority::Low, Box::new(TestLowPriorityTemplate {})), + ( + PromptPriority::Ordered { order: 0 }, + Box::new(TestPromptTemplate {}), + ), + ( + PromptPriority::Ordered { order: 1 }, + Box::new(TestLowPriorityTemplate {}), + ), + ( + PromptPriority::Ordered { order: 2 }, + Box::new(TestLowPriorityTemplate {}), + ), ]; let chain = PromptChain::new(args, templates); @@ -283,14 +338,22 @@ pub(crate) mod tests { user_prompt: None, }; let templates: Vec<(PromptPriority, Box)> = vec![ - (PromptPriority::Medium, Box::new(TestPromptTemplate {})), - (PromptPriority::High, Box::new(TestLowPriorityTemplate {})), - (PromptPriority::Low, Box::new(TestLowPriorityTemplate {})), + ( + PromptPriority::Mandatory, + Box::new(TestLowPriorityTemplate {}), + ), + ( + PromptPriority::Ordered { order: 0 }, + Box::new(TestPromptTemplate {}), + ), + ( + PromptPriority::Ordered { order: 1 }, + Box::new(TestLowPriorityTemplate {}), + ), ]; let chain = PromptChain::new(args, templates); let (prompt, token_count) = chain.generate(true).unwrap(); - println!("TOKEN COUNT: {:?}", token_count); assert_eq!( prompt, diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 6d0630504983fbe90597525ea8f49dd23e0a1036..94b194d9bf7ac4a247d8feb9c8327a50e034cf2a 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -30,8 +30,6 @@ impl PromptTemplate for FileContext { writeln!(prompt, "```{language_name}").unwrap(); if let Some(buffer) = &args.buffer { - let mut content = String::new(); - if let Some(selected_range) = &args.selected_range { let start = selected_range.start.to_offset(buffer); let end = selected_range.end.to_offset(buffer); diff --git a/crates/ai/src/templates/repository_context.rs b/crates/ai/src/templates/repository_context.rs index 7dd1647c440a228b5fc2c8317fe35e0931d4c1a9..a8e7f4b5af7bee4d3f29d70c665965dc7e05ed4b 100644 --- a/crates/ai/src/templates/repository_context.rs +++ b/crates/ai/src/templates/repository_context.rs @@ -60,7 +60,7 @@ impl PromptTemplate for RepositoryContext { max_token_length: Option, ) -> anyhow::Result<(String, usize)> { const MAXIMUM_SNIPPET_TOKEN_COUNT: usize = 500; - let mut template = "You are working inside a large repository, here are a few code snippets that may be useful."; + let template = "You are working inside a large repository, here are a few code snippets that may be useful."; let mut prompt = String::new(); let mut remaining_tokens = max_token_length.clone(); diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index 1457d28fff22c83c29090dcded37aa9a915918bd..dffcbc29234d3f24174d1d9a6610045105eae890 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -1,4 +1,3 @@ -use crate::codegen::CodegenKind; use ai::models::{LanguageModel, OpenAILanguageModel}; use ai::templates::base::{PromptArguments, PromptChain, PromptPriority, PromptTemplate}; use ai::templates::file_context::FileContext; @@ -7,10 +6,8 @@ use ai::templates::preamble::EngineerPreamble; use ai::templates::repository_context::{PromptCodeSnippet, RepositoryContext}; use language::{BufferSnapshot, OffsetRangeExt, ToOffset}; use std::cmp::{self, Reverse}; -use std::fmt::Write; use std::ops::Range; use std::sync::Arc; -use tiktoken_rs::ChatCompletionRequestMessage; #[allow(dead_code)] fn summarize(buffer: &BufferSnapshot, selected_range: Range) -> String { @@ -152,10 +149,19 @@ pub fn generate_content_prompt( }; let templates: Vec<(PromptPriority, Box)> = vec![ - (PromptPriority::High, Box::new(EngineerPreamble {})), - (PromptPriority::Low, Box::new(RepositoryContext {})), - (PromptPriority::Medium, Box::new(FileContext {})), - (PromptPriority::High, Box::new(GenerateInlineContent {})), + (PromptPriority::Mandatory, Box::new(EngineerPreamble {})), + ( + PromptPriority::Ordered { order: 1 }, + Box::new(RepositoryContext {}), + ), + ( + PromptPriority::Ordered { order: 0 }, + Box::new(FileContext {}), + ), + ( + PromptPriority::Mandatory, + Box::new(GenerateInlineContent {}), + ), ]; let chain = PromptChain::new(args, templates); let (prompt, _) = chain.generate(true)?; From 32853c20447d35abbf732441cc2e02cd48587938 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 16:23:53 -0400 Subject: [PATCH 17/41] added initial placeholder for truncation without a valid strategy --- crates/ai/src/templates/file_context.rs | 7 +++++++ crates/ai/src/templates/generate.rs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 94b194d9bf7ac4a247d8feb9c8327a50e034cf2a..e28f9ccdedb293817c22f54e0b2a12f17a40ac9f 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use language::ToOffset; use crate::templates::base::PromptArguments; @@ -12,6 +13,12 @@ impl PromptTemplate for FileContext { args: &PromptArguments, max_token_length: Option, ) -> anyhow::Result<(String, usize)> { + if max_token_length.is_some() { + return Err(anyhow!( + "no truncation strategy established for file_context template" + )); + } + let mut prompt = String::new(); // Add Initial Preamble diff --git a/crates/ai/src/templates/generate.rs b/crates/ai/src/templates/generate.rs index d8a1ff6cf142fe8a4a81079ed3ada3c4f803eb75..053398e873828a22e35e90e5a12b7137c83b2de0 100644 --- a/crates/ai/src/templates/generate.rs +++ b/crates/ai/src/templates/generate.rs @@ -18,6 +18,12 @@ impl PromptTemplate for GenerateInlineContent { args: &PromptArguments, max_token_length: Option, ) -> anyhow::Result<(String, usize)> { + if max_token_length.is_some() { + return Err(anyhow!( + "no truncation strategy established for generating inline content template" + )); + } + let Some(user_prompt) = &args.user_prompt else { return Err(anyhow!("user prompt not provided")); }; @@ -83,6 +89,7 @@ impl PromptTemplate for GenerateInlineContent { } let token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count)) } } From a0e01e075d46b05ddc0737065348e57f38952edf Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 16:31:29 -0400 Subject: [PATCH 18/41] fix for error when truncating a length less than the string length --- crates/ai/src/models.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ai/src/models.rs b/crates/ai/src/models.rs index 69e73e9b56ec7db7023983a67f5ee994c97c5725..0cafb49d94705a25322e3d6540d9a4a87e434c41 100644 --- a/crates/ai/src/models.rs +++ b/crates/ai/src/models.rs @@ -38,7 +38,11 @@ impl LanguageModel for OpenAILanguageModel { fn truncate(&self, content: &str, length: usize) -> anyhow::Result { if let Some(bpe) = &self.bpe { let tokens = bpe.encode_with_special_tokens(content); - bpe.decode(tokens[..length].to_vec()) + if tokens.len() > length { + bpe.decode(tokens[..length].to_vec()) + } else { + bpe.decode(tokens) + } } else { Err(anyhow!("bpe for open ai model was not retrieved")) } From f59f2eccd5e7f0706f0a3c5b1db6832d67380708 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 16:32:14 -0400 Subject: [PATCH 19/41] added dumb truncation strategies to file_context and generate --- crates/ai/src/templates/base.rs | 26 ++----------------------- crates/ai/src/templates/file_context.rs | 12 +++++------- crates/ai/src/templates/generate.rs | 11 +++++------ 3 files changed, 12 insertions(+), 37 deletions(-) diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index 2afcc87ff5dc49072b558fffc4f22da1a34909e9..923e1833c2115953a27044d198497db256287907 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -1,4 +1,3 @@ -use anyhow::anyhow; use std::cmp::Reverse; use std::ops::Range; use std::sync::Arc; @@ -96,36 +95,15 @@ impl PromptChain { let mut prompts = vec!["".to_string(); sorted_indices.len()]; for idx in sorted_indices { - let (priority, template) = &self.templates[idx]; - - // If PromptPriority is marked as mandatory, we ignore the tokens outstanding - // However, if a prompt is generated in excess of the available tokens, - // we raise an error outlining that a mandatory prompt has exceeded the available - // balance - let template_tokens = if let Some(template_tokens) = tokens_outstanding { - match priority { - &PromptPriority::Mandatory => None, - _ => Some(template_tokens), - } - } else { - None - }; + let (_, template) = &self.templates[idx]; if let Some((template_prompt, prompt_token_count)) = - template.generate(&self.args, template_tokens).log_err() + template.generate(&self.args, tokens_outstanding).log_err() { if template_prompt != "" { prompts[idx] = template_prompt; if let Some(remaining_tokens) = tokens_outstanding { - if prompt_token_count > remaining_tokens - && priority == &PromptPriority::Mandatory - { - return Err(anyhow!( - "mandatory template added in excess of model capacity" - )); - } - let new_tokens = prompt_token_count + seperator_tokens; tokens_outstanding = if remaining_tokens > new_tokens { Some(remaining_tokens - new_tokens) diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index e28f9ccdedb293817c22f54e0b2a12f17a40ac9f..00fe99dd7ffc257339caa8c5198e532b967bee40 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -1,4 +1,3 @@ -use anyhow::anyhow; use language::ToOffset; use crate::templates::base::PromptArguments; @@ -13,12 +12,6 @@ impl PromptTemplate for FileContext { args: &PromptArguments, max_token_length: Option, ) -> anyhow::Result<(String, usize)> { - if max_token_length.is_some() { - return Err(anyhow!( - "no truncation strategy established for file_context template" - )); - } - let mut prompt = String::new(); // Add Initial Preamble @@ -84,6 +77,11 @@ impl PromptTemplate for FileContext { } } + // Really dumb truncation strategy + if let Some(max_tokens) = max_token_length { + prompt = args.model.truncate(&prompt, max_tokens)?; + } + let token_count = args.model.count_tokens(&prompt)?; anyhow::Ok((prompt, token_count)) } diff --git a/crates/ai/src/templates/generate.rs b/crates/ai/src/templates/generate.rs index 053398e873828a22e35e90e5a12b7137c83b2de0..34d874cc4128ccae034a0ecf3beace159bbec1ac 100644 --- a/crates/ai/src/templates/generate.rs +++ b/crates/ai/src/templates/generate.rs @@ -18,12 +18,6 @@ impl PromptTemplate for GenerateInlineContent { args: &PromptArguments, max_token_length: Option, ) -> anyhow::Result<(String, usize)> { - if max_token_length.is_some() { - return Err(anyhow!( - "no truncation strategy established for generating inline content template" - )); - } - let Some(user_prompt) = &args.user_prompt else { return Err(anyhow!("user prompt not provided")); }; @@ -88,6 +82,11 @@ impl PromptTemplate for GenerateInlineContent { _ => {} } + // Really dumb truncation strategy + if let Some(max_tokens) = max_token_length { + prompt = args.model.truncate(&prompt, max_tokens)?; + } + let token_count = args.model.count_tokens(&prompt)?; anyhow::Ok((prompt, token_count)) From 587fd707ba9c19fcef18b6bb0f5507fab79641d9 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 16:40:09 -0400 Subject: [PATCH 20/41] added smarter error handling for file_context prompts without provided buffers --- crates/ai/src/templates/file_context.rs | 49 +++++++++++++------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 00fe99dd7ffc257339caa8c5198e532b967bee40..5a6489a00c89f637f03d5c25a357f82f24accfd3 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use language::ToOffset; use crate::templates::base::PromptArguments; @@ -12,24 +13,23 @@ impl PromptTemplate for FileContext { args: &PromptArguments, max_token_length: Option, ) -> anyhow::Result<(String, usize)> { - let mut prompt = String::new(); - - // Add Initial Preamble - // TODO: Do we want to add the path in here? - writeln!( - prompt, - "The file you are currently working on has the following content:" - ) - .unwrap(); + if let Some(buffer) = &args.buffer { + let mut prompt = String::new(); + // Add Initial Preamble + // TODO: Do we want to add the path in here? + writeln!( + prompt, + "The file you are currently working on has the following content:" + ) + .unwrap(); - let language_name = args - .language_name - .clone() - .unwrap_or("".to_string()) - .to_lowercase(); - writeln!(prompt, "```{language_name}").unwrap(); + let language_name = args + .language_name + .clone() + .unwrap_or("".to_string()) + .to_lowercase(); + writeln!(prompt, "```{language_name}").unwrap(); - if let Some(buffer) = &args.buffer { if let Some(selected_range) = &args.selected_range { let start = selected_range.start.to_offset(buffer); let end = selected_range.end.to_offset(buffer); @@ -74,15 +74,18 @@ impl PromptTemplate for FileContext { } else { // If we dont have a selected range, include entire file. writeln!(prompt, "{}", &buffer.text()).unwrap(); + writeln!(prompt, "```").unwrap(); } - } - // Really dumb truncation strategy - if let Some(max_tokens) = max_token_length { - prompt = args.model.truncate(&prompt, max_tokens)?; - } + // Really dumb truncation strategy + if let Some(max_tokens) = max_token_length { + prompt = args.model.truncate(&prompt, max_tokens)?; + } - let token_count = args.model.count_tokens(&prompt)?; - anyhow::Ok((prompt, token_count)) + let token_count = args.model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count)) + } else { + Err(anyhow!("no buffer provided to retrieve file context from")) + } } } From 178a84bcf62641ec39bd185b389a6381db06c4ba Mon Sep 17 00:00:00 2001 From: KCaverly Date: Wed, 18 Oct 2023 17:56:59 -0400 Subject: [PATCH 21/41] progress on smarter truncation strategy for file context --- crates/ai/src/models.rs | 13 +++ crates/ai/src/templates/base.rs | 7 ++ crates/ai/src/templates/file_context.rs | 139 +++++++++++++++++------- crates/assistant/src/prompts.rs | 2 + 4 files changed, 124 insertions(+), 37 deletions(-) diff --git a/crates/ai/src/models.rs b/crates/ai/src/models.rs index 0cafb49d94705a25322e3d6540d9a4a87e434c41..d0206cc41c526f171fef8521a120f8f4ff70aa74 100644 --- a/crates/ai/src/models.rs +++ b/crates/ai/src/models.rs @@ -6,6 +6,7 @@ pub trait LanguageModel { fn name(&self) -> String; fn count_tokens(&self, content: &str) -> anyhow::Result; fn truncate(&self, content: &str, length: usize) -> anyhow::Result; + fn truncate_start(&self, content: &str, length: usize) -> anyhow::Result; fn capacity(&self) -> anyhow::Result; } @@ -47,6 +48,18 @@ impl LanguageModel for OpenAILanguageModel { Err(anyhow!("bpe for open ai model was not retrieved")) } } + fn truncate_start(&self, content: &str, length: usize) -> anyhow::Result { + if let Some(bpe) = &self.bpe { + let tokens = bpe.encode_with_special_tokens(content); + if tokens.len() > length { + bpe.decode(tokens[length..].to_vec()) + } else { + bpe.decode(tokens) + } + } else { + Err(anyhow!("bpe for open ai model was not retrieved")) + } + } fn capacity(&self) -> anyhow::Result { anyhow::Ok(tiktoken_rs::model::get_context_size(&self.name)) } diff --git a/crates/ai/src/templates/base.rs b/crates/ai/src/templates/base.rs index 923e1833c2115953a27044d198497db256287907..bda1d6c30e61a9e2fd3808fa45a34cbe041cf2b6 100644 --- a/crates/ai/src/templates/base.rs +++ b/crates/ai/src/templates/base.rs @@ -190,6 +190,13 @@ pub(crate) mod tests { .collect::(), ) } + fn truncate_start(&self, content: &str, length: usize) -> anyhow::Result { + anyhow::Ok( + content.chars().collect::>()[length..] + .into_iter() + .collect::(), + ) + } fn capacity(&self) -> anyhow::Result { anyhow::Ok(self.capacity) } diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 5a6489a00c89f637f03d5c25a357f82f24accfd3..253d24e4691d52371d2af13e07e160ec3ac6e0f6 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -1,9 +1,103 @@ use anyhow::anyhow; +use language::BufferSnapshot; use language::ToOffset; +use crate::models::LanguageModel; use crate::templates::base::PromptArguments; use crate::templates::base::PromptTemplate; use std::fmt::Write; +use std::ops::Range; +use std::sync::Arc; + +fn retrieve_context( + buffer: &BufferSnapshot, + selected_range: &Option>, + model: Arc, + max_token_count: Option, +) -> anyhow::Result<(String, usize, bool)> { + let mut prompt = String::new(); + let mut truncated = false; + if let Some(selected_range) = selected_range { + let start = selected_range.start.to_offset(buffer); + let end = selected_range.end.to_offset(buffer); + + let start_window = buffer.text_for_range(0..start).collect::(); + + let mut selected_window = String::new(); + if start == end { + write!(selected_window, "<|START|>").unwrap(); + } else { + write!(selected_window, "<|START|").unwrap(); + } + + write!( + selected_window, + "{}", + buffer.text_for_range(start..end).collect::() + ) + .unwrap(); + + if start != end { + write!(selected_window, "|END|>").unwrap(); + } + + let end_window = buffer.text_for_range(end..buffer.len()).collect::(); + + if let Some(max_token_count) = max_token_count { + let selected_tokens = model.count_tokens(&selected_window)?; + if selected_tokens > max_token_count { + return Err(anyhow!( + "selected range is greater than model context window, truncation not possible" + )); + }; + + let mut remaining_tokens = max_token_count - selected_tokens; + let start_window_tokens = model.count_tokens(&start_window)?; + let end_window_tokens = model.count_tokens(&end_window)?; + let outside_tokens = start_window_tokens + end_window_tokens; + if outside_tokens > remaining_tokens { + let (start_goal_tokens, end_goal_tokens) = + if start_window_tokens < end_window_tokens { + let start_goal_tokens = (remaining_tokens / 2).min(start_window_tokens); + remaining_tokens -= start_goal_tokens; + let end_goal_tokens = remaining_tokens.min(end_window_tokens); + (start_goal_tokens, end_goal_tokens) + } else { + let end_goal_tokens = (remaining_tokens / 2).min(end_window_tokens); + remaining_tokens -= end_goal_tokens; + let start_goal_tokens = remaining_tokens.min(start_window_tokens); + (start_goal_tokens, end_goal_tokens) + }; + + let truncated_start_window = + model.truncate_start(&start_window, start_goal_tokens)?; + let truncated_end_window = model.truncate(&end_window, end_goal_tokens)?; + writeln!( + prompt, + "{truncated_start_window}{selected_window}{truncated_end_window}" + ) + .unwrap(); + truncated = true; + } else { + writeln!(prompt, "{start_window}{selected_window}{end_window}").unwrap(); + } + } else { + // If we dont have a selected range, include entire file. + writeln!(prompt, "{}", &buffer.text()).unwrap(); + + // Dumb truncation strategy + if let Some(max_token_count) = max_token_count { + if model.count_tokens(&prompt)? > max_token_count { + truncated = true; + prompt = model.truncate(&prompt, max_token_count)?; + } + } + } + } + + let token_count = model.count_tokens(&prompt)?; + anyhow::Ok((prompt, token_count, truncated)) +} pub struct FileContext {} @@ -28,53 +122,24 @@ impl PromptTemplate for FileContext { .clone() .unwrap_or("".to_string()) .to_lowercase(); - writeln!(prompt, "```{language_name}").unwrap(); + + let (context, _, truncated) = retrieve_context( + buffer, + &args.selected_range, + args.model.clone(), + max_token_length, + )?; + writeln!(prompt, "```{language_name}\n{context}\n```").unwrap(); if let Some(selected_range) = &args.selected_range { let start = selected_range.start.to_offset(buffer); let end = selected_range.end.to_offset(buffer); - writeln!( - prompt, - "{}", - buffer.text_for_range(0..start).collect::() - ) - .unwrap(); - - if start == end { - write!(prompt, "<|START|>").unwrap(); - } else { - write!(prompt, "<|START|").unwrap(); - } - - write!( - prompt, - "{}", - buffer.text_for_range(start..end).collect::() - ) - .unwrap(); - if start != end { - write!(prompt, "|END|>").unwrap(); - } - - write!( - prompt, - "{}", - buffer.text_for_range(end..buffer.len()).collect::() - ) - .unwrap(); - - writeln!(prompt, "```").unwrap(); - if start == end { writeln!(prompt, "In particular, the user's cursor is currently on the '<|START|>' span in the above content, with no text selected.").unwrap(); } else { writeln!(prompt, "In particular, the user has selected a section of the text between the '<|START|' and '|END|>' spans.").unwrap(); } - } else { - // If we dont have a selected range, include entire file. - writeln!(prompt, "{}", &buffer.text()).unwrap(); - writeln!(prompt, "```").unwrap(); } // Really dumb truncation strategy diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index dffcbc29234d3f24174d1d9a6610045105eae890..c7b52a35405deadc3a9319ea77aea34e1989f273 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -166,6 +166,8 @@ pub fn generate_content_prompt( let chain = PromptChain::new(args, templates); let (prompt, _) = chain.generate(true)?; + println!("PROMPT: {:?}", &prompt); + anyhow::Ok(prompt) } From cb76b2a6ad4371eec12cbee612d476f91fd59b97 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Tue, 10 Oct 2023 08:53:30 -0600 Subject: [PATCH 22/41] Make vim visual block work better --- crates/editor/src/display_map.rs | 11 +-- crates/editor/src/element.rs | 2 +- crates/editor/src/movement.rs | 2 - crates/editor/src/selections_collection.rs | 6 +- crates/text/src/selection.rs | 1 - crates/vim/src/motion.rs | 1 - crates/vim/src/test.rs | 1 + crates/vim/src/visual.rs | 73 ++++++++++++++----- .../test_visual_block_issue_2123.json | 5 ++ .../vim/test_data/test_wrapped_motions.json | 15 ++++ 10 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 crates/vim/test_data/test_visual_block_issue_2123.json create mode 100644 crates/vim/test_data/test_wrapped_motions.json diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 0f2b5665c6eff88a631e630b4e6751d5e91b4431..a92c46e072e00cfe7cda9a091a5fe94e0e88cfac 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -509,11 +509,12 @@ impl DisplaySnapshot { pub fn highlighted_chunks<'a>( &'a self, display_rows: Range, + language_aware: bool, style: &'a EditorStyle, ) -> impl Iterator> { self.chunks( display_rows, - true, + language_aware, Some(style.theme.hint), Some(style.theme.suggestion), ) @@ -562,7 +563,7 @@ impl DisplaySnapshot { }) } - fn layout_line_for_row( + pub fn lay_out_line_for_row( &self, display_row: u32, TextLayoutDetails { @@ -575,7 +576,7 @@ impl DisplaySnapshot { let mut line = String::new(); let range = display_row..display_row + 1; - for chunk in self.highlighted_chunks(range, editor_style) { + for chunk in self.highlighted_chunks(range, false, editor_style) { line.push_str(chunk.chunk); let text_style = if let Some(style) = chunk.style { @@ -607,7 +608,7 @@ impl DisplaySnapshot { display_point: DisplayPoint, text_layout_details: &TextLayoutDetails, ) -> f32 { - let layout_line = self.layout_line_for_row(display_point.row(), text_layout_details); + let layout_line = self.lay_out_line_for_row(display_point.row(), text_layout_details); layout_line.x_for_index(display_point.column() as usize) } @@ -617,7 +618,7 @@ impl DisplaySnapshot { x_coordinate: f32, text_layout_details: &TextLayoutDetails, ) -> u32 { - let layout_line = self.layout_line_for_row(display_row, text_layout_details); + let layout_line = self.lay_out_line_for_row(display_row, text_layout_details); layout_line.closest_index_for_x(x_coordinate) as u32 } diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index 30015eb760d98474d420a41ca13bc71c691d95fd..39d91fe8997e0378be2abcc875f5b238865cb0fc 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -1583,7 +1583,7 @@ impl EditorElement { .collect() } else { let style = &self.style; - let chunks = snapshot.highlighted_chunks(rows.clone(), style); + let chunks = snapshot.highlighted_chunks(rows.clone(), true, style); LineWithInvisibles::from_chunks( chunks, diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 7e75ae5e5d22c3f114c936c649d8be367d247f55..0305d853e6f433fa207ca821dfc8293a385300ae 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -107,7 +107,6 @@ pub fn up_by_rows( SelectionGoal::HorizontalPosition(x) => x, SelectionGoal::WrappedHorizontalPosition((_, x)) => x, SelectionGoal::HorizontalRange { end, .. } => end, - SelectionGoal::WrappedHorizontalRange { end: (_, end), .. } => end, _ => map.x_for_point(start, text_layout_details), }; @@ -144,7 +143,6 @@ pub fn down_by_rows( SelectionGoal::HorizontalPosition(x) => x, SelectionGoal::WrappedHorizontalPosition((_, x)) => x, SelectionGoal::HorizontalRange { end, .. } => end, - SelectionGoal::WrappedHorizontalRange { end: (_, end), .. } => end, _ => map.x_for_point(start, text_layout_details), }; diff --git a/crates/editor/src/selections_collection.rs b/crates/editor/src/selections_collection.rs index 2fa8ffe408742f6e1a3d9c80dd8d124b2ddaa1ec..148604bd23f64c41ddd639414355aa37c2a5ca6a 100644 --- a/crates/editor/src/selections_collection.rs +++ b/crates/editor/src/selections_collection.rs @@ -313,10 +313,12 @@ impl SelectionsCollection { let is_empty = positions.start == positions.end; let line_len = display_map.line_len(row); - let start_col = display_map.column_for_x(row, positions.start, text_layout_details); + let layed_out_line = display_map.lay_out_line_for_row(row, &text_layout_details); + + let start_col = layed_out_line.closest_index_for_x(positions.start) as u32; if start_col < line_len || (is_empty && start_col == line_len) { let start = DisplayPoint::new(row, start_col); - let end_col = display_map.column_for_x(row, positions.end, text_layout_details); + let end_col = layed_out_line.closest_index_for_x(positions.end) as u32; let end = DisplayPoint::new(row, end_col); Some(Selection { diff --git a/crates/text/src/selection.rs b/crates/text/src/selection.rs index e127083112caef9c8762179fe2c8a2cac276388b..480cb99d747783b7c7bfc100af8b57401781a984 100644 --- a/crates/text/src/selection.rs +++ b/crates/text/src/selection.rs @@ -8,7 +8,6 @@ pub enum SelectionGoal { HorizontalPosition(f32), HorizontalRange { start: f32, end: f32 }, WrappedHorizontalPosition((u32, f32)), - WrappedHorizontalRange { start: (u32, f32), end: (u32, f32) }, } #[derive(Clone, Debug, PartialEq)] diff --git a/crates/vim/src/motion.rs b/crates/vim/src/motion.rs index 36514f8cc46e046934529a73a7aa185d7d108148..e8d954bc1321d3f518680c45c938aca51c8a038b 100644 --- a/crates/vim/src/motion.rs +++ b/crates/vim/src/motion.rs @@ -568,7 +568,6 @@ fn up_down_buffer_rows( let (goal_wrap, goal_x) = match goal { SelectionGoal::WrappedHorizontalPosition((row, x)) => (row, x), - SelectionGoal::WrappedHorizontalRange { end: (row, x), .. } => (row, x), SelectionGoal::HorizontalRange { end, .. } => (select_nth_wrapped_row, end), SelectionGoal::HorizontalPosition(x) => (select_nth_wrapped_row, x), _ => { diff --git a/crates/vim/src/test.rs b/crates/vim/src/test.rs index d6d9a1936018fb08df149609d23f0935d991868a..3c4867921215d9355dbceb08ba07b925445d1860 100644 --- a/crates/vim/src/test.rs +++ b/crates/vim/src/test.rs @@ -653,6 +653,7 @@ async fn test_selection_goal(cx: &mut gpui::TestAppContext) { .await; } +#[gpui::test] async fn test_wrapped_motions(cx: &mut gpui::TestAppContext) { let mut cx = NeovimBackedTestContext::new(cx).await; diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index ac4c5478a1d924def738fbc711aee3c3bb12c509..f4a450c932452810aadbc514bdc97e5e329de200 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -145,16 +145,18 @@ pub fn visual_block_motion( let map = &s.display_map(); let mut head = s.newest_anchor().head().to_display_point(map); let mut tail = s.oldest_anchor().tail().to_display_point(map); + dbg!(head, tail); + dbg!(s.newest_anchor().goal); let (start, end) = match s.newest_anchor().goal { SelectionGoal::HorizontalRange { start, end } if preserve_goal => (start, end), - SelectionGoal::HorizontalPosition(start) if preserve_goal => (start, start + 10.0), + SelectionGoal::HorizontalPosition(start) if preserve_goal => (start, start), _ => ( map.x_for_point(tail, &text_layout_details), map.x_for_point(head, &text_layout_details), ), }; - let goal = SelectionGoal::HorizontalRange { start, end }; + let mut goal = SelectionGoal::HorizontalRange { start, end }; let was_reversed = tail.column() > head.column(); if !was_reversed && !preserve_goal { @@ -179,35 +181,44 @@ pub fn visual_block_motion( let positions = if is_reversed { map.x_for_point(head, &text_layout_details)..map.x_for_point(tail, &text_layout_details) } else if head.column() == tail.column() { - map.x_for_point(head, &text_layout_details) - ..map.x_for_point(head, &text_layout_details) + 10.0 + let head_forward = movement::saturating_right(map, head); + map.x_for_point(head, &text_layout_details)..map.x_for_point(head, &text_layout_details) } else { map.x_for_point(tail, &text_layout_details)..map.x_for_point(head, &text_layout_details) }; + if !preserve_goal { + goal = SelectionGoal::HorizontalRange { + start: positions.start, + end: positions.end, + }; + } + let mut selections = Vec::new(); let mut row = tail.row(); loop { - let start = map.clip_point( - DisplayPoint::new( - row, - map.column_for_x(row, positions.start, &text_layout_details), - ), - Bias::Left, + let layed_out_line = map.lay_out_line_for_row(row, &text_layout_details); + let start = DisplayPoint::new( + row, + layed_out_line.closest_index_for_x(positions.start) as u32, ); - let end = map.clip_point( - DisplayPoint::new( - row, - map.column_for_x(row, positions.end, &text_layout_details), - ), - Bias::Left, + let mut end = DisplayPoint::new( + row, + layed_out_line.closest_index_for_x(positions.end) as u32, ); + if end <= start { + if start.column() == map.line_len(start.row()) { + end = start; + } else { + end = movement::saturating_right(map, start); + } + } + if positions.start - <= map.x_for_point( - DisplayPoint::new(row, map.line_len(row)), - &text_layout_details, - ) + <= + //map.x_for_point(DisplayPoint::new(row, map.line_len(row)), &text_layout_details) + layed_out_line.width() { let selection = Selection { id: s.new_selection_id(), @@ -915,6 +926,28 @@ mod test { .await; } + #[gpui::test] + async fn test_visual_block_issue_2123(cx: &mut gpui::TestAppContext) { + let mut cx = NeovimBackedTestContext::new(cx).await; + + cx.set_shared_state(indoc! { + "The ˇquick brown + fox jumps over + the lazy dog + " + }) + .await; + cx.simulate_shared_keystrokes(["ctrl-v", "right", "down"]) + .await; + cx.assert_shared_state(indoc! { + "The «quˇ»ick brown + fox «juˇ»mps over + the lazy dog + " + }) + .await; + } + #[gpui::test] async fn test_visual_block_insert(cx: &mut gpui::TestAppContext) { let mut cx = NeovimBackedTestContext::new(cx).await; diff --git a/crates/vim/test_data/test_visual_block_issue_2123.json b/crates/vim/test_data/test_visual_block_issue_2123.json new file mode 100644 index 0000000000000000000000000000000000000000..0f48bcc8904f8aabc0b5df1e92f22b5c29fd6166 --- /dev/null +++ b/crates/vim/test_data/test_visual_block_issue_2123.json @@ -0,0 +1,5 @@ +{"Put":{"state":"The ˇquick brown\nfox jumps over\nthe lazy dog\n"}} +{"Key":"ctrl-v"} +{"Key":"right"} +{"Key":"down"} +{"Get":{"state":"The «quˇ»ick brown\nfox «juˇ»mps over\nthe lazy dog\n","mode":"VisualBlock"}} diff --git a/crates/vim/test_data/test_wrapped_motions.json b/crates/vim/test_data/test_wrapped_motions.json new file mode 100644 index 0000000000000000000000000000000000000000..195a58f6b5cb94b94a65730e1995760e61b8c3ec --- /dev/null +++ b/crates/vim/test_data/test_wrapped_motions.json @@ -0,0 +1,15 @@ +{"SetOption":{"value":"wrap"}} +{"SetOption":{"value":"columns=12"}} +{"Put":{"state":"aaˇaa\n😃😃"}} +{"Key":"j"} +{"Get":{"state":"aaaa\n😃ˇ😃","mode":"Normal"}} +{"Put":{"state":"123456789012aaˇaa\n123456789012😃😃"}} +{"Key":"j"} +{"Get":{"state":"123456789012aaaa\n123456789012😃ˇ😃","mode":"Normal"}} +{"Put":{"state":"123456789012aaˇaa\n123456789012😃😃"}} +{"Key":"j"} +{"Get":{"state":"123456789012aaaa\n123456789012😃ˇ😃","mode":"Normal"}} +{"Put":{"state":"123456789012aaaaˇaaaaaaaa123456789012\nwow\n123456789012😃😃😃😃😃😃123456789012"}} +{"Key":"j"} +{"Key":"j"} +{"Get":{"state":"123456789012aaaaaaaaaaaa123456789012\nwow\n123456789012😃😃ˇ😃😃😃😃123456789012","mode":"Normal"}} From 138fa45ecb6293e60174091a477d2d1bd1ba8958 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Wed, 18 Oct 2023 22:23:38 -0600 Subject: [PATCH 23/41] recert config change --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index e22bdb0f2c70a1ffda714674253cc533e9e7c1d1..9da6b3be080072d89d16a199e2d60d527eeacd07 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,4 +3,4 @@ xtask = "run --package xtask --" [build] # v0 mangling scheme provides more detailed backtraces around closures -rustflags = ["-C", "symbol-mangling-version=v0", "-C", "link-arg=-fuse-ld=/opt/homebrew/Cellar/llvm/16.0.6/bin/ld64.lld"] +rustflags = ["-C", "symbol-mangling-version=v0"] From 3eb8aa80852497f2e57107ff88715f5ef8bf3f49 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Wed, 18 Oct 2023 22:39:25 -0600 Subject: [PATCH 24/41] Refactor TextLayoutDetails construction --- crates/editor/src/display_map.rs | 2 +- crates/editor/src/editor.rs | 26 +++++++++++------ crates/editor/src/movement.rs | 16 ++--------- crates/vim/src/normal.rs | 6 ++-- crates/vim/src/normal/change.rs | 2 +- crates/vim/src/normal/delete.rs | 6 ++-- crates/vim/src/normal/paste.rs | 8 ++---- crates/vim/src/normal/substitute.rs | 4 +-- crates/vim/src/normal/yank.rs | 3 +- crates/vim/src/visual.rs | 43 +++++++++++++---------------- 10 files changed, 52 insertions(+), 64 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index a92c46e072e00cfe7cda9a091a5fe94e0e88cfac..6e179950f357166f90ae1744590d9b8e4024ed4c 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -1288,7 +1288,7 @@ pub mod tests { cx.update_window(window, |cx| { let text_layout_details = - editor.read_with(cx, |editor, cx| TextLayoutDetails::new(editor, cx)); + editor.read_with(cx, |editor, cx| editor.text_layout_details(cx)); let font_cache = cx.font_cache().clone(); diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 88db2f1dfe12ac069943c3675ee6d95197e7d186..01b060fdb64ebb9007709ce1e104cc23412af7d7 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -3065,6 +3065,14 @@ impl Editor { .collect() } + pub fn text_layout_details(&self, cx: &WindowContext) -> TextLayoutDetails { + TextLayoutDetails { + font_cache: cx.font_cache().clone(), + text_layout_cache: cx.text_layout_cache().clone(), + editor_style: self.style(cx), + } + } + fn splice_inlay_hints( &self, to_remove: Vec, @@ -4988,7 +4996,7 @@ impl Editor { } pub fn transpose(&mut self, _: &Transpose, cx: &mut ViewContext) { - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.transact(cx, |this, cx| { let edits = this.change_selections(Some(Autoscroll::fit()), cx, |s| { let mut edits: Vec<(Range, String)> = Default::default(); @@ -5279,7 +5287,7 @@ impl Editor { return; } - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { let line_mode = s.line_mode; @@ -5321,7 +5329,7 @@ impl Editor { Autoscroll::fit() }; - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(autoscroll), cx, |s| { let line_mode = s.line_mode; @@ -5343,7 +5351,7 @@ impl Editor { } pub fn select_up(&mut self, _: &SelectUp, cx: &mut ViewContext) { - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { s.move_heads_with(|map, head, goal| { movement::up(map, head, goal, false, &text_layout_details) @@ -5359,7 +5367,7 @@ impl Editor { return; } - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { @@ -5409,7 +5417,7 @@ impl Editor { Autoscroll::fit() }; - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(autoscroll), cx, |s| { let line_mode = s.line_mode; s.move_with(|map, selection| { @@ -5430,7 +5438,7 @@ impl Editor { } pub fn select_down(&mut self, _: &SelectDown, cx: &mut ViewContext) { - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.change_selections(Some(Autoscroll::fit()), cx, |s| { s.move_heads_with(|map, head, goal| { movement::down(map, head, goal, false, &text_layout_details) @@ -5953,7 +5961,7 @@ impl Editor { fn add_selection(&mut self, above: bool, cx: &mut ViewContext) { let display_map = self.display_map.update(cx, |map, cx| map.snapshot(cx)); let mut selections = self.selections.all::(cx); - let text_layout_details = TextLayoutDetails::new(self, cx); + let text_layout_details = self.text_layout_details(cx); let mut state = self.add_selections_state.take().unwrap_or_else(|| { let oldest_selection = selections.iter().min_by_key(|s| s.id).unwrap().clone(); let range = oldest_selection.display_range(&display_map).sorted(); @@ -6315,7 +6323,7 @@ impl Editor { } pub fn toggle_comments(&mut self, action: &ToggleComments, cx: &mut ViewContext) { - let text_layout_details = TextLayoutDetails::new(&self, cx); + let text_layout_details = &self.text_layout_details(cx); self.transact(cx, |this, cx| { let mut selections = this.selections.all::(cx); let mut edits = Vec::new(); diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 0305d853e6f433fa207ca821dfc8293a385300ae..62fd931eccc101c977aea3d1533b5853bd023972 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -1,6 +1,6 @@ use super::{Bias, DisplayPoint, DisplaySnapshot, SelectionGoal, ToDisplayPoint}; -use crate::{char_kind, CharKind, Editor, EditorStyle, ToOffset, ToPoint}; -use gpui::{FontCache, TextLayoutCache, WindowContext}; +use crate::{char_kind, CharKind, EditorStyle, ToOffset, ToPoint}; +use gpui::{FontCache, TextLayoutCache}; use language::Point; use std::{ops::Range, sync::Arc}; @@ -18,16 +18,6 @@ pub struct TextLayoutDetails { pub editor_style: EditorStyle, } -impl TextLayoutDetails { - pub fn new(editor: &Editor, cx: &WindowContext) -> TextLayoutDetails { - TextLayoutDetails { - font_cache: cx.font_cache().clone(), - text_layout_cache: cx.text_layout_cache().clone(), - editor_style: editor.style(cx), - } - } -} - pub fn left(map: &DisplaySnapshot, mut point: DisplayPoint) -> DisplayPoint { if point.column() > 0 { *point.column_mut() -= 1; @@ -743,7 +733,7 @@ mod tests { let window = cx.window.clone(); cx.update_window(window, |cx| { let text_layout_details = - editor.read_with(cx, |editor, cx| TextLayoutDetails::new(editor, cx)); + editor.read_with(cx, |editor, cx| editor.text_layout_details(cx)); let family_id = cx .font_cache() diff --git a/crates/vim/src/normal.rs b/crates/vim/src/normal.rs index 0e883cd758472a3395571521a8169efd6bae8270..1277d58b419b4c9e7af792a7dd528ede269adc79 100644 --- a/crates/vim/src/normal.rs +++ b/crates/vim/src/normal.rs @@ -18,7 +18,7 @@ use crate::{ Vim, }; use collections::HashSet; -use editor::{movement::TextLayoutDetails, scroll::autoscroll::Autoscroll}; +use editor::scroll::autoscroll::Autoscroll; use editor::{Bias, DisplayPoint}; use gpui::{actions, AppContext, ViewContext, WindowContext}; use language::SelectionGoal; @@ -177,7 +177,7 @@ pub(crate) fn move_cursor( cx: &mut WindowContext, ) { vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.change_selections(Some(Autoscroll::fit()), cx, |s| { s.move_cursors_with(|map, cursor, goal| { motion @@ -280,7 +280,7 @@ fn insert_line_below(_: &mut Workspace, _: &InsertLineBelow, cx: &mut ViewContex vim.start_recording(cx); vim.switch_mode(Mode::Insert, false, cx); vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.transact(cx, |editor, cx| { let (map, old_selections) = editor.selections.all_display(cx); diff --git a/crates/vim/src/normal/change.rs b/crates/vim/src/normal/change.rs index 042897ec3c78c509353b13a1caffc0f9e8983f6b..bf2a25a98d5ccf622a7b802b84962d6aed6bf308 100644 --- a/crates/vim/src/normal/change.rs +++ b/crates/vim/src/normal/change.rs @@ -20,7 +20,7 @@ pub fn change_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &m | Motion::StartOfLine { .. } ); vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.transact(cx, |editor, cx| { // We are swapping to insert mode anyway. Just set the line end clipping behavior now editor.set_clip_at_line_ends(false, cx); diff --git a/crates/vim/src/normal/delete.rs b/crates/vim/src/normal/delete.rs index 1ad91ff308376aeecd01deee2eea89171aa701ea..77e0e47be5954c4a79182c835d6d221f6195981d 100644 --- a/crates/vim/src/normal/delete.rs +++ b/crates/vim/src/normal/delete.rs @@ -1,15 +1,13 @@ use crate::{motion::Motion, object::Object, utils::copy_selections_content, Vim}; use collections::{HashMap, HashSet}; -use editor::{ - display_map::ToDisplayPoint, movement::TextLayoutDetails, scroll::autoscroll::Autoscroll, Bias, -}; +use editor::{display_map::ToDisplayPoint, scroll::autoscroll::Autoscroll, Bias}; use gpui::WindowContext; use language::Point; pub fn delete_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &mut WindowContext) { vim.stop_recording(); vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); let mut original_columns: HashMap<_, _> = Default::default(); diff --git a/crates/vim/src/normal/paste.rs b/crates/vim/src/normal/paste.rs index 7cb5261c493eefb0b5bfea9b4ad12cd57eae0e51..6141e7c66f29d192aac827a2fb8cccd053e1bee6 100644 --- a/crates/vim/src/normal/paste.rs +++ b/crates/vim/src/normal/paste.rs @@ -1,10 +1,8 @@ use std::{borrow::Cow, cmp}; use editor::{ - display_map::ToDisplayPoint, - movement::{self, TextLayoutDetails}, - scroll::autoscroll::Autoscroll, - ClipboardSelection, DisplayPoint, + display_map::ToDisplayPoint, movement, scroll::autoscroll::Autoscroll, ClipboardSelection, + DisplayPoint, }; use gpui::{impl_actions, AppContext, ViewContext}; use language::{Bias, SelectionGoal}; @@ -32,7 +30,7 @@ fn paste(_: &mut Workspace, action: &Paste, cx: &mut ViewContext) { Vim::update(cx, |vim, cx| { vim.record_current_action(cx); vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); diff --git a/crates/vim/src/normal/substitute.rs b/crates/vim/src/normal/substitute.rs index e244e1d5d64de60c4b40a15c26f269555ae42444..f0369a89bfffabe3283bd138661ad75168bc3ef2 100644 --- a/crates/vim/src/normal/substitute.rs +++ b/crates/vim/src/normal/substitute.rs @@ -1,4 +1,4 @@ -use editor::movement::{self, TextLayoutDetails}; +use editor::movement; use gpui::{actions, AppContext, WindowContext}; use language::Point; use workspace::Workspace; @@ -32,7 +32,7 @@ pub fn substitute(vim: &mut Vim, count: Option, line_mode: bool, cx: &mut vim.update_active_editor(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); editor.transact(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.change_selections(None, cx, |s| { s.move_with(|map, selection| { if selection.start == selection.end { diff --git a/crates/vim/src/normal/yank.rs b/crates/vim/src/normal/yank.rs index b50fcdf7ecbc9691b3e230fbf9d4ba2419c7bfd8..33833500fabc7c42e946aa1a4c790e09cc233744 100644 --- a/crates/vim/src/normal/yank.rs +++ b/crates/vim/src/normal/yank.rs @@ -1,11 +1,10 @@ use crate::{motion::Motion, object::Object, utils::copy_selections_content, Vim}; use collections::HashMap; -use editor::movement::TextLayoutDetails; use gpui::WindowContext; pub fn yank_motion(vim: &mut Vim, motion: Motion, times: Option, cx: &mut WindowContext) { vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.transact(cx, |editor, cx| { editor.set_clip_at_line_ends(false, cx); let mut original_positions: HashMap<_, _> = Default::default(); diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index f4a450c932452810aadbc514bdc97e5e329de200..3157374b246941676cb7f77f3405c9b7a065de64 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -4,7 +4,7 @@ use std::{cmp, sync::Arc}; use collections::HashMap; use editor::{ display_map::{DisplaySnapshot, ToDisplayPoint}, - movement::{self, TextLayoutDetails}, + movement, scroll::autoscroll::Autoscroll, Bias, DisplayPoint, Editor, }; @@ -57,7 +57,7 @@ pub fn init(cx: &mut AppContext) { pub fn visual_motion(motion: Motion, times: Option, cx: &mut WindowContext) { Vim::update(cx, |vim, cx| { vim.update_active_editor(cx, |editor, cx| { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); if vim.state().mode == Mode::VisualBlock && !matches!( motion, @@ -140,25 +140,23 @@ pub fn visual_block_motion( SelectionGoal, ) -> Option<(DisplayPoint, SelectionGoal)>, ) { - let text_layout_details = TextLayoutDetails::new(editor, cx); + let text_layout_details = editor.text_layout_details(cx); editor.change_selections(Some(Autoscroll::fit()), cx, |s| { let map = &s.display_map(); let mut head = s.newest_anchor().head().to_display_point(map); let mut tail = s.oldest_anchor().tail().to_display_point(map); - dbg!(head, tail); - dbg!(s.newest_anchor().goal); + + let mut head_x = map.x_for_point(head, &text_layout_details); + let mut tail_x = map.x_for_point(tail, &text_layout_details); let (start, end) = match s.newest_anchor().goal { SelectionGoal::HorizontalRange { start, end } if preserve_goal => (start, end), SelectionGoal::HorizontalPosition(start) if preserve_goal => (start, start), - _ => ( - map.x_for_point(tail, &text_layout_details), - map.x_for_point(head, &text_layout_details), - ), + _ => (tail_x, head_x), }; let mut goal = SelectionGoal::HorizontalRange { start, end }; - let was_reversed = tail.column() > head.column(); + let was_reversed = head_x > tail_x; if !was_reversed && !preserve_goal { head = movement::saturating_left(map, head); } @@ -167,24 +165,25 @@ pub fn visual_block_motion( return; }; head = new_head; + head_x = map.x_for_point(head, &text_layout_details); - let is_reversed = tail.column() > head.column(); + let is_reversed = tail_x > head_x; if was_reversed && !is_reversed { - tail = movement::left(map, tail) + tail = movement::left(map, tail); + tail_x = map.x_for_point(tail, &text_layout_details); } else if !was_reversed && is_reversed { - tail = movement::right(map, tail) + tail = movement::right(map, tail); + tail_x = map.x_for_point(tail, &text_layout_details); } if !is_reversed && !preserve_goal { - head = movement::saturating_right(map, head) + head = movement::saturating_right(map, head); + head_x = map.x_for_point(head, &text_layout_details); } let positions = if is_reversed { - map.x_for_point(head, &text_layout_details)..map.x_for_point(tail, &text_layout_details) - } else if head.column() == tail.column() { - let head_forward = movement::saturating_right(map, head); - map.x_for_point(head, &text_layout_details)..map.x_for_point(head, &text_layout_details) + head_x..tail_x } else { - map.x_for_point(tail, &text_layout_details)..map.x_for_point(head, &text_layout_details) + tail_x..head_x }; if !preserve_goal { @@ -215,11 +214,7 @@ pub fn visual_block_motion( } } - if positions.start - <= - //map.x_for_point(DisplayPoint::new(row, map.line_len(row)), &text_layout_details) - layed_out_line.width() - { + if positions.start <= layed_out_line.width() { let selection = Selection { id: s.new_selection_id(), start: start.to_point(map), From 36731ba6b3a046ce28a9652d7fe6fbd646914d29 Mon Sep 17 00:00:00 2001 From: Julia Date: Thu, 19 Oct 2023 01:54:42 -0400 Subject: [PATCH 25/41] Magic incantations for Tailwind autocomplete in Svelte, Elixir ~H, Heex --- crates/zed/src/languages.rs | 17 ++++++++++++----- crates/zed/src/languages/elixir/config.toml | 5 +++++ crates/zed/src/languages/heex/config.toml | 5 +++++ crates/zed/src/languages/heex/overrides.scm | 4 ++++ crates/zed/src/languages/svelte/config.toml | 7 ++++--- crates/zed/src/languages/svelte/overrides.scm | 7 +++++++ crates/zed/src/languages/tailwind.rs | 3 +++ 7 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 crates/zed/src/languages/heex/overrides.scm create mode 100644 crates/zed/src/languages/svelte/overrides.scm diff --git a/crates/zed/src/languages.rs b/crates/zed/src/languages.rs index caf3cbf7c948ec239b8e620d30587f79c5e9b291..e5b3dfdcae84ed35fdf15d1dd53a75c19883fb36 100644 --- a/crates/zed/src/languages.rs +++ b/crates/zed/src/languages.rs @@ -76,7 +76,10 @@ pub fn init( elixir::ElixirLspSetting::ElixirLs => language( "elixir", tree_sitter_elixir::language(), - vec![Arc::new(elixir::ElixirLspAdapter)], + vec![ + Arc::new(elixir::ElixirLspAdapter), + Arc::new(tailwind::TailwindLspAdapter::new(node_runtime.clone())), + ], ), elixir::ElixirLspSetting::NextLs => language( "elixir", @@ -101,7 +104,10 @@ pub fn init( language( "heex", tree_sitter_heex::language(), - vec![Arc::new(elixir::ElixirLspAdapter)], + vec![ + Arc::new(elixir::ElixirLspAdapter), + Arc::new(tailwind::TailwindLspAdapter::new(node_runtime.clone())), + ], ); language( "json", @@ -184,9 +190,10 @@ pub fn init( language( "svelte", tree_sitter_svelte::language(), - vec![Arc::new(svelte::SvelteLspAdapter::new( - node_runtime.clone(), - ))], + vec![ + Arc::new(svelte::SvelteLspAdapter::new(node_runtime.clone())), + Arc::new(tailwind::TailwindLspAdapter::new(node_runtime.clone())), + ], ); language( "php", diff --git a/crates/zed/src/languages/elixir/config.toml b/crates/zed/src/languages/elixir/config.toml index 05c126e9da4ccfbe9c3beed134ed1b374cbcecab..8983c0e49b465c07ca3c0dd37a326c2244f52795 100644 --- a/crates/zed/src/languages/elixir/config.toml +++ b/crates/zed/src/languages/elixir/config.toml @@ -9,3 +9,8 @@ brackets = [ { start = "\"", end = "\"", close = true, newline = false, not_in = ["string", "comment"] }, { start = "'", end = "'", close = true, newline = false, not_in = ["string", "comment"] }, ] +scope_opt_in_language_servers = ["tailwindcss-language-server"] + +[overrides.string] +word_characters = ["-"] +opt_into_language_servers = ["tailwindcss-language-server"] diff --git a/crates/zed/src/languages/heex/config.toml b/crates/zed/src/languages/heex/config.toml index c9f952ee3c4f2813dcaf0e94fd3d5858e78d0922..74cb5ac9ff5df179bf190aac8f843fd82820e29a 100644 --- a/crates/zed/src/languages/heex/config.toml +++ b/crates/zed/src/languages/heex/config.toml @@ -5,3 +5,8 @@ brackets = [ { start = "<", end = ">", close = true, newline = true }, ] block_comment = ["<%!-- ", " --%>"] +scope_opt_in_language_servers = ["tailwindcss-language-server"] + +[overrides.string] +word_characters = ["-"] +opt_into_language_servers = ["tailwindcss-language-server"] diff --git a/crates/zed/src/languages/heex/overrides.scm b/crates/zed/src/languages/heex/overrides.scm new file mode 100644 index 0000000000000000000000000000000000000000..35ac9fe15fca6a727463c1408fa01ffbbee2c2f6 --- /dev/null +++ b/crates/zed/src/languages/heex/overrides.scm @@ -0,0 +1,4 @@ +[ + (attribute_value) + (quoted_attribute_value) +] @string diff --git a/crates/zed/src/languages/svelte/config.toml b/crates/zed/src/languages/svelte/config.toml index 41bb21a45d54db9944d50b6a93d8fbda3d34fe41..8a07b012c7c0e25c3f11e54b2d905d27179cd233 100644 --- a/crates/zed/src/languages/svelte/config.toml +++ b/crates/zed/src/languages/svelte/config.toml @@ -12,7 +12,8 @@ brackets = [ { start = "`", end = "`", close = true, newline = false, not_in = ["string"] }, { start = "/*", end = " */", close = true, newline = false, not_in = ["string", "comment"] }, ] +scope_opt_in_language_servers = ["tailwindcss-language-server"] -[overrides.element] -line_comment = { remove = true } -block_comment = ["{/* ", " */}"] +[overrides.string] +word_characters = ["-"] +opt_into_language_servers = ["tailwindcss-language-server"] diff --git a/crates/zed/src/languages/svelte/overrides.scm b/crates/zed/src/languages/svelte/overrides.scm new file mode 100644 index 0000000000000000000000000000000000000000..2a76410297833c9f1884f5e93c7851a38fc0b2f6 --- /dev/null +++ b/crates/zed/src/languages/svelte/overrides.scm @@ -0,0 +1,7 @@ +(comment) @comment + +[ + (raw_text) + (attribute_value) + (quoted_attribute_value) +] @string diff --git a/crates/zed/src/languages/tailwind.rs b/crates/zed/src/languages/tailwind.rs index 8e81f728dc1f39de023a8292018518ad772d3b3a..ef19b858be238ce542a137070d2daee1f0bc973c 100644 --- a/crates/zed/src/languages/tailwind.rs +++ b/crates/zed/src/languages/tailwind.rs @@ -123,6 +123,9 @@ impl LspAdapter for TailwindLspAdapter { ("CSS".to_string(), "css".to_string()), ("JavaScript".to_string(), "javascript".to_string()), ("TSX".to_string(), "typescriptreact".to_string()), + ("Svelte".to_string(), "svelte".to_string()), + ("Elixir".to_string(), "phoenix-heex".to_string()), + ("HEEX".to_string(), "phoenix-heex".to_string()), ] .into_iter(), ) From 40104c06e2caefa3b777d7056ebb06c6be50c06a Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 19 Oct 2023 11:05:47 +0300 Subject: [PATCH 26/41] Sort hint request ranges before asserting to avoid flackiness --- crates/editor/src/inlay_hint_cache.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/editor/src/inlay_hint_cache.rs b/crates/editor/src/inlay_hint_cache.rs index dd75d2bab6664591c1cfaf18225269a8c1f40f3f..6b2712e7bf98fd81f89b6369ddfa7d9465ecec24 100644 --- a/crates/editor/src/inlay_hint_cache.rs +++ b/crates/editor/src/inlay_hint_cache.rs @@ -2138,7 +2138,7 @@ pub mod tests { }); } - #[gpui::test] + #[gpui::test(iterations = 10)] async fn test_large_buffer_inlay_requests_split(cx: &mut gpui::TestAppContext) { init_test(cx, |settings| { settings.defaults.inlay_hints = Some(InlayHintSettings { @@ -2400,11 +2400,13 @@ pub mod tests { )); cx.foreground().run_until_parked(); editor.update(cx, |editor, cx| { - let ranges = lsp_request_ranges.lock().drain(..).collect::>(); + let mut ranges = lsp_request_ranges.lock().drain(..).collect::>(); + ranges.sort_by_key(|r| r.start); + assert_eq!(ranges.len(), 3, "On edit, should scroll to selection and query a range around it: visible + same range above and below. Instead, got query ranges {ranges:?}"); - let visible_query_range = &ranges[0]; - let above_query_range = &ranges[1]; + let above_query_range = &ranges[0]; + let visible_query_range = &ranges[1]; let below_query_range = &ranges[2]; assert!(above_query_range.end.character < visible_query_range.start.character || above_query_range.end.line + 1 == visible_query_range.start.line, "Above range {above_query_range:?} should be before visible range {visible_query_range:?}"); From 19c2df4822db4731760547f9fea6fe4e810c1115 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Thu, 19 Oct 2023 14:33:52 -0400 Subject: [PATCH 27/41] outlined when truncation is taking place in the prompt --- crates/ai/src/templates/file_context.rs | 4 ++++ crates/ai/src/templates/generate.rs | 3 ++- crates/assistant/src/prompts.rs | 2 -- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/ai/src/templates/file_context.rs b/crates/ai/src/templates/file_context.rs index 253d24e4691d52371d2af13e07e160ec3ac6e0f6..1afd61192edc02b153abe8cd00836d67caa42f02 100644 --- a/crates/ai/src/templates/file_context.rs +++ b/crates/ai/src/templates/file_context.rs @@ -131,6 +131,10 @@ impl PromptTemplate for FileContext { )?; writeln!(prompt, "```{language_name}\n{context}\n```").unwrap(); + if truncated { + writeln!(prompt, "Note the content has been truncated and only represents a portion of the file.").unwrap(); + } + if let Some(selected_range) = &args.selected_range { let start = selected_range.start.to_offset(buffer); let end = selected_range.end.to_offset(buffer); diff --git a/crates/ai/src/templates/generate.rs b/crates/ai/src/templates/generate.rs index 34d874cc4128ccae034a0ecf3beace159bbec1ac..19334340c8e5d302ef7e07f24eedf820670ea9e3 100644 --- a/crates/ai/src/templates/generate.rs +++ b/crates/ai/src/templates/generate.rs @@ -50,7 +50,8 @@ impl PromptTemplate for GenerateInlineContent { .unwrap(); } else { writeln!(prompt, "Modify the user's selected {content_type} based upon the users prompt: '{user_prompt}'").unwrap(); - writeln!(prompt, "You MUST reply with only the adjusted {content_type} (within the '<|START|' and '|END|>' spans), not the entire file.").unwrap(); + writeln!(prompt, "You must reply with only the adjusted {content_type} (within the '<|START|' and '|END|>' spans) not the entire file.").unwrap(); + writeln!(prompt, "Double check that you only return code and not the '<|START|' and '|END|'> spans").unwrap(); } } else { writeln!( diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index c7b52a35405deadc3a9319ea77aea34e1989f273..dffcbc29234d3f24174d1d9a6610045105eae890 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -166,8 +166,6 @@ pub fn generate_content_prompt( let chain = PromptChain::new(args, templates); let (prompt, _) = chain.generate(true)?; - println!("PROMPT: {:?}", &prompt); - anyhow::Ok(prompt) } From 71fb23f769b120f366184a269766c980dacb4be5 Mon Sep 17 00:00:00 2001 From: KCaverly Date: Thu, 19 Oct 2023 15:44:49 -0400 Subject: [PATCH 28/41] decrease temperature for inline assist on code content --- crates/ai/src/completion.rs | 2 ++ crates/ai/src/templates/generate.rs | 2 +- crates/assistant/src/assistant_panel.rs | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/ai/src/completion.rs b/crates/ai/src/completion.rs index 170b2268f9ed1132fad1bfe69194d8cc7a2e91bf..de6ce9da711ee17f9fc072276a499d1769b874ce 100644 --- a/crates/ai/src/completion.rs +++ b/crates/ai/src/completion.rs @@ -53,6 +53,8 @@ pub struct OpenAIRequest { pub model: String, pub messages: Vec, pub stream: bool, + pub stop: Vec, + pub temperature: f32, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] diff --git a/crates/ai/src/templates/generate.rs b/crates/ai/src/templates/generate.rs index 19334340c8e5d302ef7e07f24eedf820670ea9e3..1eeb197f932db0dc13963982e7e8bc983c338db7 100644 --- a/crates/ai/src/templates/generate.rs +++ b/crates/ai/src/templates/generate.rs @@ -78,7 +78,7 @@ impl PromptTemplate for GenerateInlineContent { match file_type { PromptFileType::Code => { - writeln!(prompt, "Always wrap your code in a Markdown block.").unwrap(); + // writeln!(prompt, "Always wrap your code in a Markdown block.").unwrap(); } _ => {} } diff --git a/crates/assistant/src/assistant_panel.rs b/crates/assistant/src/assistant_panel.rs index 4dd4e2a98315c042d74c7ef6bde78200287ab6ad..ca8c54a285d70d3eaa9f1aee09437994708ebdfb 100644 --- a/crates/assistant/src/assistant_panel.rs +++ b/crates/assistant/src/assistant_panel.rs @@ -661,6 +661,19 @@ impl AssistantPanel { None }; + // Higher Temperature increases the randomness of model outputs. + // If Markdown or No Language is Known, increase the randomness for more creative output + // If Code, decrease temperature to get more deterministic outputs + let temperature = if let Some(language) = language_name.clone() { + if language.to_string() != "Markdown".to_string() { + 0.5 + } else { + 1.0 + } + } else { + 1.0 + }; + let user_prompt = user_prompt.to_string(); let snippets = if retrieve_context { @@ -731,10 +744,13 @@ impl AssistantPanel { role: Role::User, content: prompt, }); + let request = OpenAIRequest { model: model.full_name().into(), messages, stream: true, + stop: vec!["|END|>".to_string()], + temperature, }; codegen.update(&mut cx, |codegen, cx| codegen.start(request, cx)); anyhow::Ok(()) @@ -1727,6 +1743,8 @@ impl Conversation { .map(|message| message.to_open_ai_message(self.buffer.read(cx))) .collect(), stream: true, + stop: vec![], + temperature: 1.0, }; let stream = stream_completion(api_key, cx.background().clone(), request); @@ -2011,6 +2029,8 @@ impl Conversation { model: self.model.full_name().to_string(), messages: messages.collect(), stream: true, + stop: vec![], + temperature: 1.0, }; let stream = stream_completion(api_key, cx.background().clone(), request); From 7c3a8a37744d09a4ab265e175ea58f17553d28a5 Mon Sep 17 00:00:00 2001 From: Julia Date: Fri, 20 Oct 2023 00:37:02 -0400 Subject: [PATCH 29/41] Add Tailwind autocomplete in ERB, PHP, and Laravel Blade --- crates/zed/src/languages.rs | 12 ++++++++---- crates/zed/src/languages/erb/config.toml | 1 + crates/zed/src/languages/php/config.toml | 1 + crates/zed/src/languages/tailwind.rs | 23 +++++++++++------------ 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/zed/src/languages.rs b/crates/zed/src/languages.rs index e5b3dfdcae84ed35fdf15d1dd53a75c19883fb36..2398f81c78a8d9cf26c6282694b7353057f59ae9 100644 --- a/crates/zed/src/languages.rs +++ b/crates/zed/src/languages.rs @@ -173,7 +173,10 @@ pub fn init( language( "erb", tree_sitter_embedded_template::language(), - vec![Arc::new(ruby::RubyLanguageServer)], + vec![ + Arc::new(ruby::RubyLanguageServer), + Arc::new(tailwind::TailwindLspAdapter::new(node_runtime.clone())), + ], ); language("scheme", tree_sitter_scheme::language(), vec![]); language("racket", tree_sitter_racket::language(), vec![]); @@ -198,9 +201,10 @@ pub fn init( language( "php", tree_sitter_php::language(), - vec![Arc::new(php::IntelephenseLspAdapter::new( - node_runtime.clone(), - ))], + vec![ + Arc::new(php::IntelephenseLspAdapter::new(node_runtime.clone())), + Arc::new(tailwind::TailwindLspAdapter::new(node_runtime.clone())), + ], ); language("elm", tree_sitter_elm::language(), vec![]); diff --git a/crates/zed/src/languages/erb/config.toml b/crates/zed/src/languages/erb/config.toml index 9cfcef0c8ba8ff4b3a5aedd574f01f6050c5798a..ebc45e9984b63dab1a960f96a0e2004a48a8a412 100644 --- a/crates/zed/src/languages/erb/config.toml +++ b/crates/zed/src/languages/erb/config.toml @@ -5,3 +5,4 @@ brackets = [ { start = "<", end = ">", close = true, newline = true }, ] block_comment = ["<%#", "%>"] +scope_opt_in_language_servers = ["tailwindcss-language-server"] diff --git a/crates/zed/src/languages/php/config.toml b/crates/zed/src/languages/php/config.toml index 60dd2335551bc746ec6b26f2afe039b377e18442..f5ad67c12d2a0722f4033861d96fcecc955c4cd5 100644 --- a/crates/zed/src/languages/php/config.toml +++ b/crates/zed/src/languages/php/config.toml @@ -11,3 +11,4 @@ brackets = [ ] collapsed_placeholder = "/* ... */" word_characters = ["$"] +scope_opt_in_language_servers = ["tailwindcss-language-server"] diff --git a/crates/zed/src/languages/tailwind.rs b/crates/zed/src/languages/tailwind.rs index ef19b858be238ce542a137070d2daee1f0bc973c..296d9a490b1b115f58b518ba26be639be5af4d37 100644 --- a/crates/zed/src/languages/tailwind.rs +++ b/crates/zed/src/languages/tailwind.rs @@ -117,18 +117,17 @@ impl LspAdapter for TailwindLspAdapter { } async fn language_ids(&self) -> HashMap { - HashMap::from_iter( - [ - ("HTML".to_string(), "html".to_string()), - ("CSS".to_string(), "css".to_string()), - ("JavaScript".to_string(), "javascript".to_string()), - ("TSX".to_string(), "typescriptreact".to_string()), - ("Svelte".to_string(), "svelte".to_string()), - ("Elixir".to_string(), "phoenix-heex".to_string()), - ("HEEX".to_string(), "phoenix-heex".to_string()), - ] - .into_iter(), - ) + HashMap::from_iter([ + ("HTML".to_string(), "html".to_string()), + ("CSS".to_string(), "css".to_string()), + ("JavaScript".to_string(), "javascript".to_string()), + ("TSX".to_string(), "typescriptreact".to_string()), + ("Svelte".to_string(), "svelte".to_string()), + ("Elixir".to_string(), "phoenix-heex".to_string()), + ("HEEX".to_string(), "phoenix-heex".to_string()), + ("ERB".to_string(), "erb".to_string()), + ("PHP".to_string(), "php".to_string()), + ]) } fn enabled_formatters(&self) -> Vec { From b596b4153ffbc4b911c6519eacf36256f31b0745 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 10:34:09 -0600 Subject: [PATCH 30/41] Fix test --- crates/editor/src/display_map.rs | 2 +- crates/editor/src/editor_tests.rs | 17 ++++++++--------- crates/editor/src/movement.rs | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 6e179950f357166f90ae1744590d9b8e4024ed4c..184990ea78dc759fa8e1d24f6ae57ac58cc0685c 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -1007,7 +1007,7 @@ pub mod tests { use settings::SettingsStore; use smol::stream::StreamExt; use std::{env, sync::Arc}; - use theme::{SyntaxTheme, Theme}; + use theme::SyntaxTheme; use util::test::{marked_text_ranges, sample_text}; use Bias::*; diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs index 6cc78385a3ea56c84a55c58f705c2784010481e1..664db53e1ab82fe04600c7c2e9876b26e5a7d34c 100644 --- a/crates/editor/src/editor_tests.rs +++ b/crates/editor/src/editor_tests.rs @@ -847,7 +847,6 @@ fn test_move_cursor(cx: &mut TestAppContext) { #[gpui::test] fn test_move_cursor_multibyte(cx: &mut TestAppContext) { - todo!(); init_test(cx, |_| {}); let view = cx @@ -889,6 +888,11 @@ fn test_move_cursor_multibyte(cx: &mut TestAppContext) { ); view.move_down(&MoveDown, cx); + assert_eq!( + view.selections.display_ranges(cx), + &[empty_range(1, "ab⋯e".len())] + ); + view.move_left(&MoveLeft, cx); assert_eq!( view.selections.display_ranges(cx), &[empty_range(1, "ab⋯".len())] @@ -933,22 +937,17 @@ fn test_move_cursor_multibyte(cx: &mut TestAppContext) { view.move_up(&MoveUp, cx); assert_eq!( view.selections.display_ranges(cx), - &[empty_range(0, "ⓐⓑ⋯ⓔ".len())] - ); - view.move_left(&MoveLeft, cx); - assert_eq!( - view.selections.display_ranges(cx), - &[empty_range(0, "ⓐⓑ⋯".len())] + &[empty_range(0, "ⓐⓑ".len())] ); view.move_left(&MoveLeft, cx); assert_eq!( view.selections.display_ranges(cx), - &[empty_range(0, "ⓐⓑ".len())] + &[empty_range(0, "ⓐ".len())] ); view.move_left(&MoveLeft, cx); assert_eq!( view.selections.display_ranges(cx), - &[empty_range(0, "ⓐ".len())] + &[empty_range(0, "".len())] ); }); } diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 8f5d0d802c06cdc1e20745b6db695250520ef5e1..580faf10506f38bb971bf548c3b65d911fb1fd45 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -429,7 +429,6 @@ mod tests { test::{editor_test_context::EditorTestContext, marked_display_snapshot}, Buffer, DisplayMap, ExcerptRange, InlayId, MultiBuffer, }; - use language::language_settings::AllLanguageSettings; use project::Project; use settings::SettingsStore; use util::post_inc; From 1c36134cf994e7cf753e700b15c76cd8f97d2e03 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 11:17:19 -0600 Subject: [PATCH 31/41] Fix single column vim selections --- crates/editor/src/editor_tests.rs | 15 +++++++++++++-- crates/editor/src/movement.rs | 2 ++ crates/gpui/src/text_layout.rs | 6 +++++- crates/vim/src/normal/increment.rs | 12 +++++++++++- crates/vim/src/visual.rs | 8 ++++++-- crates/vim/test_data/test_increment_steps.json | 1 + 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs index 664db53e1ab82fe04600c7c2e9876b26e5a7d34c..8104b0ee9b681b6b16b91047f5531981b1a455a3 100644 --- a/crates/editor/src/editor_tests.rs +++ b/crates/editor/src/editor_tests.rs @@ -851,7 +851,7 @@ fn test_move_cursor_multibyte(cx: &mut TestAppContext) { let view = cx .add_window(|cx| { - let buffer = MultiBuffer::build_simple("ⓐⓑⓒⓓⓔ\nabcde\nαβγδε\n", cx); + let buffer = MultiBuffer::build_simple("ⓐⓑⓒⓓⓔ\nabcde\nαβγδε", cx); build_editor(buffer.clone(), cx) }) .root(cx); @@ -869,7 +869,7 @@ fn test_move_cursor_multibyte(cx: &mut TestAppContext) { true, cx, ); - assert_eq!(view.display_text(cx), "ⓐⓑ⋯ⓔ\nab⋯e\nαβ⋯ε\n"); + assert_eq!(view.display_text(cx), "ⓐⓑ⋯ⓔ\nab⋯e\nαβ⋯ε"); view.move_right(&MoveRight, cx); assert_eq!( @@ -934,6 +934,17 @@ fn test_move_cursor_multibyte(cx: &mut TestAppContext) { view.selections.display_ranges(cx), &[empty_range(1, "ab⋯e".len())] ); + view.move_down(&MoveDown, cx); + assert_eq!( + view.selections.display_ranges(cx), + &[empty_range(2, "αβ⋯ε".len())] + ); + view.move_up(&MoveUp, cx); + assert_eq!( + view.selections.display_ranges(cx), + &[empty_range(1, "ab⋯e".len())] + ); + view.move_up(&MoveUp, cx); assert_eq!( view.selections.display_ranges(cx), diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 580faf10506f38bb971bf548c3b65d911fb1fd45..1bb4154b1f744d24cba6b7ec23d25bf3117590d6 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -147,7 +147,9 @@ pub fn down_by_rows( goal_x = map.x_for_point(point, text_layout_details) } + dbg!(point); let mut clipped_point = map.clip_point(point, Bias::Right); + dbg!(clipped_point); if clipped_point.row() > point.row() { clipped_point = map.clip_point(point, Bias::Left); } diff --git a/crates/gpui/src/text_layout.rs b/crates/gpui/src/text_layout.rs index 7fb87b10df2ce3baf822fbe5a6fddb4955e5f134..0d0b8bebb50f145f8ae68978b4ad15088f1e3f37 100644 --- a/crates/gpui/src/text_layout.rs +++ b/crates/gpui/src/text_layout.rs @@ -302,7 +302,11 @@ impl Line { prev_x = glyph.position.x(); } } - prev_index + if self.width() - x < x - prev_x { + prev_index + 1 + } else { + prev_index + } } pub fn paint( diff --git a/crates/vim/src/normal/increment.rs b/crates/vim/src/normal/increment.rs index 9d62f8ab7b2f5210b4a68f73029b8181902a4bb1..ee70ab1f5d0d569669281b2b2b4f51cc60b6e149 100644 --- a/crates/vim/src/normal/increment.rs +++ b/crates/vim/src/normal/increment.rs @@ -255,8 +255,18 @@ mod test { 4 5"}) .await; - cx.simulate_shared_keystrokes(["shift-g", "ctrl-v", "g", "g", "g", "ctrl-x"]) + + cx.simulate_shared_keystrokes(["shift-g", "ctrl-v", "g", "g"]) + .await; + cx.assert_shared_state(indoc! {" + «1ˇ» + «2ˇ» + «3ˇ» 2 + «4ˇ» + «5ˇ»"}) .await; + + cx.simulate_shared_keystrokes(["g", "ctrl-x"]).await; cx.assert_shared_state(indoc! {" ˇ0 0 diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index 3157374b246941676cb7f77f3405c9b7a065de64..05118e22f8caf5f58a084bbd3eae88550c3959fd 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -169,10 +169,10 @@ pub fn visual_block_motion( let is_reversed = tail_x > head_x; if was_reversed && !is_reversed { - tail = movement::left(map, tail); + tail = movement::saturating_left(map, tail); tail_x = map.x_for_point(tail, &text_layout_details); } else if !was_reversed && is_reversed { - tail = movement::right(map, tail); + tail = movement::saturating_right(map, tail); tail_x = map.x_for_point(tail, &text_layout_details); } if !is_reversed && !preserve_goal { @@ -180,8 +180,12 @@ pub fn visual_block_motion( head_x = map.x_for_point(head, &text_layout_details); } + dbg!(head, head_x, tail, tail_x); + let positions = if is_reversed { head_x..tail_x + } else if head_x == tail_x { + map.x_for_point(movement::saturating_left(map, tail), &text_layout_details)..head_x } else { tail_x..head_x }; diff --git a/crates/vim/test_data/test_increment_steps.json b/crates/vim/test_data/test_increment_steps.json index fffaf1fd299377574f108e69aaefd2eadb3d0fe8..2e8711d1cc5757207405a14c78b8d944f352d664 100644 --- a/crates/vim/test_data/test_increment_steps.json +++ b/crates/vim/test_data/test_increment_steps.json @@ -9,6 +9,7 @@ {"Key":"ctrl-v"} {"Key":"g"} {"Key":"g"} +{"Get":{"state":"«1ˇ»\n«2ˇ»\n«3ˇ» 2\n«4ˇ»\n«5ˇ»","mode":"VisualBlock"}} {"Key":"g"} {"Key":"ctrl-x"} {"Get":{"state":"ˇ0\n0\n0 2\n0\n0","mode":"Normal"}} From c7991ef04cc5102064a9c25473f5b044f6dd96a7 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 11:41:13 -0600 Subject: [PATCH 32/41] Add "workspace: Toggle Vim Mode" Co-Authored-By: Marshall --- crates/vim/src/vim.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/vim/src/vim.rs b/crates/vim/src/vim.rs index aad97c558e2d22e37d349e380b8970720890d642..c34607d1c0c29fb36d64e6868d20ad20d9a84f88 100644 --- a/crates/vim/src/vim.rs +++ b/crates/vim/src/vim.rs @@ -25,7 +25,7 @@ pub use mode_indicator::ModeIndicator; use motion::Motion; use normal::normal_replace; use serde::Deserialize; -use settings::{Setting, SettingsStore}; +use settings::{update_settings_file, Setting, SettingsStore}; use state::{EditorState, Mode, Operator, RecordedSelection, WorkspaceState}; use std::{ops::Range, sync::Arc}; use visual::{visual_block_motion, visual_replace}; @@ -48,6 +48,7 @@ actions!( vim, [Tab, Enter, Object, InnerObject, FindForward, FindBackward] ); +actions!(workspace, [ToggleVimMode]); impl_actions!(vim, [Number, SwitchMode, PushOperator]); #[derive(Copy, Clone, Debug)] @@ -88,6 +89,14 @@ pub fn init(cx: &mut AppContext) { Vim::active_editor_input_ignored("\n".into(), cx) }); + cx.add_action(|workspace: &mut Workspace, _: &ToggleVimMode, cx| { + let fs = workspace.app_state().fs.clone(); + let currently_enabled = settings::get::(cx).0; + update_settings_file::(fs, cx, move |setting| { + *setting = Some(!currently_enabled) + }) + }); + // Any time settings change, update vim mode to match. The Vim struct // will be initialized as disabled by default, so we filter its commands // out when starting up. From 352a554c748feb88b3f5a36a152a7e0863b3bc18 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 11:30:32 -0600 Subject: [PATCH 33/41] Fix bug in vim visual block mode --- crates/gpui/src/text_layout.rs | 6 +----- crates/vim/src/visual.rs | 4 +--- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/gpui/src/text_layout.rs b/crates/gpui/src/text_layout.rs index 0d0b8bebb50f145f8ae68978b4ad15088f1e3f37..7fb87b10df2ce3baf822fbe5a6fddb4955e5f134 100644 --- a/crates/gpui/src/text_layout.rs +++ b/crates/gpui/src/text_layout.rs @@ -302,11 +302,7 @@ impl Line { prev_x = glyph.position.x(); } } - if self.width() - x < x - prev_x { - prev_index + 1 - } else { - prev_index - } + prev_index } pub fn paint( diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index 05118e22f8caf5f58a084bbd3eae88550c3959fd..0e108251e2e75ac34f7543c4c028c602ab1f97a3 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -156,7 +156,7 @@ pub fn visual_block_motion( }; let mut goal = SelectionGoal::HorizontalRange { start, end }; - let was_reversed = head_x > tail_x; + let was_reversed = tail_x > head_x; if !was_reversed && !preserve_goal { head = movement::saturating_left(map, head); } @@ -184,8 +184,6 @@ pub fn visual_block_motion( let positions = if is_reversed { head_x..tail_x - } else if head_x == tail_x { - map.x_for_point(movement::saturating_left(map, tail), &text_layout_details)..head_x } else { tail_x..head_x }; From 43d682f6b887ea80260c9d9591fcecbc0100774d Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 12:46:14 -0600 Subject: [PATCH 34/41] Handle pixel-down to last line when no trailing newline --- crates/editor/src/display_map.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index 184990ea78dc759fa8e1d24f6ae57ac58cc0685c..1d6deb910aa8d55d8f780cdfd8ce169662cfc940 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -574,6 +574,7 @@ impl DisplaySnapshot { ) -> Line { let mut styles = Vec::new(); let mut line = String::new(); + let mut ended_in_newline = false; let range = display_row..display_row + 1; for chunk in self.highlighted_chunks(range, false, editor_style) { @@ -589,6 +590,7 @@ impl DisplaySnapshot { } else { Cow::Borrowed(&editor_style.text) }; + ended_in_newline = chunk.chunk.ends_with("\n"); styles.push(( chunk.chunk.len(), @@ -600,6 +602,22 @@ impl DisplaySnapshot { )); } + // our pixel positioning logic assumes each line ends in \n, + // this is almost always true except for the last line which + // may have no trailing newline. + if !ended_in_newline && display_row == self.max_point().row() { + line.push_str("\n"); + + styles.push(( + "\n".len(), + RunStyle { + font_id: editor_style.text.font_id, + color: editor_style.text_color, + underline: editor_style.text.underline, + }, + )); + } + text_layout_cache.layout_str(&line, editor_style.text.font_size, &styles) } From fae5b1e3912e7924cc3fa3f6c49bb1090668ee6d Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 12:55:41 -0600 Subject: [PATCH 35/41] Fix build columnar selection logic --- crates/editor/src/selections_collection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/editor/src/selections_collection.rs b/crates/editor/src/selections_collection.rs index 148604bd23f64c41ddd639414355aa37c2a5ca6a..4b2dc855c39312fe62c38c621e086601901011e2 100644 --- a/crates/editor/src/selections_collection.rs +++ b/crates/editor/src/selections_collection.rs @@ -316,7 +316,7 @@ impl SelectionsCollection { let layed_out_line = display_map.lay_out_line_for_row(row, &text_layout_details); let start_col = layed_out_line.closest_index_for_x(positions.start) as u32; - if start_col < line_len || (is_empty && start_col == line_len) { + if start_col < line_len || (is_empty && positions.start == layed_out_line.width()) { let start = DisplayPoint::new(row, start_col); let end_col = layed_out_line.closest_index_for_x(positions.end) as u32; let end = DisplayPoint::new(row, end_col); From 31872227f168fbb91af55cf65e4b4079df80e931 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 13:03:21 -0600 Subject: [PATCH 36/41] Remove screen sharing indicator This is now redundant given macOS has the same icon, and it panics when you click on it :D. --- crates/collab_ui/src/collab_ui.rs | 2 - .../collab_ui/src/sharing_status_indicator.rs | 62 ------------------- 2 files changed, 64 deletions(-) delete mode 100644 crates/collab_ui/src/sharing_status_indicator.rs diff --git a/crates/collab_ui/src/collab_ui.rs b/crates/collab_ui/src/collab_ui.rs index 57d6f7b4f6b7e17b9426c75cbac2e9b80491c048..76d4427aea1bb9a61ad3d2c01026ad5c3d35bdbe 100644 --- a/crates/collab_ui/src/collab_ui.rs +++ b/crates/collab_ui/src/collab_ui.rs @@ -8,7 +8,6 @@ mod incoming_call_notification; mod notifications; mod panel_settings; pub mod project_shared_notification; -mod sharing_status_indicator; use call::{report_call_event_for_room, ActiveCall, Room}; use gpui::{ @@ -42,7 +41,6 @@ pub fn init(app_state: &Arc, cx: &mut AppContext) { chat_panel::init(cx); incoming_call_notification::init(&app_state, cx); project_shared_notification::init(&app_state, cx); - sharing_status_indicator::init(cx); cx.add_global_action(toggle_screen_sharing); cx.add_global_action(toggle_mute); diff --git a/crates/collab_ui/src/sharing_status_indicator.rs b/crates/collab_ui/src/sharing_status_indicator.rs deleted file mode 100644 index 6a6acb4b707b5acfb93c1be889523982be4c9c10..0000000000000000000000000000000000000000 --- a/crates/collab_ui/src/sharing_status_indicator.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::toggle_screen_sharing; -use call::ActiveCall; -use gpui::{ - color::Color, - elements::{MouseEventHandler, Svg}, - platform::{Appearance, MouseButton}, - AnyElement, AppContext, Element, Entity, View, ViewContext, -}; -use workspace::WorkspaceSettings; - -pub fn init(cx: &mut AppContext) { - let active_call = ActiveCall::global(cx); - - let mut status_indicator = None; - cx.observe(&active_call, move |call, cx| { - if let Some(room) = call.read(cx).room() { - if room.read(cx).is_screen_sharing() { - if status_indicator.is_none() - && settings::get::(cx).show_call_status_icon - { - status_indicator = Some(cx.add_status_bar_item(|_| SharingStatusIndicator)); - } - } else if let Some(window) = status_indicator.take() { - window.update(cx, |cx| cx.remove_window()); - } - } else if let Some(window) = status_indicator.take() { - window.update(cx, |cx| cx.remove_window()); - } - }) - .detach(); -} - -pub struct SharingStatusIndicator; - -impl Entity for SharingStatusIndicator { - type Event = (); -} - -impl View for SharingStatusIndicator { - fn ui_name() -> &'static str { - "SharingStatusIndicator" - } - - fn render(&mut self, cx: &mut ViewContext) -> AnyElement { - let color = match cx.window_appearance() { - Appearance::Light | Appearance::VibrantLight => Color::black(), - Appearance::Dark | Appearance::VibrantDark => Color::white(), - }; - - MouseEventHandler::new::(0, cx, |_, _| { - Svg::new("icons/desktop.svg") - .with_color(color) - .constrained() - .with_width(18.) - .aligned() - }) - .on_click(MouseButton::Left, |_, _, cx| { - toggle_screen_sharing(&Default::default(), cx) - }) - .into_any() - } -} From 2de34a905d1901d503a2053dc3c33504791574cb Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Fri, 20 Oct 2023 14:45:37 -0600 Subject: [PATCH 37/41] Hide any circumstantial evidence that this didn't work perfectly first time --- crates/editor/src/movement.rs | 2 -- crates/vim/src/visual.rs | 2 -- 2 files changed, 4 deletions(-) diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 1bb4154b1f744d24cba6b7ec23d25bf3117590d6..580faf10506f38bb971bf548c3b65d911fb1fd45 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -147,9 +147,7 @@ pub fn down_by_rows( goal_x = map.x_for_point(point, text_layout_details) } - dbg!(point); let mut clipped_point = map.clip_point(point, Bias::Right); - dbg!(clipped_point); if clipped_point.row() > point.row() { clipped_point = map.clip_point(point, Bias::Left); } diff --git a/crates/vim/src/visual.rs b/crates/vim/src/visual.rs index 0e108251e2e75ac34f7543c4c028c602ab1f97a3..5d6477ff5be0134c0bbde6bbcf292ec60c6527a8 100644 --- a/crates/vim/src/visual.rs +++ b/crates/vim/src/visual.rs @@ -180,8 +180,6 @@ pub fn visual_block_motion( head_x = map.x_for_point(head, &text_layout_details); } - dbg!(head, head_x, tail, tail_x); - let positions = if is_reversed { head_x..tail_x } else { From 7748848b6e6d74537ee54c6cdc3affd04b8fe680 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sat, 21 Oct 2023 01:07:43 +0200 Subject: [PATCH 38/41] Move prettier parsers data into languages from LSP adapters --- crates/collab/src/tests/integration_tests.rs | 10 ++-- crates/editor/src/editor_tests.rs | 19 +++---- crates/language/src/language.rs | 46 ++++++----------- crates/prettier/src/prettier.rs | 50 +++++-------------- crates/project/src/project.rs | 36 ++++++------- crates/zed/src/languages/css.rs | 6 +-- crates/zed/src/languages/css/config.toml | 1 + crates/zed/src/languages/html.rs | 6 +-- crates/zed/src/languages/html/config.toml | 1 + .../zed/src/languages/javascript/config.toml | 1 + crates/zed/src/languages/json.rs | 8 +-- crates/zed/src/languages/json/config.toml | 1 + crates/zed/src/languages/svelte.rs | 9 ++-- crates/zed/src/languages/svelte/config.toml | 1 + crates/zed/src/languages/tailwind.rs | 9 ++-- crates/zed/src/languages/tsx/config.toml | 1 + crates/zed/src/languages/typescript.rs | 10 +--- .../zed/src/languages/typescript/config.toml | 1 + crates/zed/src/languages/yaml.rs | 7 +-- crates/zed/src/languages/yaml/config.toml | 1 + 20 files changed, 73 insertions(+), 151 deletions(-) diff --git a/crates/collab/src/tests/integration_tests.rs b/crates/collab/src/tests/integration_tests.rs index d6d449fd476b0d6f967641268bf1798a21ccf81d..8396e8947fde93a40728a723596f23d251b098a0 100644 --- a/crates/collab/src/tests/integration_tests.rs +++ b/crates/collab/src/tests/integration_tests.rs @@ -15,8 +15,8 @@ use gpui::{executor::Deterministic, test::EmptyView, AppContext, ModelHandle, Te use indoc::indoc; use language::{ language_settings::{AllLanguageSettings, Formatter, InlayHintSettings}, - tree_sitter_rust, Anchor, BundledFormatter, Diagnostic, DiagnosticEntry, FakeLspAdapter, - Language, LanguageConfig, LineEnding, OffsetRangeExt, Point, Rope, + tree_sitter_rust, Anchor, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language, + LanguageConfig, LineEnding, OffsetRangeExt, Point, Rope, }; use live_kit_client::MacOSDisplay; use lsp::LanguageServerId; @@ -4530,6 +4530,7 @@ async fn test_prettier_formatting_buffer( LanguageConfig { name: "Rust".into(), path_suffixes: vec!["rs".to_string()], + prettier_parser_name: Some("test_parser".to_string()), ..Default::default() }, Some(tree_sitter_rust::language()), @@ -4537,10 +4538,7 @@ async fn test_prettier_formatting_buffer( let test_plugin = "test_plugin"; let mut fake_language_servers = language .set_fake_lsp_adapter(Arc::new(FakeLspAdapter { - enabled_formatters: vec![BundledFormatter::Prettier { - parser_name: Some("test_parser"), - plugin_names: vec![test_plugin], - }], + prettier_plugins: vec![test_plugin], ..Default::default() })) .await; diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs index 435e05018cd4fa905f0dff4637fed29f847625e9..62cc53808f0341976c1f8450267db7c82fb2eb12 100644 --- a/crates/editor/src/editor_tests.rs +++ b/crates/editor/src/editor_tests.rs @@ -19,8 +19,8 @@ use gpui::{ use indoc::indoc; use language::{ language_settings::{AllLanguageSettings, AllLanguageSettingsContent, LanguageSettingsContent}, - BracketPairConfig, BundledFormatter, FakeLspAdapter, LanguageConfig, LanguageConfigOverride, - LanguageRegistry, Override, Point, + BracketPairConfig, FakeLspAdapter, LanguageConfig, LanguageConfigOverride, LanguageRegistry, + Override, Point, }; use parking_lot::Mutex; use project::project_settings::{LspSettings, ProjectSettings}; @@ -5084,6 +5084,9 @@ async fn test_document_format_manual_trigger(cx: &mut gpui::TestAppContext) { LanguageConfig { name: "Rust".into(), path_suffixes: vec!["rs".to_string()], + // Enable Prettier formatting for the same buffer, and ensure + // LSP is called instead of Prettier. + prettier_parser_name: Some("test_parser".to_string()), ..Default::default() }, Some(tree_sitter_rust::language()), @@ -5094,12 +5097,6 @@ async fn test_document_format_manual_trigger(cx: &mut gpui::TestAppContext) { document_formatting_provider: Some(lsp::OneOf::Left(true)), ..Default::default() }, - // Enable Prettier formatting for the same buffer, and ensure - // LSP is called instead of Prettier. - enabled_formatters: vec![BundledFormatter::Prettier { - parser_name: Some("test_parser"), - plugin_names: Vec::new(), - }], ..Default::default() })) .await; @@ -7838,6 +7835,7 @@ async fn test_document_format_with_prettier(cx: &mut gpui::TestAppContext) { LanguageConfig { name: "Rust".into(), path_suffixes: vec!["rs".to_string()], + prettier_parser_name: Some("test_parser".to_string()), ..Default::default() }, Some(tree_sitter_rust::language()), @@ -7846,10 +7844,7 @@ async fn test_document_format_with_prettier(cx: &mut gpui::TestAppContext) { let test_plugin = "test_plugin"; let _ = language .set_fake_lsp_adapter(Arc::new(FakeLspAdapter { - enabled_formatters: vec![BundledFormatter::Prettier { - parser_name: Some("test_parser"), - plugin_names: vec![test_plugin], - }], + prettier_plugins: vec![test_plugin], ..Default::default() })) .await; diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 0b49d92125c31a1c01f24d2dcac7f0b335213f05..59d1d12cb97862505a7a89cee714d8b7244c6783 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -226,8 +226,8 @@ impl CachedLspAdapter { self.adapter.label_for_symbol(name, kind, language).await } - pub fn enabled_formatters(&self) -> Vec { - self.adapter.enabled_formatters() + pub fn prettier_plugins(&self) -> &[&'static str] { + self.adapter.prettier_plugins() } } @@ -336,31 +336,8 @@ pub trait LspAdapter: 'static + Send + Sync { Default::default() } - fn enabled_formatters(&self) -> Vec { - Vec::new() - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum BundledFormatter { - Prettier { - // See https://prettier.io/docs/en/options.html#parser for a list of valid values. - // Usually, every language has a single parser (standard or plugin-provided), hence `Some("parser_name")` can be used. - // There can not be multiple parsers for a single language, in case of a conflict, we would attempt to select the one with most plugins. - // - // But exceptions like Tailwind CSS exist, which uses standard parsers for CSS/JS/HTML/etc. but require an extra plugin to be installed. - // For those cases, `None` will install the plugin but apply other, regular parser defined for the language, and this would not be a conflict. - parser_name: Option<&'static str>, - plugin_names: Vec<&'static str>, - }, -} - -impl BundledFormatter { - pub fn prettier(parser_name: &'static str) -> Self { - Self::Prettier { - parser_name: Some(parser_name), - plugin_names: Vec::new(), - } + fn prettier_plugins(&self) -> &[&'static str] { + &[] } } @@ -398,6 +375,8 @@ pub struct LanguageConfig { pub overrides: HashMap, #[serde(default)] pub word_characters: HashSet, + #[serde(default)] + pub prettier_parser_name: Option, } #[derive(Debug, Default)] @@ -471,6 +450,7 @@ impl Default for LanguageConfig { overrides: Default::default(), collapsed_placeholder: Default::default(), word_characters: Default::default(), + prettier_parser_name: None, } } } @@ -496,7 +476,7 @@ pub struct FakeLspAdapter { pub initializer: Option>, pub disk_based_diagnostics_progress_token: Option, pub disk_based_diagnostics_sources: Vec, - pub enabled_formatters: Vec, + pub prettier_plugins: Vec<&'static str>, } #[derive(Clone, Debug, Default)] @@ -1597,6 +1577,10 @@ impl Language { override_id: None, } } + + pub fn prettier_parser_name(&self) -> Option<&str> { + self.config.prettier_parser_name.as_deref() + } } impl LanguageScope { @@ -1759,7 +1743,7 @@ impl Default for FakeLspAdapter { disk_based_diagnostics_progress_token: None, initialization_options: None, disk_based_diagnostics_sources: Vec::new(), - enabled_formatters: Vec::new(), + prettier_plugins: Vec::new(), } } } @@ -1817,8 +1801,8 @@ impl LspAdapter for Arc { self.initialization_options.clone() } - fn enabled_formatters(&self) -> Vec { - self.enabled_formatters.clone() + fn prettier_plugins(&self) -> &[&'static str] { + &self.prettier_plugins } } diff --git a/crates/prettier/src/prettier.rs b/crates/prettier/src/prettier.rs index c3811b567b590c632fd3f33ccf0df3c7b9d1b0f3..09b793e5a23e0e6954998e995189ecd90a06587b 100644 --- a/crates/prettier/src/prettier.rs +++ b/crates/prettier/src/prettier.rs @@ -3,11 +3,11 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::Context; -use collections::{HashMap, HashSet}; +use collections::HashMap; use fs::Fs; use gpui::{AsyncAppContext, ModelHandle}; use language::language_settings::language_settings; -use language::{Buffer, BundledFormatter, Diff}; +use language::{Buffer, Diff}; use lsp::{LanguageServer, LanguageServerId}; use node_runtime::NodeRuntime; use serde::{Deserialize, Serialize}; @@ -242,40 +242,16 @@ impl Prettier { Self::Real(local) => { let params = buffer.read_with(cx, |buffer, cx| { let buffer_language = buffer.language(); - let parsers_with_plugins = buffer_language - .into_iter() - .flat_map(|language| { - language - .lsp_adapters() - .iter() - .flat_map(|adapter| adapter.enabled_formatters()) - .filter_map(|formatter| match formatter { - BundledFormatter::Prettier { - parser_name, - plugin_names, - } => Some((parser_name, plugin_names)), - }) - }) - .fold( - HashMap::default(), - |mut parsers_with_plugins, (parser_name, plugins)| { - match parser_name { - Some(parser_name) => parsers_with_plugins - .entry(parser_name) - .or_insert_with(HashSet::default) - .extend(plugins), - None => parsers_with_plugins.values_mut().for_each(|existing_plugins| { - existing_plugins.extend(plugins.iter()); - }), - } - parsers_with_plugins - }, - ); - - let selected_parser_with_plugins = parsers_with_plugins.iter().max_by_key(|(_, plugins)| plugins.len()); - if parsers_with_plugins.len() > 1 { - log::warn!("Found multiple parsers with plugins {parsers_with_plugins:?}, will select only one: {selected_parser_with_plugins:?}"); - } + let parser_with_plugins = buffer_language.and_then(|l| { + let prettier_parser = l.prettier_parser_name()?; + let mut prettier_plugins = l + .lsp_adapters() + .iter() + .flat_map(|adapter| adapter.prettier_plugins()) + .collect::>(); + prettier_plugins.dedup(); + Some((prettier_parser, prettier_plugins)) + }); let prettier_node_modules = self.prettier_dir().join("node_modules"); anyhow::ensure!(prettier_node_modules.is_dir(), "Prettier node_modules dir does not exist: {prettier_node_modules:?}"); @@ -296,7 +272,7 @@ impl Prettier { } None }; - let (parser, located_plugins) = match selected_parser_with_plugins { + let (parser, located_plugins) = match parser_with_plugins { Some((parser, plugins)) => { // Tailwind plugin requires being added last // https://github.com/tailwindlabs/prettier-plugin-tailwindcss#compatibility-with-other-prettier-plugins diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index e91c91f7c3403f01716c4ea2e855460a9e9bc470..2eb1fd421c82e419f2aa0445d3d9774927968ea7 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -39,11 +39,11 @@ use language::{ deserialize_anchor, deserialize_fingerprint, deserialize_line_ending, deserialize_version, serialize_anchor, serialize_version, split_operations, }, - range_from_lsp, range_to_lsp, Bias, Buffer, BufferSnapshot, BundledFormatter, CachedLspAdapter, - CodeAction, CodeLabel, Completion, Diagnostic, DiagnosticEntry, DiagnosticSet, Diff, - Event as BufferEvent, File as _, Language, LanguageRegistry, LanguageServerName, LocalFile, - LspAdapterDelegate, OffsetRangeExt, Operation, Patch, PendingLanguageServer, PointUtf16, - TextBufferSnapshot, ToOffset, ToPointUtf16, Transaction, Unclipped, + range_from_lsp, range_to_lsp, Bias, Buffer, BufferSnapshot, CachedLspAdapter, CodeAction, + CodeLabel, Completion, Diagnostic, DiagnosticEntry, DiagnosticSet, Diff, Event as BufferEvent, + File as _, Language, LanguageRegistry, LanguageServerName, LocalFile, LspAdapterDelegate, + OffsetRangeExt, Operation, Patch, PendingLanguageServer, PointUtf16, TextBufferSnapshot, + ToOffset, ToPointUtf16, Transaction, Unclipped, }; use log::error; use lsp::{ @@ -8352,12 +8352,7 @@ impl Project { let Some(buffer_language) = buffer.language() else { return Task::ready(None); }; - if !buffer_language - .lsp_adapters() - .iter() - .flat_map(|adapter| adapter.enabled_formatters()) - .any(|formatter| matches!(formatter, BundledFormatter::Prettier { .. })) - { + if buffer_language.prettier_parser_name().is_none() { return Task::ready(None); } @@ -8510,16 +8505,15 @@ impl Project { }; let mut prettier_plugins = None; - for formatter in new_language - .lsp_adapters() - .into_iter() - .flat_map(|adapter| adapter.enabled_formatters()) - { - match formatter { - BundledFormatter::Prettier { plugin_names, .. } => prettier_plugins - .get_or_insert_with(|| HashSet::default()) - .extend(plugin_names), - } + if new_language.prettier_parser_name().is_some() { + prettier_plugins + .get_or_insert_with(|| HashSet::default()) + .extend( + new_language + .lsp_adapters() + .iter() + .flat_map(|adapter| adapter.prettier_plugins()), + ) } let Some(prettier_plugins) = prettier_plugins else { return Task::ready(Ok(())); diff --git a/crates/zed/src/languages/css.rs b/crates/zed/src/languages/css.rs index f046437d75ec2ac824c81b33498522581fb89a9a..fdbc179209603ea41c16e3a5aa6aac0d6a7a7f8e 100644 --- a/crates/zed/src/languages/css.rs +++ b/crates/zed/src/languages/css.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use futures::StreamExt; -use language::{BundledFormatter, LanguageServerName, LspAdapter, LspAdapterDelegate}; +use language::{LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; use serde_json::json; @@ -96,10 +96,6 @@ impl LspAdapter for CssLspAdapter { "provideFormatter": true })) } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("css")] - } } async fn get_cached_server_binary( diff --git a/crates/zed/src/languages/css/config.toml b/crates/zed/src/languages/css/config.toml index da63d6df2d8dc9af3b4a329b1df90a3ae13cbe69..24a844c239da56b841036fefca82712a566db1fb 100644 --- a/crates/zed/src/languages/css/config.toml +++ b/crates/zed/src/languages/css/config.toml @@ -10,3 +10,4 @@ brackets = [ ] word_characters = ["-"] block_comment = ["/* ", " */"] +prettier_parser_name = "css" diff --git a/crates/zed/src/languages/html.rs b/crates/zed/src/languages/html.rs index 6f27b7ca8faa8ca5474a9dbcf7b6af219610ab2b..b8f1c70cce2ae00ca2a1647840e483844ea2a2e9 100644 --- a/crates/zed/src/languages/html.rs +++ b/crates/zed/src/languages/html.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use futures::StreamExt; -use language::{BundledFormatter, LanguageServerName, LspAdapter, LspAdapterDelegate}; +use language::{LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; use serde_json::json; @@ -96,10 +96,6 @@ impl LspAdapter for HtmlLspAdapter { "provideFormatter": true })) } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("html")] - } } async fn get_cached_server_binary( diff --git a/crates/zed/src/languages/html/config.toml b/crates/zed/src/languages/html/config.toml index 164e095cee9a71f62531fc576c44f40a1c4e9260..0105f0d60de3daeb169267b7f09e94fe853f76f2 100644 --- a/crates/zed/src/languages/html/config.toml +++ b/crates/zed/src/languages/html/config.toml @@ -11,3 +11,4 @@ brackets = [ { start = "!--", end = " --", close = true, newline = false, not_in = ["comment", "string"] }, ] word_characters = ["-"] +prettier_parser_name = "html" diff --git a/crates/zed/src/languages/javascript/config.toml b/crates/zed/src/languages/javascript/config.toml index 2394c575392e2d441732cd0cb6717fd315ccaac7..3b8862e3588caeb6587f37ee006fc0b1589675fe 100644 --- a/crates/zed/src/languages/javascript/config.toml +++ b/crates/zed/src/languages/javascript/config.toml @@ -15,6 +15,7 @@ brackets = [ ] word_characters = ["$", "#"] scope_opt_in_language_servers = ["tailwindcss-language-server"] +prettier_parser_name = "babel" [overrides.element] line_comment = { remove = true } diff --git a/crates/zed/src/languages/json.rs b/crates/zed/src/languages/json.rs index f017af0a22a278dc10d8a36c6941e0fc3adaa1a1..63f909ae2a2e264ea672dee48e305ba1be82e066 100644 --- a/crates/zed/src/languages/json.rs +++ b/crates/zed/src/languages/json.rs @@ -4,9 +4,7 @@ use collections::HashMap; use feature_flags::FeatureFlagAppExt; use futures::{future::BoxFuture, FutureExt, StreamExt}; use gpui::AppContext; -use language::{ - BundledFormatter, LanguageRegistry, LanguageServerName, LspAdapter, LspAdapterDelegate, -}; +use language::{LanguageRegistry, LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; use serde_json::json; @@ -146,10 +144,6 @@ impl LspAdapter for JsonLspAdapter { async fn language_ids(&self) -> HashMap { [("JSON".into(), "jsonc".into())].into_iter().collect() } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("json")] - } } async fn get_cached_server_binary( diff --git a/crates/zed/src/languages/json/config.toml b/crates/zed/src/languages/json/config.toml index 87f41882a5fd92ee4ce66f8fb32031fc85d68bbf..37a6d3a54cc744de519c29148e9e961c8e8c208a 100644 --- a/crates/zed/src/languages/json/config.toml +++ b/crates/zed/src/languages/json/config.toml @@ -7,3 +7,4 @@ brackets = [ { start = "[", end = "]", close = true, newline = true }, { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] }, ] +prettier_parser_name = "json" diff --git a/crates/zed/src/languages/svelte.rs b/crates/zed/src/languages/svelte.rs index 2089fe88b1ce03702a62e61033ec954fa9bdb789..34dab81772c0b418d3c4be796078d28a49f8a147 100644 --- a/crates/zed/src/languages/svelte.rs +++ b/crates/zed/src/languages/svelte.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use futures::StreamExt; -use language::{BundledFormatter, LanguageServerName, LspAdapter, LspAdapterDelegate}; +use language::{LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; use serde_json::json; @@ -96,11 +96,8 @@ impl LspAdapter for SvelteLspAdapter { })) } - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::Prettier { - parser_name: Some("svelte"), - plugin_names: vec!["prettier-plugin-svelte"], - }] + fn prettier_plugins(&self) -> &[&'static str] { + &["prettier-plugin-svelte"] } } diff --git a/crates/zed/src/languages/svelte/config.toml b/crates/zed/src/languages/svelte/config.toml index 8a07b012c7c0e25c3f11e54b2d905d27179cd233..76f03493b559232df1cd09d36fff6ebb5391f8d7 100644 --- a/crates/zed/src/languages/svelte/config.toml +++ b/crates/zed/src/languages/svelte/config.toml @@ -13,6 +13,7 @@ brackets = [ { start = "/*", end = " */", close = true, newline = false, not_in = ["string", "comment"] }, ] scope_opt_in_language_servers = ["tailwindcss-language-server"] +prettier_parser_name = "svelte" [overrides.string] word_characters = ["-"] diff --git a/crates/zed/src/languages/tailwind.rs b/crates/zed/src/languages/tailwind.rs index 296d9a490b1b115f58b518ba26be639be5af4d37..6d6006dbd48c3d4ea065e12e909fdbd3cf775e7f 100644 --- a/crates/zed/src/languages/tailwind.rs +++ b/crates/zed/src/languages/tailwind.rs @@ -6,7 +6,7 @@ use futures::{ FutureExt, StreamExt, }; use gpui::AppContext; -use language::{BundledFormatter, LanguageServerName, LspAdapter, LspAdapterDelegate}; +use language::{LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; use serde_json::{json, Value}; @@ -130,11 +130,8 @@ impl LspAdapter for TailwindLspAdapter { ]) } - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::Prettier { - parser_name: None, - plugin_names: vec!["prettier-plugin-tailwindcss"], - }] + fn prettier_plugins(&self) -> &[&'static str] { + &["prettier-plugin-tailwindcss"] } } diff --git a/crates/zed/src/languages/tsx/config.toml b/crates/zed/src/languages/tsx/config.toml index a7f99bef5ebfdf29428f85430b78acc1352ba42f..0dae25d7795fb484e5e02a436b3ac841756e3225 100644 --- a/crates/zed/src/languages/tsx/config.toml +++ b/crates/zed/src/languages/tsx/config.toml @@ -14,6 +14,7 @@ brackets = [ ] word_characters = ["#", "$"] scope_opt_in_language_servers = ["tailwindcss-language-server"] +prettier_parser_name = "typescript" [overrides.element] line_comment = { remove = true } diff --git a/crates/zed/src/languages/typescript.rs b/crates/zed/src/languages/typescript.rs index f09c9645881401aaa43ba00c9f13757ca0b53224..676d0fd4c0d7afeaf1d3d39bd0c91f17c1a862cf 100644 --- a/crates/zed/src/languages/typescript.rs +++ b/crates/zed/src/languages/typescript.rs @@ -4,7 +4,7 @@ use async_tar::Archive; use async_trait::async_trait; use futures::{future::BoxFuture, FutureExt}; use gpui::AppContext; -use language::{BundledFormatter, LanguageServerName, LspAdapter, LspAdapterDelegate}; +use language::{LanguageServerName, LspAdapter, LspAdapterDelegate}; use lsp::{CodeActionKind, LanguageServerBinary}; use node_runtime::NodeRuntime; use serde_json::{json, Value}; @@ -161,10 +161,6 @@ impl LspAdapter for TypeScriptLspAdapter { "provideFormatter": true })) } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("typescript")] - } } async fn get_cached_ts_server_binary( @@ -313,10 +309,6 @@ impl LspAdapter for EsLintLspAdapter { async fn initialization_options(&self) -> Option { None } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("babel")] - } } async fn get_cached_eslint_server_binary( diff --git a/crates/zed/src/languages/typescript/config.toml b/crates/zed/src/languages/typescript/config.toml index 2fad1f13e182c52e2886e459fd0efa6fb673a7d6..d1ebffc559a96c55035f9f71c82141c16417f9ff 100644 --- a/crates/zed/src/languages/typescript/config.toml +++ b/crates/zed/src/languages/typescript/config.toml @@ -13,3 +13,4 @@ brackets = [ { start = "/*", end = " */", close = true, newline = false, not_in = ["string", "comment"] }, ] word_characters = ["#", "$"] +prettier_parser_name = "typescript" diff --git a/crates/zed/src/languages/yaml.rs b/crates/zed/src/languages/yaml.rs index 1c1ce1866866260b909262ae66e0771713b0df0d..8b438d0949dc0ef1f514f3c315c3eab98174d506 100644 --- a/crates/zed/src/languages/yaml.rs +++ b/crates/zed/src/languages/yaml.rs @@ -3,8 +3,7 @@ use async_trait::async_trait; use futures::{future::BoxFuture, FutureExt, StreamExt}; use gpui::AppContext; use language::{ - language_settings::all_language_settings, BundledFormatter, LanguageServerName, LspAdapter, - LspAdapterDelegate, + language_settings::all_language_settings, LanguageServerName, LspAdapter, LspAdapterDelegate, }; use lsp::LanguageServerBinary; use node_runtime::NodeRuntime; @@ -109,10 +108,6 @@ impl LspAdapter for YamlLspAdapter { })) .boxed() } - - fn enabled_formatters(&self) -> Vec { - vec![BundledFormatter::prettier("yaml")] - } } async fn get_cached_server_binary( diff --git a/crates/zed/src/languages/yaml/config.toml b/crates/zed/src/languages/yaml/config.toml index 6912d9245701dacd41c0c33fa49065d361dcbd22..4e91dd348bda85648a05ed12e2075b7ed876d505 100644 --- a/crates/zed/src/languages/yaml/config.toml +++ b/crates/zed/src/languages/yaml/config.toml @@ -9,3 +9,4 @@ brackets = [ ] increase_indent_pattern = ":\\s*[|>]?\\s*$" +prettier_parser_name = "yaml" From fc37abc356e7243c56afcebe094256d7bb7296b3 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:01:01 +0200 Subject: [PATCH 39/41] vcs_menu: Query branches on open instead of per keystroke (#3144) Release Notes: - Improved performance of branch picker by querying branches on menu open instead of querying once per each keystroke. (fixes zed-industries/community#2161) --- Cargo.lock | 1 + crates/collab_ui/src/collab_titlebar_item.rs | 6 +- crates/vcs_menu/Cargo.toml | 1 + crates/vcs_menu/src/lib.rs | 126 +++++++++---------- 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56c9ceda04f62262ea199d2a7424fee053d68284..5fe28590a14fe4e8874022972a3baf3e7d6b7c4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9098,6 +9098,7 @@ name = "vcs_menu" version = "0.1.0" dependencies = [ "anyhow", + "fs", "fuzzy", "gpui", "picker", diff --git a/crates/collab_ui/src/collab_titlebar_item.rs b/crates/collab_ui/src/collab_titlebar_item.rs index 211ee863e89f6ce7bfe3aa44826c0a8f827a6f85..c9bfdd26907ec2c5f8b0052564a06c52c1feacfb 100644 --- a/crates/collab_ui/src/collab_titlebar_item.rs +++ b/crates/collab_ui/src/collab_titlebar_item.rs @@ -488,7 +488,11 @@ impl CollabTitlebarItem { pub fn toggle_vcs_menu(&mut self, _: &ToggleVcsMenu, cx: &mut ViewContext) { if self.branch_popover.take().is_none() { if let Some(workspace) = self.workspace.upgrade(cx) { - let view = cx.add_view(|cx| build_branch_list(workspace, cx)); + let Some(view) = + cx.add_option_view(|cx| build_branch_list(workspace, cx).log_err()) + else { + return; + }; cx.subscribe(&view, |this, _, event, cx| { match event { PickerEvent::Dismiss => { diff --git a/crates/vcs_menu/Cargo.toml b/crates/vcs_menu/Cargo.toml index 4ddf1214d0db179f8033547d31dfa2960b254ef5..c712cfd6f7938da110896223c316aa6ade35d5b4 100644 --- a/crates/vcs_menu/Cargo.toml +++ b/crates/vcs_menu/Cargo.toml @@ -7,6 +7,7 @@ publish = false [dependencies] fuzzy = {path = "../fuzzy"} +fs = {path = "../fs"} gpui = {path = "../gpui"} picker = {path = "../picker"} util = {path = "../util"} diff --git a/crates/vcs_menu/src/lib.rs b/crates/vcs_menu/src/lib.rs index 73ed4b059ea37ba6770280a2e84318d1c4517aec..dce3724ccd3c2c0f4a795a9dfe61960d491e9d11 100644 --- a/crates/vcs_menu/src/lib.rs +++ b/crates/vcs_menu/src/lib.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, bail, Result}; +use fs::repository::Branch; use fuzzy::{StringMatch, StringMatchCandidate}; use gpui::{ actions, @@ -22,18 +23,9 @@ pub type BranchList = Picker; pub fn build_branch_list( workspace: ViewHandle, cx: &mut ViewContext, -) -> BranchList { - Picker::new( - BranchListDelegate { - matches: vec![], - workspace, - selected_index: 0, - last_query: String::default(), - branch_name_trailoff_after: 29, - }, - cx, - ) - .with_theme(|theme| theme.picker.clone()) +) -> Result { + Ok(Picker::new(BranchListDelegate::new(workspace, 29, cx)?, cx) + .with_theme(|theme| theme.picker.clone())) } fn toggle( @@ -43,31 +35,24 @@ fn toggle( ) -> Option>> { Some(cx.spawn(|workspace, mut cx| async move { workspace.update(&mut cx, |workspace, cx| { + // Modal branch picker has a longer trailoff than a popover one. + let delegate = BranchListDelegate::new(cx.handle(), 70, cx)?; workspace.toggle_modal(cx, |_, cx| { - let workspace = cx.handle(); cx.add_view(|cx| { - Picker::new( - BranchListDelegate { - matches: vec![], - workspace, - selected_index: 0, - last_query: String::default(), - /// Modal branch picker has a longer trailoff than a popover one. - branch_name_trailoff_after: 70, - }, - cx, - ) - .with_theme(|theme| theme.picker.clone()) - .with_max_size(800., 1200.) + Picker::new(delegate, cx) + .with_theme(|theme| theme.picker.clone()) + .with_max_size(800., 1200.) }) }); - })?; + Ok::<_, anyhow::Error>(()) + })??; Ok(()) })) } pub struct BranchListDelegate { matches: Vec, + all_branches: Vec, workspace: ViewHandle, selected_index: usize, last_query: String, @@ -76,6 +61,31 @@ pub struct BranchListDelegate { } impl BranchListDelegate { + fn new( + workspace: ViewHandle, + branch_name_trailoff_after: usize, + cx: &AppContext, + ) -> Result { + let project = workspace.read(cx).project().read(&cx); + + let Some(worktree) = project.visible_worktrees(cx).next() else { + bail!("Cannot update branch list as there are no visible worktrees") + }; + let mut cwd = worktree.read(cx).abs_path().to_path_buf(); + cwd.push(".git"); + let Some(repo) = project.fs().open_repo(&cwd) else { + bail!("Project does not have associated git repository.") + }; + let all_branches = repo.lock().branches()?; + Ok(Self { + matches: vec![], + workspace, + all_branches, + selected_index: 0, + last_query: Default::default(), + branch_name_trailoff_after, + }) + } fn display_error_toast(&self, message: String, cx: &mut ViewContext) { const GIT_CHECKOUT_FAILURE_ID: usize = 2048; self.workspace.update(cx, |model, ctx| { @@ -83,6 +93,7 @@ impl BranchListDelegate { }); } } + impl PickerDelegate for BranchListDelegate { fn placeholder_text(&self) -> Arc { "Select branch...".into() @@ -102,45 +113,28 @@ impl PickerDelegate for BranchListDelegate { fn update_matches(&mut self, query: String, cx: &mut ViewContext>) -> Task<()> { cx.spawn(move |picker, mut cx| async move { - let Some(candidates) = picker - .read_with(&mut cx, |view, cx| { - let delegate = view.delegate(); - let project = delegate.workspace.read(cx).project().read(&cx); - - let Some(worktree) = project.visible_worktrees(cx).next() else { - bail!("Cannot update branch list as there are no visible worktrees") - }; - let mut cwd = worktree.read(cx).abs_path().to_path_buf(); - cwd.push(".git"); - let Some(repo) = project.fs().open_repo(&cwd) else { - bail!("Project does not have associated git repository.") - }; - let mut branches = repo.lock().branches()?; - const RECENT_BRANCHES_COUNT: usize = 10; - if query.is_empty() && branches.len() > RECENT_BRANCHES_COUNT { - // Truncate list of recent branches - // Do a partial sort to show recent-ish branches first. - branches.select_nth_unstable_by(RECENT_BRANCHES_COUNT - 1, |lhs, rhs| { - rhs.unix_timestamp.cmp(&lhs.unix_timestamp) - }); - branches.truncate(RECENT_BRANCHES_COUNT); - branches.sort_unstable_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); - } - Ok(branches - .iter() - .cloned() - .enumerate() - .map(|(ix, command)| StringMatchCandidate { - id: ix, - char_bag: command.name.chars().collect(), - string: command.name.into(), - }) - .collect::>()) - }) - .log_err() - else { - return; - }; + let candidates = picker.read_with(&mut cx, |view, _| { + const RECENT_BRANCHES_COUNT: usize = 10; + let mut branches = view.delegate().all_branches.clone(); + if query.is_empty() && branches.len() > RECENT_BRANCHES_COUNT { + // Truncate list of recent branches + // Do a partial sort to show recent-ish branches first. + branches.select_nth_unstable_by(RECENT_BRANCHES_COUNT - 1, |lhs, rhs| { + rhs.unix_timestamp.cmp(&lhs.unix_timestamp) + }); + branches.truncate(RECENT_BRANCHES_COUNT); + branches.sort_unstable_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); + } + branches + .into_iter() + .enumerate() + .map(|(ix, command)| StringMatchCandidate { + id: ix, + char_bag: command.name.chars().collect(), + string: command.name.into(), + }) + .collect::>() + }); let Some(candidates) = candidates.log_err() else { return; }; From 106115676d31ede3171f7500f23f900f68d60dbc Mon Sep 17 00:00:00 2001 From: KCaverly Date: Sat, 21 Oct 2023 10:19:50 -0400 Subject: [PATCH 40/41] update semantic search to use keychain as fallback --- crates/ai/src/embedding.rs | 45 +++++++++++++++++---- crates/search/src/project_search.rs | 2 +- crates/semantic_index/src/semantic_index.rs | 20 ++++++++- crates/zed/examples/semantic_index_eval.rs | 18 ++++++++- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/crates/ai/src/embedding.rs b/crates/ai/src/embedding.rs index 4587ece0a23d116c55f07405e009497486d583d7..4d5e40fad984229ff5c7bf6b562ee86793cdb283 100644 --- a/crates/ai/src/embedding.rs +++ b/crates/ai/src/embedding.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use futures::AsyncReadExt; use gpui::executor::Background; -use gpui::serde_json; +use gpui::{serde_json, ViewContext}; use isahc::http::StatusCode; use isahc::prelude::Configurable; use isahc::{AsyncBody, Response}; @@ -20,9 +20,11 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tiktoken_rs::{cl100k_base, CoreBPE}; use util::http::{HttpClient, Request}; +use util::ResultExt; + +use crate::completion::OPENAI_API_URL; lazy_static! { - static ref OPENAI_API_KEY: Option = env::var("OPENAI_API_KEY").ok(); static ref OPENAI_BPE_TOKENIZER: CoreBPE = cl100k_base().unwrap(); } @@ -87,6 +89,7 @@ impl Embedding { #[derive(Clone)] pub struct OpenAIEmbeddings { + pub api_key: Option, pub client: Arc, pub executor: Arc, rate_limit_count_rx: watch::Receiver>, @@ -166,11 +169,36 @@ impl EmbeddingProvider for DummyEmbeddings { const OPENAI_INPUT_LIMIT: usize = 8190; impl OpenAIEmbeddings { - pub fn new(client: Arc, executor: Arc) -> Self { + pub fn authenticate(&mut self, cx: &mut ViewContext) { + if self.api_key.is_none() { + let api_key = if let Ok(api_key) = env::var("OPENAI_API_KEY") { + Some(api_key) + } else if let Some((_, api_key)) = cx + .platform() + .read_credentials(OPENAI_API_URL) + .log_err() + .flatten() + { + String::from_utf8(api_key).log_err() + } else { + None + }; + + if let Some(api_key) = api_key { + self.api_key = Some(api_key); + } + } + } + pub fn new( + api_key: Option, + client: Arc, + executor: Arc, + ) -> Self { let (rate_limit_count_tx, rate_limit_count_rx) = watch::channel_with(None); let rate_limit_count_tx = Arc::new(Mutex::new(rate_limit_count_tx)); OpenAIEmbeddings { + api_key, client, executor, rate_limit_count_rx, @@ -237,8 +265,9 @@ impl OpenAIEmbeddings { #[async_trait] impl EmbeddingProvider for OpenAIEmbeddings { fn is_authenticated(&self) -> bool { - OPENAI_API_KEY.as_ref().is_some() + self.api_key.is_some() } + fn max_tokens_per_batch(&self) -> usize { 50000 } @@ -265,9 +294,9 @@ impl EmbeddingProvider for OpenAIEmbeddings { const BACKOFF_SECONDS: [usize; 4] = [3, 5, 15, 45]; const MAX_RETRIES: usize = 4; - let api_key = OPENAI_API_KEY - .as_ref() - .ok_or_else(|| anyhow!("no api key"))?; + let Some(api_key) = self.api_key.clone() else { + return Err(anyhow!("no open ai key provided")); + }; let mut request_number = 0; let mut rate_limiting = false; @@ -276,7 +305,7 @@ impl EmbeddingProvider for OpenAIEmbeddings { while request_number < MAX_RETRIES { response = self .send_request( - api_key, + &api_key, spans.iter().map(|x| &**x).collect(), request_timeout, ) diff --git a/crates/search/src/project_search.rs b/crates/search/src/project_search.rs index c03e5dc80e6fe09bd25bebacbb1349d078b1402d..5f8b92b7323b48fc69e474bd0478102c11a63e36 100644 --- a/crates/search/src/project_search.rs +++ b/crates/search/src/project_search.rs @@ -352,7 +352,7 @@ impl View for ProjectSearchView { major_text = Cow::Borrowed("Not Authenticated"); show_minor_text = false; Some( - "API Key Missing: Please set 'OPENAI_API_KEY' in Environment Variables" + "API Key Missing: Please set 'OPENAI_API_KEY' in Environment Variables.\nIf this variable was set using the Assistant Panel, please restart Zed to Authenticate." .to_string(), ) } diff --git a/crates/semantic_index/src/semantic_index.rs b/crates/semantic_index/src/semantic_index.rs index ecdba4364315eb8f0a4ed2cf579fcc3149e56e67..aae289e41789a0092dfdeeda792b779b51843f7b 100644 --- a/crates/semantic_index/src/semantic_index.rs +++ b/crates/semantic_index/src/semantic_index.rs @@ -7,7 +7,10 @@ pub mod semantic_index_settings; mod semantic_index_tests; use crate::semantic_index_settings::SemanticIndexSettings; -use ai::embedding::{Embedding, EmbeddingProvider, OpenAIEmbeddings}; +use ai::{ + completion::OPENAI_API_URL, + embedding::{Embedding, EmbeddingProvider, OpenAIEmbeddings}, +}; use anyhow::{anyhow, Result}; use collections::{BTreeMap, HashMap, HashSet}; use db::VectorDatabase; @@ -55,6 +58,19 @@ pub fn init( .join(Path::new(RELEASE_CHANNEL_NAME.as_str())) .join("embeddings_db"); + let api_key = if let Ok(api_key) = env::var("OPENAI_API_KEY") { + Some(api_key) + } else if let Some((_, api_key)) = cx + .platform() + .read_credentials(OPENAI_API_URL) + .log_err() + .flatten() + { + String::from_utf8(api_key).log_err() + } else { + None + }; + cx.subscribe_global::({ move |event, cx| { let Some(semantic_index) = SemanticIndex::global(cx) else { @@ -88,7 +104,7 @@ pub fn init( let semantic_index = SemanticIndex::new( fs, db_file_path, - Arc::new(OpenAIEmbeddings::new(http_client, cx.background())), + Arc::new(OpenAIEmbeddings::new(api_key, http_client, cx.background())), language_registry, cx.clone(), ) diff --git a/crates/zed/examples/semantic_index_eval.rs b/crates/zed/examples/semantic_index_eval.rs index 33d6b3689c1f617a96d5fab00d41e51fa28d63f4..73b3b9987b029589b1a63f71625edee72dbe13e8 100644 --- a/crates/zed/examples/semantic_index_eval.rs +++ b/crates/zed/examples/semantic_index_eval.rs @@ -1,3 +1,4 @@ +use ai::completion::OPENAI_API_URL; use ai::embedding::OpenAIEmbeddings; use anyhow::{anyhow, Result}; use client::{self, UserStore}; @@ -17,6 +18,7 @@ use std::{cmp, env, fs}; use util::channel::{RELEASE_CHANNEL, RELEASE_CHANNEL_NAME}; use util::http::{self}; use util::paths::EMBEDDINGS_DIR; +use util::ResultExt; use zed::languages; #[derive(Deserialize, Clone, Serialize)] @@ -469,12 +471,26 @@ fn main() { .join("embeddings_db"); let languages = languages.clone(); + + let api_key = if let Ok(api_key) = env::var("OPENAI_API_KEY") { + Some(api_key) + } else if let Some((_, api_key)) = cx + .platform() + .read_credentials(OPENAI_API_URL) + .log_err() + .flatten() + { + String::from_utf8(api_key).log_err() + } else { + None + }; + let fs = fs.clone(); cx.spawn(|mut cx| async move { let semantic_index = SemanticIndex::new( fs.clone(), db_file_path, - Arc::new(OpenAIEmbeddings::new(http_client, cx.background())), + Arc::new(OpenAIEmbeddings::new(api_key, http_client, cx.background())), languages.clone(), cx.clone(), ) From 4835c77840c50712ffa23b8d2e3527aba7a2671d Mon Sep 17 00:00:00 2001 From: KCaverly Date: Sat, 21 Oct 2023 10:28:54 -0400 Subject: [PATCH 41/41] update text for project search if not authenticated --- crates/search/src/project_search.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/search/src/project_search.rs b/crates/search/src/project_search.rs index 5f8b92b7323b48fc69e474bd0478102c11a63e36..55e3f6babddaf39a48ce7b56efe52333b7c0073d 100644 --- a/crates/search/src/project_search.rs +++ b/crates/search/src/project_search.rs @@ -351,33 +351,32 @@ impl View for ProjectSearchView { SemanticIndexStatus::NotAuthenticated => { major_text = Cow::Borrowed("Not Authenticated"); show_minor_text = false; - Some( - "API Key Missing: Please set 'OPENAI_API_KEY' in Environment Variables.\nIf this variable was set using the Assistant Panel, please restart Zed to Authenticate." - .to_string(), - ) + Some(vec![ + "API Key Missing: Please set 'OPENAI_API_KEY' in Environment Variables." + .to_string(), "If you authenticated using the Assistant Panel, please restart Zed to Authenticate.".to_string()]) } - SemanticIndexStatus::Indexed => Some("Indexing complete".to_string()), + SemanticIndexStatus::Indexed => Some(vec!["Indexing complete".to_string()]), SemanticIndexStatus::Indexing { remaining_files, rate_limit_expiry, } => { if remaining_files == 0 { - Some(format!("Indexing...")) + Some(vec![format!("Indexing...")]) } else { if let Some(rate_limit_expiry) = rate_limit_expiry { let remaining_seconds = rate_limit_expiry.duration_since(Instant::now()); if remaining_seconds > Duration::from_secs(0) { - Some(format!( + Some(vec![format!( "Remaining files to index (rate limit resets in {}s): {}", remaining_seconds.as_secs(), remaining_files - )) + )]) } else { - Some(format!("Remaining files to index: {}", remaining_files)) + Some(vec![format!("Remaining files to index: {}", remaining_files)]) } } else { - Some(format!("Remaining files to index: {}", remaining_files)) + Some(vec![format!("Remaining files to index: {}", remaining_files)]) } } } @@ -394,9 +393,11 @@ impl View for ProjectSearchView { } else { match current_mode { SearchMode::Semantic => { - let mut minor_text = Vec::new(); + let mut minor_text: Vec = Vec::new(); minor_text.push("".into()); - minor_text.extend(semantic_status); + if let Some(semantic_status) = semantic_status { + minor_text.extend(semantic_status); + } if show_minor_text { minor_text .push("Simply explain the code you are looking to find.".into());