From 273a8b43689651d123415673daa3584bc79a6e19 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 18 Aug 2023 17:25:20 +0200 Subject: [PATCH 01/12] editor: Recognize '$' as a Word character. This fixes PHP variable completion. When we were querying for completions, PHP LS returned proper matches for variables which we filtered out as our query did not include a `$` character. Z-2819 --- crates/language/src/buffer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 0b10432a9f4747d93ff974ac72ddbbb6783fe676..48dc936a949c5cc47e12199d4d895b9f8a09ddf6 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -2987,7 +2987,7 @@ pub fn contiguous_ranges( pub fn char_kind(c: char) -> CharKind { if c.is_whitespace() { CharKind::Whitespace - } else if c.is_alphanumeric() || c == '_' { + } else if c.is_alphanumeric() || c == '_' || c == '$' { CharKind::Word } else { CharKind::Punctuation From 3e8522b5f24d0a90f44ceeb9ff4db77821418a4f Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:09:03 +0200 Subject: [PATCH 02/12] WIP: Saved state from Friday. Co-authored-by: Julia Risley --- crates/editor/src/editor.rs | 3 ++- crates/editor/src/movement.rs | 8 ++++++-- crates/editor/src/multi_buffer.rs | 11 +++++++---- crates/language/src/buffer.rs | 13 ++++++++----- crates/project/src/search.rs | 13 +++++++++---- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 904e77c9f0fdd3dccb84758b97e0005573dcaee2..0cbaf61793338772f1252af90e8118359fbd679b 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -2657,7 +2657,8 @@ impl Editor { false }); } - + // $language = true; + // language fn completion_query(buffer: &MultiBufferSnapshot, position: impl ToOffset) -> Option { let offset = position.to_offset(buffer); let (word_range, kind) = buffer.surrounding_word(offset); diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index f70436abeb45f248ee8c306613a220655313797f..345b7404d9d9707be9dc062965ef2ac84347dc24 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -176,14 +176,18 @@ pub fn line_end( } pub fn previous_word_start(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { + let language = map.buffer_snapshot.language_at(point); find_preceding_boundary(map, point, |left, right| { - (char_kind(left) != char_kind(right) && !right.is_whitespace()) || left == '\n' + (char_kind(language, left) != char_kind(language, right) && !right.is_whitespace()) + || left == '\n' }) } pub fn previous_subword_start(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { + let language = map.buffer_snapshot.language_at(point); find_preceding_boundary(map, point, |left, right| { - let is_word_start = char_kind(left) != char_kind(right) && !right.is_whitespace(); + let is_word_start = + char_kind(language, left) != char_kind(language, right) && !right.is_whitespace(); let is_subword_start = left == '_' && right != '_' || left.is_lowercase() && right.is_uppercase(); is_word_start || is_subword_start || left == '\n' diff --git a/crates/editor/src/multi_buffer.rs b/crates/editor/src/multi_buffer.rs index 8417c411f243938661afdd2756b1621f60507fb8..d4061f25dc6a22b8a3abd79ae0471c811e2b08e0 100644 --- a/crates/editor/src/multi_buffer.rs +++ b/crates/editor/src/multi_buffer.rs @@ -1865,13 +1865,16 @@ impl MultiBufferSnapshot { let mut end = start; let mut next_chars = self.chars_at(start).peekable(); let mut prev_chars = self.reversed_chars_at(start).peekable(); + + let language = self.language_at(start); + let kind = |c| char_kind(language, c); let word_kind = cmp::max( - prev_chars.peek().copied().map(char_kind), - next_chars.peek().copied().map(char_kind), + prev_chars.peek().copied().map(kind), + next_chars.peek().copied().map(kind), ); for ch in prev_chars { - if Some(char_kind(ch)) == word_kind && ch != '\n' { + if Some(kind(ch)) == word_kind && ch != '\n' { start -= ch.len_utf8(); } else { break; @@ -1879,7 +1882,7 @@ impl MultiBufferSnapshot { } for ch in next_chars { - if Some(char_kind(ch)) == word_kind && ch != '\n' { + if Some(kind(ch)) == word_kind && ch != '\n' { end += ch.len_utf8(); } else { break; diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 48dc936a949c5cc47e12199d4d895b9f8a09ddf6..171df8f882e672572da57b1469843495cb459e0a 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -2173,13 +2173,16 @@ impl BufferSnapshot { let mut end = start; let mut next_chars = self.chars_at(start).peekable(); let mut prev_chars = self.reversed_chars_at(start).peekable(); + + let language = self.language_at(start); + let kind = |c| char_kind(language, c); let word_kind = cmp::max( - prev_chars.peek().copied().map(char_kind), - next_chars.peek().copied().map(char_kind), + prev_chars.peek().copied().map(kind), + next_chars.peek().copied().map(kind), ); for ch in prev_chars { - if Some(char_kind(ch)) == word_kind && ch != '\n' { + if Some(kind(ch)) == word_kind && ch != '\n' { start -= ch.len_utf8(); } else { break; @@ -2187,7 +2190,7 @@ impl BufferSnapshot { } for ch in next_chars { - if Some(char_kind(ch)) == word_kind && ch != '\n' { + if Some(kind(ch)) == word_kind && ch != '\n' { end += ch.len_utf8(); } else { break; @@ -2984,7 +2987,7 @@ pub fn contiguous_ranges( }) } -pub fn char_kind(c: char) -> CharKind { +pub fn char_kind(language: Option<&Arc>, c: char) -> CharKind { if c.is_whitespace() { CharKind::Whitespace } else if c.is_alphanumeric() || c == '_' || c == '$' { diff --git a/crates/project/src/search.rs b/crates/project/src/search.rs index 08ff803598f32d9a8fac0ab1c8e2e982130f9d09..6d9de9c974eed4a00ce45de234dab8e66a897b2e 100644 --- a/crates/project/src/search.rs +++ b/crates/project/src/search.rs @@ -39,6 +39,7 @@ pub enum SearchQuery { case_sensitive: bool, inner: SearchInputs, }, + Regex { regex: Regex, @@ -220,6 +221,8 @@ impl SearchQuery { if self.as_str().is_empty() { return Default::default(); } + let language = rope.language(cx); + let kind = |c| char_kind(language, c); let mut matches = Vec::new(); match self { @@ -236,10 +239,10 @@ impl SearchQuery { let mat = mat.unwrap(); if *whole_word { - let prev_kind = rope.reversed_chars_at(mat.start()).next().map(char_kind); - let start_kind = char_kind(rope.chars_at(mat.start()).next().unwrap()); - let end_kind = char_kind(rope.reversed_chars_at(mat.end()).next().unwrap()); - let next_kind = rope.chars_at(mat.end()).next().map(char_kind); + let prev_kind = rope.reversed_chars_at(mat.start()).next().map(kind); + let start_kind = kind(rope.chars_at(mat.start()).next().unwrap()); + let end_kind = kind(rope.reversed_chars_at(mat.end()).next().unwrap()); + let next_kind = rope.chars_at(mat.end()).next().map(kind); if Some(start_kind) == prev_kind || Some(end_kind) == next_kind { continue; } @@ -247,6 +250,7 @@ impl SearchQuery { matches.push(mat.start()..mat.end()) } } + Self::Regex { regex, multiline, .. } => { @@ -284,6 +288,7 @@ impl SearchQuery { } } } + matches } From ab5bd0ac5a0260d9616cb99450330c2315dae00e Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:39:38 +0200 Subject: [PATCH 03/12] Use new char_kind (parameterized by language) --- crates/editor/src/items.rs | 29 +++++++++++++++++------------ crates/editor/src/movement.rs | 26 ++++++++++++++++++++------ crates/language/src/buffer.rs | 14 +++++++++----- crates/language/src/language.rs | 5 ++++- crates/project/src/project.rs | 2 +- crates/project/src/search.rs | 16 +++++++++++++--- crates/vim/src/motion.rs | 18 +++++++++++------- crates/vim/src/normal/change.rs | 9 ++++++--- crates/vim/src/object.rs | 21 ++++++++++++--------- 9 files changed, 93 insertions(+), 47 deletions(-) diff --git a/crates/editor/src/items.rs b/crates/editor/src/items.rs index b99977a60eb45dc3a0a616169067063e6c4e691f..4a2b03bbdf15716b16a8aa989c302d102b88d575 100644 --- a/crates/editor/src/items.rs +++ b/crates/editor/src/items.rs @@ -1028,7 +1028,7 @@ impl SearchableItem for Editor { if let Some((_, _, excerpt_buffer)) = buffer.as_singleton() { ranges.extend( query - .search(excerpt_buffer.as_rope()) + .search(excerpt_buffer, None) .await .into_iter() .map(|range| { @@ -1038,17 +1038,22 @@ impl SearchableItem for Editor { } else { for excerpt in buffer.excerpt_boundaries_in_range(0..buffer.len()) { let excerpt_range = excerpt.range.context.to_offset(&excerpt.buffer); - let rope = excerpt.buffer.as_rope().slice(excerpt_range.clone()); - ranges.extend(query.search(&rope).await.into_iter().map(|range| { - let start = excerpt - .buffer - .anchor_after(excerpt_range.start + range.start); - let end = excerpt - .buffer - .anchor_before(excerpt_range.start + range.end); - buffer.anchor_in_excerpt(excerpt.id.clone(), start) - ..buffer.anchor_in_excerpt(excerpt.id.clone(), end) - })); + ranges.extend( + query + .search(&excerpt.buffer, Some(excerpt_range.clone())) + .await + .into_iter() + .map(|range| { + let start = excerpt + .buffer + .anchor_after(excerpt_range.start + range.start); + let end = excerpt + .buffer + .anchor_before(excerpt_range.start + range.end); + buffer.anchor_in_excerpt(excerpt.id.clone(), start) + ..buffer.anchor_in_excerpt(excerpt.id.clone(), end) + }), + ); } } ranges diff --git a/crates/editor/src/movement.rs b/crates/editor/src/movement.rs index 345b7404d9d9707be9dc062965ef2ac84347dc24..5917b8b3bdbe9f9f97e89f5567196b0e23226280 100644 --- a/crates/editor/src/movement.rs +++ b/crates/editor/src/movement.rs @@ -176,7 +176,9 @@ pub fn line_end( } pub fn previous_word_start(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { - let language = map.buffer_snapshot.language_at(point); + let raw_point = point.to_point(map); + let language = map.buffer_snapshot.language_at(raw_point); + find_preceding_boundary(map, point, |left, right| { (char_kind(language, left) != char_kind(language, right) && !right.is_whitespace()) || left == '\n' @@ -184,7 +186,8 @@ pub fn previous_word_start(map: &DisplaySnapshot, point: DisplayPoint) -> Displa } pub fn previous_subword_start(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { - let language = map.buffer_snapshot.language_at(point); + let raw_point = point.to_point(map); + let language = map.buffer_snapshot.language_at(raw_point); find_preceding_boundary(map, point, |left, right| { let is_word_start = char_kind(language, left) != char_kind(language, right) && !right.is_whitespace(); @@ -195,14 +198,20 @@ pub fn previous_subword_start(map: &DisplaySnapshot, point: DisplayPoint) -> Dis } pub fn next_word_end(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { + let raw_point = point.to_point(map); + let language = map.buffer_snapshot.language_at(raw_point); find_boundary(map, point, |left, right| { - (char_kind(left) != char_kind(right) && !left.is_whitespace()) || right == '\n' + (char_kind(language, left) != char_kind(language, right) && !left.is_whitespace()) + || right == '\n' }) } pub fn next_subword_end(map: &DisplaySnapshot, point: DisplayPoint) -> DisplayPoint { + let raw_point = point.to_point(map); + let language = map.buffer_snapshot.language_at(raw_point); find_boundary(map, point, |left, right| { - let is_word_end = (char_kind(left) != char_kind(right)) && !left.is_whitespace(); + let is_word_end = + (char_kind(language, left) != char_kind(language, right)) && !left.is_whitespace(); let is_subword_end = left != '_' && right == '_' || left.is_lowercase() && right.is_uppercase(); is_word_end || is_subword_end || right == '\n' @@ -389,10 +398,15 @@ pub fn find_boundary_in_line( } pub fn is_inside_word(map: &DisplaySnapshot, point: DisplayPoint) -> bool { + let raw_point = point.to_point(map); + let language = map.buffer_snapshot.language_at(raw_point); let ix = map.clip_point(point, Bias::Left).to_offset(map, Bias::Left); let text = &map.buffer_snapshot; - let next_char_kind = text.chars_at(ix).next().map(char_kind); - let prev_char_kind = text.reversed_chars_at(ix).next().map(char_kind); + let next_char_kind = text.chars_at(ix).next().map(|c| char_kind(language, c)); + let prev_char_kind = text + .reversed_chars_at(ix) + .next() + .map(|c| char_kind(language, c)); prev_char_kind.zip(next_char_kind) == Some((CharKind::Word, CharKind::Word)) } diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 171df8f882e672572da57b1469843495cb459e0a..8dcebd04d8e5862c300272faadc2ebc613ea719d 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -2989,12 +2989,16 @@ pub fn contiguous_ranges( pub fn char_kind(language: Option<&Arc>, c: char) -> CharKind { if c.is_whitespace() { - CharKind::Whitespace - } else if c.is_alphanumeric() || c == '_' || c == '$' { - CharKind::Word - } else { - CharKind::Punctuation + return CharKind::Whitespace; + } else if c.is_alphanumeric() || c == '_' { + return CharKind::Word; } + if let Some(language) = language { + if language.config.word_boundaries.contains(&c) { + return CharKind::Word; + } + } + CharKind::Punctuation } /// Find all of the ranges of whitespace that occur at the ends of lines diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 223f5679aed05c8a5fee158f3a14959133c9151e..f5e07a077d9a1e58f827c4d100bb85d79d3ce940 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -11,7 +11,7 @@ mod buffer_tests; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use collections::HashMap; +use collections::{HashMap, HashSet}; use futures::{ channel::oneshot, future::{BoxFuture, Shared}, @@ -344,6 +344,8 @@ pub struct LanguageConfig { pub block_comment: Option<(Arc, Arc)>, #[serde(default)] pub overrides: HashMap, + #[serde(default)] + pub word_boundaries: HashSet, } #[derive(Debug, Default)] @@ -411,6 +413,7 @@ impl Default for LanguageConfig { block_comment: Default::default(), overrides: Default::default(), collapsed_placeholder: Default::default(), + word_boundaries: Default::default(), } } } diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index 933f259700964df58a6c303a27090127f1ff2261..47248634372449a0486b32913118bdd8a8b81d27 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -5170,7 +5170,7 @@ impl Project { snapshot.file().map(|file| file.path().as_ref()), ) { query - .search(snapshot.as_rope()) + .search(&snapshot, None) .await .iter() .map(|range| { diff --git a/crates/project/src/search.rs b/crates/project/src/search.rs index 6d9de9c974eed4a00ce45de234dab8e66a897b2e..a3c6583052ada8379894720cd06cc056ce759a29 100644 --- a/crates/project/src/search.rs +++ b/crates/project/src/search.rs @@ -3,7 +3,7 @@ use anyhow::{Context, Result}; use client::proto; use globset::{Glob, GlobMatcher}; use itertools::Itertools; -use language::{char_kind, Rope}; +use language::{char_kind, BufferSnapshot}; use regex::{Regex, RegexBuilder}; use smol::future::yield_now; use std::{ @@ -215,13 +215,23 @@ impl SearchQuery { } } - pub async fn search(&self, rope: &Rope) -> Vec> { + pub async fn search( + &self, + buffer: &BufferSnapshot, + subrange: Option>, + ) -> Vec> { const YIELD_INTERVAL: usize = 20000; if self.as_str().is_empty() { return Default::default(); } - let language = rope.language(cx); + let language = buffer.language_at(0); + let rope = if let Some(range) = subrange { + buffer.as_rope().slice(range) + } else { + buffer.as_rope().clone() + }; + let kind = |c| char_kind(language, c); let mut matches = Vec::new(); diff --git a/crates/vim/src/motion.rs b/crates/vim/src/motion.rs index acf9d46ad3e66ff992ce717d00fbc39bd85bbd1b..1defee70da3e33502b8247e20e75554ff0f19bbc 100644 --- a/crates/vim/src/motion.rs +++ b/crates/vim/src/motion.rs @@ -439,11 +439,12 @@ pub(crate) fn next_word_start( ignore_punctuation: bool, times: usize, ) -> DisplayPoint { + let language = map.buffer_snapshot.language_at(point.to_point(map)); for _ in 0..times { let mut crossed_newline = false; point = movement::find_boundary(map, point, |left, right| { - let left_kind = char_kind(left).coerce_punctuation(ignore_punctuation); - let right_kind = char_kind(right).coerce_punctuation(ignore_punctuation); + let left_kind = char_kind(language, left).coerce_punctuation(ignore_punctuation); + let right_kind = char_kind(language, right).coerce_punctuation(ignore_punctuation); let at_newline = right == '\n'; let found = (left_kind != right_kind && right_kind != CharKind::Whitespace) @@ -463,11 +464,12 @@ fn next_word_end( ignore_punctuation: bool, times: usize, ) -> DisplayPoint { + let language = map.buffer_snapshot.language_at(point.to_point(map)); for _ in 0..times { *point.column_mut() += 1; point = movement::find_boundary(map, point, |left, right| { - let left_kind = char_kind(left).coerce_punctuation(ignore_punctuation); - let right_kind = char_kind(right).coerce_punctuation(ignore_punctuation); + let left_kind = char_kind(language, left).coerce_punctuation(ignore_punctuation); + let right_kind = char_kind(language, right).coerce_punctuation(ignore_punctuation); left_kind != right_kind && left_kind != CharKind::Whitespace }); @@ -493,12 +495,13 @@ fn previous_word_start( ignore_punctuation: bool, times: usize, ) -> DisplayPoint { + let language = map.buffer_snapshot.language_at(point.to_point(map)); for _ in 0..times { // This works even though find_preceding_boundary is called for every character in the line containing // cursor because the newline is checked only once. point = movement::find_preceding_boundary(map, point, |left, right| { - let left_kind = char_kind(left).coerce_punctuation(ignore_punctuation); - let right_kind = char_kind(right).coerce_punctuation(ignore_punctuation); + let left_kind = char_kind(language, left).coerce_punctuation(ignore_punctuation); + let right_kind = char_kind(language, right).coerce_punctuation(ignore_punctuation); (left_kind != right_kind && !right.is_whitespace()) || left == '\n' }); @@ -508,6 +511,7 @@ fn previous_word_start( fn first_non_whitespace(map: &DisplaySnapshot, from: DisplayPoint) -> DisplayPoint { let mut last_point = DisplayPoint::new(from.row(), 0); + let language = map.buffer_snapshot.language_at(from.to_point(map)); for (ch, point) in map.chars_at(last_point) { if ch == '\n' { return from; @@ -515,7 +519,7 @@ fn first_non_whitespace(map: &DisplaySnapshot, from: DisplayPoint) -> DisplayPoi last_point = point; - if char_kind(ch) != CharKind::Whitespace { + if char_kind(language, ch) != CharKind::Whitespace { break; } } diff --git a/crates/vim/src/normal/change.rs b/crates/vim/src/normal/change.rs index d226c704108b6bd036c08f1700cbb7812a3c1c51..50bc049a3aa96d37ae9acce6a1505369333bf534 100644 --- a/crates/vim/src/normal/change.rs +++ b/crates/vim/src/normal/change.rs @@ -82,16 +82,19 @@ fn expand_changed_word_selection( ignore_punctuation: bool, ) -> bool { if times.is_none() || times.unwrap() == 1 { + let language = map + .buffer_snapshot + .language_at(selection.start.to_point(map)); let in_word = map .chars_at(selection.head()) .next() - .map(|(c, _)| char_kind(c) != CharKind::Whitespace) + .map(|(c, _)| char_kind(language, c) != CharKind::Whitespace) .unwrap_or_default(); if in_word { selection.end = movement::find_boundary(map, selection.end, |left, right| { - let left_kind = char_kind(left).coerce_punctuation(ignore_punctuation); - let right_kind = char_kind(right).coerce_punctuation(ignore_punctuation); + let left_kind = char_kind(language, left).coerce_punctuation(ignore_punctuation); + let right_kind = char_kind(language, right).coerce_punctuation(ignore_punctuation); left_kind != right_kind && left_kind != CharKind::Whitespace }); diff --git a/crates/vim/src/object.rs b/crates/vim/src/object.rs index 85e6eab6927a8370ea9646fa43baa383bf13e8b0..d0bcad36c22a5e8775418d571c46d5bf7ab38883 100644 --- a/crates/vim/src/object.rs +++ b/crates/vim/src/object.rs @@ -122,17 +122,18 @@ fn in_word( ignore_punctuation: bool, ) -> Option> { // Use motion::right so that we consider the character under the cursor when looking for the start + let language = map.buffer_snapshot.language_at(relative_to.to_point(map)); let start = movement::find_preceding_boundary_in_line( map, right(map, relative_to, 1), |left, right| { - char_kind(left).coerce_punctuation(ignore_punctuation) - != char_kind(right).coerce_punctuation(ignore_punctuation) + char_kind(language, left).coerce_punctuation(ignore_punctuation) + != char_kind(language, right).coerce_punctuation(ignore_punctuation) }, ); let end = movement::find_boundary_in_line(map, relative_to, |left, right| { - char_kind(left).coerce_punctuation(ignore_punctuation) - != char_kind(right).coerce_punctuation(ignore_punctuation) + char_kind(language, left).coerce_punctuation(ignore_punctuation) + != char_kind(language, right).coerce_punctuation(ignore_punctuation) }); Some(start..end) @@ -155,10 +156,11 @@ fn around_word( relative_to: DisplayPoint, ignore_punctuation: bool, ) -> Option> { + let language = map.buffer_snapshot.language_at(relative_to.to_point(map)); let in_word = map .chars_at(relative_to) .next() - .map(|(c, _)| char_kind(c) != CharKind::Whitespace) + .map(|(c, _)| char_kind(language, c) != CharKind::Whitespace) .unwrap_or(false); if in_word { @@ -182,20 +184,21 @@ fn around_next_word( relative_to: DisplayPoint, ignore_punctuation: bool, ) -> Option> { + let language = map.buffer_snapshot.language_at(relative_to.to_point(map)); // Get the start of the word let start = movement::find_preceding_boundary_in_line( map, right(map, relative_to, 1), |left, right| { - char_kind(left).coerce_punctuation(ignore_punctuation) - != char_kind(right).coerce_punctuation(ignore_punctuation) + char_kind(language, left).coerce_punctuation(ignore_punctuation) + != char_kind(language, right).coerce_punctuation(ignore_punctuation) }, ); let mut word_found = false; let end = movement::find_boundary(map, relative_to, |left, right| { - let left_kind = char_kind(left).coerce_punctuation(ignore_punctuation); - let right_kind = char_kind(right).coerce_punctuation(ignore_punctuation); + let left_kind = char_kind(language, left).coerce_punctuation(ignore_punctuation); + let right_kind = char_kind(language, right).coerce_punctuation(ignore_punctuation); let found = (word_found && left_kind != right_kind) || right == '\n' && left == '\n'; From 00caad2f1774e04c747d0d962915d1d3a72ac293 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:41:05 +0200 Subject: [PATCH 04/12] ..and use it in PHP language config --- crates/zed/src/languages/php/config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/zed/src/languages/php/config.toml b/crates/zed/src/languages/php/config.toml index 19acb949e25c66be9890f9931f49a8e82f8bb972..be97b80e5eaba75555f9e0f86b9253b0d15f67e5 100644 --- a/crates/zed/src/languages/php/config.toml +++ b/crates/zed/src/languages/php/config.toml @@ -10,3 +10,4 @@ brackets = [ { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] }, ] collapsed_placeholder = "/* ... */" +word_boundaries = ["$"] From 046759a366a3ed65ae3c0deacbd880ffbfe291f9 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 01:01:27 +0200 Subject: [PATCH 05/12] Remove dead comment --- crates/editor/src/editor.rs | 2 - crates/gpui_platform/src/tests/mod.rs | 125 ++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 crates/gpui_platform/src/tests/mod.rs diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 60576617eba288b7df8ffc9d23c5f36485524247..cbc7a7cd42bea007acb56d22cf4401f600eefa97 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -2667,8 +2667,6 @@ impl Editor { false }); } - // $language = true; - // language fn completion_query(buffer: &MultiBufferSnapshot, position: impl ToOffset) -> Option { let offset = position.to_offset(buffer); let (word_range, kind) = buffer.surrounding_word(offset); diff --git a/crates/gpui_platform/src/tests/mod.rs b/crates/gpui_platform/src/tests/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..949c9c3e275de58a7dfc861c09179af1fbd32315 --- /dev/null +++ b/crates/gpui_platform/src/tests/mod.rs @@ -0,0 +1,125 @@ + + + use gpui::text_layout::*; + use gpui::fonts::{Properties, Weight}; + + #[gpui::test] + fn test_wrap_line(cx: &mut gpui::AppContext) { + let font_cache = cx.font_cache().clone(); + let font_system = cx.platform().fonts(); + let family = font_cache + .load_family(&["Courier"], &Default::default()) + .unwrap(); + let font_id = font_cache.select_font(family, &Default::default()).unwrap(); + + let mut wrapper = LineWrapper::new(font_id, 16., font_system); + assert_eq!( + wrapper + .wrap_line("aa bbb cccc ddddd eeee", 72.0) + .collect::>(), + &[ + Boundary::new(7, 0), + Boundary::new(12, 0), + Boundary::new(18, 0) + ], + ); + assert_eq!( + wrapper + .wrap_line("aaa aaaaaaaaaaaaaaaaaa", 72.0) + .collect::>(), + &[ + Boundary::new(4, 0), + Boundary::new(11, 0), + Boundary::new(18, 0) + ], + ); + assert_eq!( + wrapper.wrap_line(" aaaaaaa", 72.).collect::>(), + &[ + Boundary::new(7, 5), + Boundary::new(9, 5), + Boundary::new(11, 5), + ] + ); + assert_eq!( + wrapper + .wrap_line(" ", 72.) + .collect::>(), + &[ + Boundary::new(7, 0), + Boundary::new(14, 0), + Boundary::new(21, 0) + ] + ); + assert_eq!( + wrapper + .wrap_line(" aaaaaaaaaaaaaa", 72.) + .collect::>(), + &[ + Boundary::new(7, 0), + Boundary::new(14, 3), + Boundary::new(18, 3), + Boundary::new(22, 3), + ] + ); + } + + #[gpui::test(retries = 5)] + fn test_wrap_shaped_line(cx: &mut gpui::AppContext) { + // This is failing intermittently on CI and we don't have time to figure it out + let font_cache = cx.font_cache().clone(); + let font_system = cx.platform().fonts(); + let text_layout_cache = TextLayoutCache::new(font_system.clone()); + + let family = font_cache + .load_family(&["Helvetica"], &Default::default()) + .unwrap(); + let font_id = font_cache.select_font(family, &Default::default()).unwrap(); + let normal = RunStyle { + font_id, + color: Default::default(), + underline: Default::default(), + }; + let bold = RunStyle { + font_id: font_cache + .select_font( + family, + &Properties { + weight: Weight::BOLD, + ..Default::default() + }, + ) + .unwrap(), + color: Default::default(), + underline: Default::default(), + }; + + let text = "aa bbb cccc ddddd eeee"; + let line = text_layout_cache.layout_str( + text, + 16.0, + &[(4, normal), (5, bold), (6, normal), (1, bold), (7, normal)], + ); + + let mut wrapper = LineWrapper::new(font_id, 16., font_system); + assert_eq!( + wrapper + .wrap_shaped_line(text, &line, 72.0) + .collect::>(), + &[ + ShapedBoundary { + run_ix: 1, + glyph_ix: 3 + }, + ShapedBoundary { + run_ix: 2, + glyph_ix: 3 + }, + ShapedBoundary { + run_ix: 4, + glyph_ix: 2 + } + ], + ); + +} From 344a09a4f878381ebacfb12be4c42b7ff38ba80e Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 01:02:25 +0200 Subject: [PATCH 06/12] Rename word_boundaries to word_characters --- crates/language/src/buffer.rs | 2 +- crates/language/src/language.rs | 4 ++-- crates/zed/src/languages/php/config.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index ad457c3236e05d6ee430c46ddfe2600b6596da6e..d032e8e0253c81681f3c9c11576185f5f04ce5e9 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -3013,7 +3013,7 @@ pub fn char_kind(language: Option<&Arc>, c: char) -> CharKind { return CharKind::Word; } if let Some(language) = language { - if language.config.word_boundaries.contains(&c) { + if language.config.word_characters.contains(&c) { return CharKind::Word; } } diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index f5e07a077d9a1e58f827c4d100bb85d79d3ce940..82245d67ca0487adb33aac8ec124f658948c3509 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -345,7 +345,7 @@ pub struct LanguageConfig { #[serde(default)] pub overrides: HashMap, #[serde(default)] - pub word_boundaries: HashSet, + pub word_characters: HashSet, } #[derive(Debug, Default)] @@ -413,7 +413,7 @@ impl Default for LanguageConfig { block_comment: Default::default(), overrides: Default::default(), collapsed_placeholder: Default::default(), - word_boundaries: Default::default(), + word_characters: Default::default(), } } } diff --git a/crates/zed/src/languages/php/config.toml b/crates/zed/src/languages/php/config.toml index be97b80e5eaba75555f9e0f86b9253b0d15f67e5..60dd2335551bc746ec6b26f2afe039b377e18442 100644 --- a/crates/zed/src/languages/php/config.toml +++ b/crates/zed/src/languages/php/config.toml @@ -10,4 +10,4 @@ brackets = [ { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] }, ] collapsed_placeholder = "/* ... */" -word_boundaries = ["$"] +word_characters = ["$"] From bca2d02a61b0d3710c4ca7fbd8af745978ae6d6c Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 01:04:15 +0200 Subject: [PATCH 07/12] Revert "Remove dead comment" This reverts commit 046759a366a3ed65ae3c0deacbd880ffbfe291f9. --- crates/editor/src/editor.rs | 2 + crates/gpui_platform/src/tests/mod.rs | 125 -------------------------- 2 files changed, 2 insertions(+), 125 deletions(-) delete mode 100644 crates/gpui_platform/src/tests/mod.rs diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index cbc7a7cd42bea007acb56d22cf4401f600eefa97..60576617eba288b7df8ffc9d23c5f36485524247 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -2667,6 +2667,8 @@ impl Editor { false }); } + // $language = true; + // language fn completion_query(buffer: &MultiBufferSnapshot, position: impl ToOffset) -> Option { let offset = position.to_offset(buffer); let (word_range, kind) = buffer.surrounding_word(offset); diff --git a/crates/gpui_platform/src/tests/mod.rs b/crates/gpui_platform/src/tests/mod.rs deleted file mode 100644 index 949c9c3e275de58a7dfc861c09179af1fbd32315..0000000000000000000000000000000000000000 --- a/crates/gpui_platform/src/tests/mod.rs +++ /dev/null @@ -1,125 +0,0 @@ - - - use gpui::text_layout::*; - use gpui::fonts::{Properties, Weight}; - - #[gpui::test] - fn test_wrap_line(cx: &mut gpui::AppContext) { - let font_cache = cx.font_cache().clone(); - let font_system = cx.platform().fonts(); - let family = font_cache - .load_family(&["Courier"], &Default::default()) - .unwrap(); - let font_id = font_cache.select_font(family, &Default::default()).unwrap(); - - let mut wrapper = LineWrapper::new(font_id, 16., font_system); - assert_eq!( - wrapper - .wrap_line("aa bbb cccc ddddd eeee", 72.0) - .collect::>(), - &[ - Boundary::new(7, 0), - Boundary::new(12, 0), - Boundary::new(18, 0) - ], - ); - assert_eq!( - wrapper - .wrap_line("aaa aaaaaaaaaaaaaaaaaa", 72.0) - .collect::>(), - &[ - Boundary::new(4, 0), - Boundary::new(11, 0), - Boundary::new(18, 0) - ], - ); - assert_eq!( - wrapper.wrap_line(" aaaaaaa", 72.).collect::>(), - &[ - Boundary::new(7, 5), - Boundary::new(9, 5), - Boundary::new(11, 5), - ] - ); - assert_eq!( - wrapper - .wrap_line(" ", 72.) - .collect::>(), - &[ - Boundary::new(7, 0), - Boundary::new(14, 0), - Boundary::new(21, 0) - ] - ); - assert_eq!( - wrapper - .wrap_line(" aaaaaaaaaaaaaa", 72.) - .collect::>(), - &[ - Boundary::new(7, 0), - Boundary::new(14, 3), - Boundary::new(18, 3), - Boundary::new(22, 3), - ] - ); - } - - #[gpui::test(retries = 5)] - fn test_wrap_shaped_line(cx: &mut gpui::AppContext) { - // This is failing intermittently on CI and we don't have time to figure it out - let font_cache = cx.font_cache().clone(); - let font_system = cx.platform().fonts(); - let text_layout_cache = TextLayoutCache::new(font_system.clone()); - - let family = font_cache - .load_family(&["Helvetica"], &Default::default()) - .unwrap(); - let font_id = font_cache.select_font(family, &Default::default()).unwrap(); - let normal = RunStyle { - font_id, - color: Default::default(), - underline: Default::default(), - }; - let bold = RunStyle { - font_id: font_cache - .select_font( - family, - &Properties { - weight: Weight::BOLD, - ..Default::default() - }, - ) - .unwrap(), - color: Default::default(), - underline: Default::default(), - }; - - let text = "aa bbb cccc ddddd eeee"; - let line = text_layout_cache.layout_str( - text, - 16.0, - &[(4, normal), (5, bold), (6, normal), (1, bold), (7, normal)], - ); - - let mut wrapper = LineWrapper::new(font_id, 16., font_system); - assert_eq!( - wrapper - .wrap_shaped_line(text, &line, 72.0) - .collect::>(), - &[ - ShapedBoundary { - run_ix: 1, - glyph_ix: 3 - }, - ShapedBoundary { - run_ix: 2, - glyph_ix: 3 - }, - ShapedBoundary { - run_ix: 4, - glyph_ix: 2 - } - ], - ); - -} From 42b0c5dfdd0d03379c6efa0e2ab219f12337b5e3 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 01:04:55 +0200 Subject: [PATCH 08/12] Remove comment;for real now --- crates/editor/src/editor.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 60576617eba288b7df8ffc9d23c5f36485524247..cbc7a7cd42bea007acb56d22cf4401f600eefa97 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -2667,8 +2667,6 @@ impl Editor { false }); } - // $language = true; - // language fn completion_query(buffer: &MultiBufferSnapshot, position: impl ToOffset) -> Option { let offset = position.to_offset(buffer); let (word_range, kind) = buffer.surrounding_word(offset); From 14fa996cdc572c4d48e68249d6afbcd9304bab65 Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Mon, 21 Aug 2023 22:36:58 -0600 Subject: [PATCH 09/12] Add # and $ for js --- crates/zed/src/languages/javascript/config.toml | 1 + crates/zed/src/languages/tsx/config.toml | 1 + crates/zed/src/languages/typescript/config.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/crates/zed/src/languages/javascript/config.toml b/crates/zed/src/languages/javascript/config.toml index 8f4670388edc40e4ff9a1bf068d4e2b945000bb2..0435f96c921fb28843e08078e34c2cebc1c33cdd 100644 --- a/crates/zed/src/languages/javascript/config.toml +++ b/crates/zed/src/languages/javascript/config.toml @@ -13,6 +13,7 @@ brackets = [ { start = "`", end = "`", close = true, newline = false, not_in = ["comment", "string"] }, { start = "/*", end = " */", close = true, newline = false, not_in = ["comment", "string"] }, ] +word_characters = ["$", "#"] [overrides.element] line_comment = { remove = true } diff --git a/crates/zed/src/languages/tsx/config.toml b/crates/zed/src/languages/tsx/config.toml index 234dc6b01326a8398b5ab5dfbdfec93a37d910ad..63d1f85e643cf2c19b6b43f5f86ae89976b7bd74 100644 --- a/crates/zed/src/languages/tsx/config.toml +++ b/crates/zed/src/languages/tsx/config.toml @@ -12,6 +12,7 @@ brackets = [ { start = "`", end = "`", close = true, newline = false, not_in = ["string"] }, { start = "/*", end = " */", close = true, newline = false, not_in = ["string", "comment"] }, ] +word_characters = ["#", "$"] [overrides.element] line_comment = { remove = true } diff --git a/crates/zed/src/languages/typescript/config.toml b/crates/zed/src/languages/typescript/config.toml index a2b764d9fe38e3058adfbdf292c52f667acae264..2fad1f13e182c52e2886e459fd0efa6fb673a7d6 100644 --- a/crates/zed/src/languages/typescript/config.toml +++ b/crates/zed/src/languages/typescript/config.toml @@ -12,3 +12,4 @@ brackets = [ { start = "`", end = "`", close = true, newline = false, not_in = ["string"] }, { start = "/*", end = " */", close = true, newline = false, not_in = ["string", "comment"] }, ] +word_characters = ["#", "$"] From 168a213a446b0b40d8ff6bf39bda98ee7f13dbfb Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Mon, 21 Aug 2023 22:37:14 -0600 Subject: [PATCH 10/12] Add test for word characters in vim --- .../src/test/editor_lsp_test_context.rs | 5 ++++ crates/vim/src/test.rs | 26 +++++++++++++++++++ crates/vim/src/test/vim_test_context.rs | 12 ++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/editor/src/test/editor_lsp_test_context.rs b/crates/editor/src/test/editor_lsp_test_context.rs index 83aaa3b703491b5f5334d173ca8d90e731aeb5e8..4d0e9c1d2af8b27fcd796205d83f37ee9a931503 100644 --- a/crates/editor/src/test/editor_lsp_test_context.rs +++ b/crates/editor/src/test/editor_lsp_test_context.rs @@ -6,6 +6,7 @@ use std::{ use anyhow::Result; +use collections::HashSet; use futures::Future; use gpui::{json, ViewContext, ViewHandle}; use indoc::indoc; @@ -154,10 +155,14 @@ impl<'a> EditorLspTestContext<'a> { capabilities: lsp::ServerCapabilities, cx: &'a mut gpui::TestAppContext, ) -> EditorLspTestContext<'a> { + let mut word_characters: HashSet = Default::default(); + word_characters.insert('$'); + word_characters.insert('#'); let language = Language::new( LanguageConfig { name: "Typescript".into(), path_suffixes: vec!["ts".to_string()], + word_characters, ..Default::default() }, Some(tree_sitter_typescript::language_typescript()), diff --git a/crates/vim/src/test.rs b/crates/vim/src/test.rs index 772d7a20339632faaf64d3a21363432d48af45af..3b1b8c24f2d301f5f843bfe56754a3d3d07233f9 100644 --- a/crates/vim/src/test.rs +++ b/crates/vim/src/test.rs @@ -261,3 +261,29 @@ async fn test_status_indicator( assert!(mode_indicator.read(cx).mode.is_some()); }); } + +#[gpui::test] +async fn test_word_characters(cx: &mut gpui::TestAppContext) { + let mut cx = VimTestContext::new_typescript(cx).await; + cx.set_state( + indoc! { " + class A { + #ˇgoop = 99; + $ˇgoop () { return this.#gˇoop }; + }; + console.log(new A().$gooˇp()) + "}, + Mode::Normal, + ); + cx.simulate_keystrokes(["v", "i", "w"]); + cx.assert_state( + indoc! {" + class A { + «#goopˇ» = 99; + «$goopˇ» () { return this.«#goopˇ» }; + }; + console.log(new A().«$goopˇ»()) + "}, + Mode::Visual, + ) +} diff --git a/crates/vim/src/test/vim_test_context.rs b/crates/vim/src/test/vim_test_context.rs index 56193723d9500d1a88954bc82bd45d6e917ba9a7..24fb16fd3d736f4afec804ce8752c1c7e42f8232 100644 --- a/crates/vim/src/test/vim_test_context.rs +++ b/crates/vim/src/test/vim_test_context.rs @@ -16,8 +16,18 @@ pub struct VimTestContext<'a> { impl<'a> VimTestContext<'a> { pub async fn new(cx: &'a mut gpui::TestAppContext, enabled: bool) -> VimTestContext<'a> { - let mut cx = EditorLspTestContext::new_rust(Default::default(), cx).await; + let lsp = EditorLspTestContext::new_rust(Default::default(), cx).await; + Self::new_with_lsp(lsp, enabled) + } + + pub async fn new_typescript(cx: &'a mut gpui::TestAppContext) -> VimTestContext<'a> { + Self::new_with_lsp( + EditorLspTestContext::new_typescript(Default::default(), cx).await, + true, + ) + } + pub fn new_with_lsp(mut cx: EditorLspTestContext<'a>, enabled: bool) -> VimTestContext<'a> { cx.update(|cx| { search::init(cx); crate::init(cx); From ccb9b5d27856204542c70475dcd0740ea27a3d24 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 18:12:05 +0200 Subject: [PATCH 11/12] Query char_kind for completion triggers. Co-authored-by: Conrad Irwin --- crates/editor/src/multi_buffer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/editor/src/multi_buffer.rs b/crates/editor/src/multi_buffer.rs index d4061f25dc6a22b8a3abd79ae0471c811e2b08e0..510f591655c9cc16a6f97af8a603d5039575c691 100644 --- a/crates/editor/src/multi_buffer.rs +++ b/crates/editor/src/multi_buffer.rs @@ -1346,9 +1346,7 @@ impl MultiBuffer { .map(|state| state.buffer.clone()) } - pub fn is_completion_trigger(&self, position: T, text: &str, cx: &AppContext) -> bool - where - T: ToOffset, + pub fn is_completion_trigger(&self, position: Anchor, text: &str, cx: &AppContext) -> bool { let mut chars = text.chars(); let char = if let Some(char) = chars.next() { @@ -1360,7 +1358,9 @@ impl MultiBuffer { return false; } - if char.is_alphanumeric() || char == '_' { + let language = self.language_at(position.clone(), cx); + + if char_kind(language.as_ref(), char) == CharKind::Word { return true; } From ccb3f6748c99df09598cfd2b78288718af1bb807 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 22 Aug 2023 18:14:07 +0200 Subject: [PATCH 12/12] chore: fmt --- crates/editor/src/multi_buffer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/editor/src/multi_buffer.rs b/crates/editor/src/multi_buffer.rs index 510f591655c9cc16a6f97af8a603d5039575c691..9dd40af8981dab6d82ba4dda237ca46804575519 100644 --- a/crates/editor/src/multi_buffer.rs +++ b/crates/editor/src/multi_buffer.rs @@ -1346,8 +1346,7 @@ impl MultiBuffer { .map(|state| state.buffer.clone()) } - pub fn is_completion_trigger(&self, position: Anchor, text: &str, cx: &AppContext) -> bool - { + pub fn is_completion_trigger(&self, position: Anchor, text: &str, cx: &AppContext) -> bool { let mut chars = text.chars(); let char = if let Some(char) = chars.next() { char